diff --git a/.github/actions/download-dl4j-test-resources-linux/action.yml b/.github/actions/download-dl4j-test-resources-linux/action.yml new file mode 100644 index 000000000..81d8266b7 --- /dev/null +++ b/.github/actions/download-dl4j-test-resources-linux/action.yml @@ -0,0 +1,11 @@ +name: Download dl4j test resources +runs: + using: composite + steps: + - name: Initial install + shell: bash + run: | + wget https://github.com/KonduitAI/dl4j-test-resources/archive/master.zip && unzip master.zip + cd dl4j-test-resources-master + mvn clean install -DskipTests + echo "Extracted test resources" \ No newline at end of file diff --git a/.github/actions/download-dl4j-test-resources-windows/action.yml b/.github/actions/download-dl4j-test-resources-windows/action.yml new file mode 100644 index 000000000..5683619e3 --- /dev/null +++ b/.github/actions/download-dl4j-test-resources-windows/action.yml @@ -0,0 +1,12 @@ +name: Download dl4j test resources +runs: + using: composite + steps: + - name: Initial install + shell: cmd + run: | + set "PATH=C:\msys64\usr\bin;%PATH%" + wget https://github.com/KonduitAI/dl4j-test-resources/archive/master.zip && unzip master.zip + cd dl4j-test-resources-master + mvn clean install -DskipTests + echo "Extracted test resources" \ No newline at end of file diff --git a/.github/actions/install-arm-cross-compile/action.yml b/.github/actions/install-arm-cross-compile/action.yml new file mode 100644 index 000000000..dcf52c4ab --- /dev/null +++ b/.github/actions/install-arm-cross-compile/action.yml @@ -0,0 +1,12 @@ +name: Download dl4j test resources +runs: + using: composite + steps: + - name: Initial install + shell: bash + run: | + sudo apt install git gcc-8-aarch64-linux-gnu g++-8-aarch64-linux-gnu libc6-armel-cross libc6-dev-armel-cross binutils-arm-linux-gnueabi libncurses5-dev build-essential bison flex libssl-dev bc \ + gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf crossbuild-essential-arm64 + mkdir -p /opt/raspberrypi && \ + cd /opt/raspberrypi && \ + git clone git://github.com/raspberrypi/tools.git diff --git a/.github/actions/install-protobuf-linux/action.yml b/.github/actions/install-protobuf-linux/action.yml new file mode 100644 index 000000000..f48a953b1 --- /dev/null +++ b/.github/actions/install-protobuf-linux/action.yml @@ -0,0 +1,16 @@ +name: Install protobuf linux +runs: + using: composite + steps: + - name: Install protobuf linux + shell: bash + run: | + curl -fsSL https://github.com/google/protobuf/releases/download/v3.5.1/protobuf-cpp-3.5.1.tar.gz \ + | tar xz && \ + cd protobuf-3.5.1 && \ + ./configure --prefix=/opt/protobuf && \ + make -j2 && \ + make install && \ + cd .. && \ + rm -rf protobuf-3.5.1 + echo "/opt/protobuf/bin" >> $GITHUB_PATH \ No newline at end of file diff --git a/.github/actions/msys2-base-setup/action.yml b/.github/actions/msys2-base-setup/action.yml new file mode 100644 index 000000000..272478d81 --- /dev/null +++ b/.github/actions/msys2-base-setup/action.yml @@ -0,0 +1,10 @@ +name: Setup for msys2 +runs: + using: composite + steps: + - name: Initial install + shell: cmd + run: | + C:\msys64\usr\bin\bash -lc "pacman -S --needed --noconfirm base-devel git tar pkg-config unzip p7zip zip autoconf autoconf-archive automake make patch gnupg" + C:\msys64\usr\bin\bash -lc "pacman -S --needed --noconfirm mingw-w64-x86_64-nasm mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool mingw-w64-x86_64-gcc mingw-w64-i686-gcc mingw-w64-x86_64-gcc-fortran mingw-w64-i686-gcc-fortran mingw-w64-x86_64-libwinpthread-git mingw-w64-i686-libwinpthread-git mingw-w64-x86_64-SDL mingw-w64-i686-SDL mingw-w64-x86_64-ragel" + echo "C:\msys64\usr\bin" >> $GITHUB_PATH \ No newline at end of file diff --git a/.github/actions/publish-gh-packages/action.yml b/.github/actions/publish-gh-packages/action.yml new file mode 100644 index 000000000..cb6f1424c --- /dev/null +++ b/.github/actions/publish-gh-packages/action.yml @@ -0,0 +1,9 @@ +name: Publish to github packages +runs: + using: composite + steps: + - name: Publish to GitHub Packages + run: mvn -Pgithub --batch-mode deploy + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/build-android-x86_64.yml b/.github/workflows/build-android-x86_64.yml new file mode 100644 index 000000000..fa79452e2 --- /dev/null +++ b/.github/workflows/build-android-x86_64.yml @@ -0,0 +1,49 @@ +on: + schedule: + - cron: "0 */12 * * *" +jobs: + android-x86_64: + runs-on: ubuntu-18.04 + steps: + - uses: AutoModality/action-clean@v1 + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: nttld/setup-ndk@v1 + id: setup-ndk + with: + ndk-version: r18b + - uses: actions/checkout@v2 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Build on linux-x86_64 + env: + ANDROID_NDK: ${{ steps.setup-ndk.outputs.ndk-path }} + OPENBLAS_PATH: "/opt/openblas/" + LIBND4J_HOME: "${GITHUB_WORKSPACE}/libnd4j" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + echo "Verifying programs on path. Path is $PATH" + echo "Path post update is $PATH. Maven is at `which mvn` cmake is at `which cmake` protoc is at `which protoc`" + mvn --version + cmake --version + protoc --version + clang --version + mvn -X -Dorg.bytedeco.javacpp.logger.debug=true -Possrh -pl ":nd4j-native,:libnd4j" --also-make \ + -Djavacpp.platform=android-x86_64 \ + -Dlibnd4j.platform=android-x86_64 -Dlibnd4j.chip=cpu \ + --batch-mode clean deploy -DskipTests + + diff --git a/.github/workflows/build-deploy-android-arm32.yml b/.github/workflows/build-deploy-android-arm32.yml new file mode 100644 index 000000000..73e96ac0a --- /dev/null +++ b/.github/workflows/build-deploy-android-arm32.yml @@ -0,0 +1,44 @@ +on: + schedule: + - cron: "0 */12 * * *" +jobs: + #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. + android-arm32: + runs-on: ubuntu-18.04 + steps: + - uses: AutoModality/action-clean@v1 + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Build on android-arm32 + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEPLOY: 1 + BUILD_USING_MAVEN: 1 + TARGET_OS: android + CURRENT_TARGET: arm32 + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + mvn --version + cmake --version + protoc --version + ${GITHUB_WORKSPACE}/libnd4j/pi_build.sh + + diff --git a/.github/workflows/build-deploy-android-arm64.yml b/.github/workflows/build-deploy-android-arm64.yml new file mode 100644 index 000000000..586806163 --- /dev/null +++ b/.github/workflows/build-deploy-android-arm64.yml @@ -0,0 +1,44 @@ +on: + schedule: + - cron: "0 */12 * * *" +jobs: + #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. + android-arm64: + runs-on: ubuntu-18.04 + steps: + - uses: AutoModality/action-clean@v1 + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + + - name: Build on android-arm64 + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEPLOY: 1 + BUILD_USING_MAVEN: 1 + TARGET_OS: android + CURRENT_TARGET: arm64 + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + mvn --version + cmake --version + protoc --version + ${GITHUB_WORKSPACE}/libnd4j/pi_build.sh + + diff --git a/.github/workflows/build-deploy-linux-arm32.yml b/.github/workflows/build-deploy-linux-arm32.yml new file mode 100644 index 000000000..90aafc1c4 --- /dev/null +++ b/.github/workflows/build-deploy-linux-arm32.yml @@ -0,0 +1,44 @@ +on: + schedule: + - cron: "0 */12 * * *" +jobs: + #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. + linux-arm32: + runs-on: ubuntu-18.04 + steps: + - uses: AutoModality/action-clean@v1 + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Build on linux-arm32 + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEPLOY: 1 + BUILD_USING_MAVEN: 1 + TARGET_OS: linux + CURRENT_TARGET: arm32 + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + mvn --version + cmake --version + protoc --version + ${GITHUB_WORKSPACE}/libnd4j/pi_build.sh + + diff --git a/.github/workflows/build-deploy-linux-arm64.yml b/.github/workflows/build-deploy-linux-arm64.yml new file mode 100644 index 000000000..f6090c52b --- /dev/null +++ b/.github/workflows/build-deploy-linux-arm64.yml @@ -0,0 +1,41 @@ +on: + schedule: + - cron: "0 */12 * * *" +jobs: + #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. + linux-arm64: + runs-on: ubuntu-18.04 + steps: + - uses: AutoModality/action-clean@v1 + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Build on linux-arm64 + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEPLOY: 1 + BUILD_USING_MAVEN: 1 + TARGET_OS: linux + CURRENT_TARGET: arm64 + PUBLISH_TO: ossrh + run: | + mvn --version + cmake --version + protoc --version + ${GITHUB_WORKSPACE}/libnd4j/pi_build.sh + + diff --git a/.github/workflows/build-deploy-linux-cuda-11.0.yml b/.github/workflows/build-deploy-linux-cuda-11.0.yml new file mode 100644 index 000000000..cc179305d --- /dev/null +++ b/.github/workflows/build-deploy-linux-cuda-11.0.yml @@ -0,0 +1,55 @@ +on: + schedule: + - cron: "0 */12 * * *" +jobs: + + linux-x86_64-cuda_11-0: + runs-on: ubuntu-18.04 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + root-reserve-mb: 512 + swap-size-mb: 8192 + remove-dotnet: 'true' + remove-haskell: 'true' + - uses: actions/checkout@v2 + - uses: konduitai/cuda-install/.github/actions/install-cuda-ubuntu@master + env: + cuda: 11.0.167 + GCC: 9 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + + - name: Build cuda + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + export PATH="/usr/local/cuda-11.0/bin:$PATH" + mvn --version + cmake --version + protoc --version + nvcc --version + sudo apt-get autoremove + sudo apt-get clean + mvn -Possrh -Djavacpp.platform=linux-x86_64 -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -Dlibnd4j.chip=cuda -pl ":nd4j-cuda-11.0,:deeplearning4j-cuda-11.0,:libnd4j" --also-make -Pcuda clean --batch-mode deploy -DskipTests + + diff --git a/.github/workflows/build-deploy-linux-cuda-11.2.yml b/.github/workflows/build-deploy-linux-cuda-11.2.yml new file mode 100644 index 000000000..958b9e0e5 --- /dev/null +++ b/.github/workflows/build-deploy-linux-cuda-11.2.yml @@ -0,0 +1,52 @@ +on: + schedule: + - cron: "0 */12 * * *" +jobs: + linux-x86_64-cuda-11-2: + runs-on: ubuntu-18.04 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + root-reserve-mb: 512 + swap-size-mb: 8192 + remove-dotnet: 'true' + remove-haskell: 'true' + - uses: actions/checkout@v2 + - uses: konduitai/cuda-install/.github/actions/install-cuda-ubuntu@master + env: + cuda: 11.2.1_461 + GCC: 9 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Run cuda compilation on linux-x86_64 + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + export PATH="/usr/local/cuda-11.2/bin:$PATH" + nvcc --version + mvn --version + cmake --version + protoc --version + sudo apt-get autoremove + sudo apt-get clean + bash ./change-cuda-versions.sh 11.2 + mvn -Possrh -Djavacpp.platform=linux-x86_64 -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -pl ":nd4j-cuda-11.2,:deeplearning4j-cuda-11.2,:libnd4j" --also-make -Dlibnd4j.chip=cuda --batch-mode deploy -DskipTests diff --git a/.github/workflows/build-deploy-linux-x86_64.yml b/.github/workflows/build-deploy-linux-x86_64.yml new file mode 100644 index 000000000..6e9951dd9 --- /dev/null +++ b/.github/workflows/build-deploy-linux-x86_64.yml @@ -0,0 +1,41 @@ +on: + schedule: + - cron: "0 */12 * * *" +jobs: + #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. + linux-x86_64: + runs-on: ubuntu-18.04 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Build on linux-x86_64 + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + mvn --version + cmake --version + protoc --version + sudo apt-get autoremove + sudo apt-get clean + mvn -X -Possrh -Djavacpp.platform=linux-x86_64 -Dlibnd4j.chip=cpu -Pcpu --batch-mode deploy -DskipTests + + diff --git a/.github/workflows/build-deploy-mac.yml b/.github/workflows/build-deploy-mac.yml new file mode 100644 index 000000000..d21646bda --- /dev/null +++ b/.github/workflows/build-deploy-mac.yml @@ -0,0 +1,34 @@ +on: + schedule: + - cron: "0 */12 * * *" +jobs: + mac-x86_64: + runs-on: macos-10.15 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Build and install + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + brew install unzip ccache gcc swig autoconf-archive automake cmake libomp libtool libusb ant maven nasm xz pkg-config sdl gpg1 bison flex perl ragel binutils gradle gmp isl libmpc mpfr wget python + mvn -Possrh -Djavacpp.platform=macosx-x86_64 -Djavacpp.platform=macosx-x86_64 -pl ":nd4j-native,:libnd4j" --also-make -Dlibnd4j.platform=macosx-x86_64 -Dlibnd4j.chip=cpu clean --batch-mode deploy -DskipTests + + diff --git a/.github/workflows/build-deploy-windows-cuda-11.0.yml b/.github/workflows/build-deploy-windows-cuda-11.0.yml new file mode 100644 index 000000000..1c1f53691 --- /dev/null +++ b/.github/workflows/build-deploy-windows-cuda-11.0.yml @@ -0,0 +1,46 @@ +on: + schedule: + - cron: "0 */12 * * *" +jobs: + windows-x86_64-cuda-11-0: + runs-on: windows-2019 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - name: Use existing msys2 to setup environment + uses: ./.github/actions/msys2-base-setup + - uses: konduitai/cuda-install/.github/actions/install-cuda-windows@master + env: + cuda: 11.0.167 + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Run windows build + shell: cmd + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + set MSYSTEM=MINGW64 + set "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0" + which cmake + dir "%CUDA_PATH%" + dir "%CUDA_PATH%\lib" + set "PATH=C:\msys64\usr\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0\lib\x64;%PATH%" + echo "Running cuda build" + mvn -Possrh -Djavacpp.platform=windows-x86_64 -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -Djavacpp.platform=windows-x86_64 -pl ":nd4j-cuda-11.0,:deeplearning4j-cuda-11.0,:libnd4j" --also-make -Dlibnd4j.platform=windows-x86_64 -Pcuda -Dlibnd4j.chip=cuda -Pcuda clean --batch-mode deploy -DskipTests + + diff --git a/.github/workflows/build-deploy-windows-cuda-11.2.yml b/.github/workflows/build-deploy-windows-cuda-11.2.yml new file mode 100644 index 000000000..d89ecfcec --- /dev/null +++ b/.github/workflows/build-deploy-windows-cuda-11.2.yml @@ -0,0 +1,48 @@ +on: + schedule: + - cron: "0 */12 * * *" +jobs: + windows-x86_64-cuda-11-2: + runs-on: windows-2019 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - name: Use existing msys2 to setup environment + uses: ./.github/actions/msys2-base-setup + - uses: konduitai/cuda-install/.github/actions/install-cuda-windows@master + env: + cuda: 11.2.1 + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Run cuda build + shell: cmd + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + set MSYSTEM=MINGW64 + set "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2" + dir "%CUDA_PATH%" + dir "%CUDA_PATH%\lib" + which cmake + set "PATH=C:\msys64\usr\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\lib\x64;%PATH%" + echo "Running cuda build" + bash ./change-cuda-versions.sh 11.2 + sudo apt-get autoremove + sudo apt-get clean + mvn -Possrh -Djavacpp.platform=linux-x86_64 -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -Djavacpp.platform=windows-x86_64 -pl ":nd4j-cuda-11.2,:libnd4j,:deeplearning4j-cuda-11.2" --also-make -Dlibnd4j.platform=windows-x86_64 -Pcuda -Dlibnd4j.chip=cuda -Pcuda clean --batch-mode deploy -DskipTests + diff --git a/.github/workflows/build-deploy-windows.yml b/.github/workflows/build-deploy-windows.yml new file mode 100644 index 000000000..5c7f1926d --- /dev/null +++ b/.github/workflows/build-deploy-windows.yml @@ -0,0 +1,35 @@ +on: + schedule: + - cron: "0 */12 * * *" +jobs: + + windows-x86_64: + runs-on: windows-2019 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - uses: ./.github/actions/msys2-base-setup + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Run windows cpu build + shell: cmd + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + set MSYSTEM=MINGW64 + set "PATH=C:\msys64\usr\bin;%PATH%" + mvn -Possrh -Djavacpp.platform=windows-x86_64 -pl ":nd4j-native,:libnd4j" --also-make -Dlibnd4j.platform=windows-x86_64 -Dlibnd4j.chip=cpu deploy -DskipTests diff --git a/libnd4j/pi_build.sh b/libnd4j/pi_build.sh index 23f97c86d..8a536d155 100755 --- a/libnd4j/pi_build.sh +++ b/libnd4j/pi_build.sh @@ -1,328 +1,328 @@ -#!/usr/bin/env bash -# -# /* ****************************************************************************** -# * -# * -# * This program and the accompanying materials are made available under the -# * terms of the Apache License, Version 2.0 which is available at -# * https://www.apache.org/licenses/LICENSE-2.0. -# * -# * See the NOTICE file distributed with this work for additional -# * information regarding copyright ownership. -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# * License for the specific language governing permissions and limitations -# * under the License. -# * -# * SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************/ -# - -function message { - echo "BUILDER:::: ${@}" -} -if [ -z "${BUILD_USING_MAVEN}" ]; then export BUILD_USING_MAVEN=; fi -if [ -z "${CURRENT_TARGET}" ]; then export CURRENT_TARGET=arm32; fi -if [ -z "${HAS_ARMCOMPUTE}" ]; then export ARMCOMPUTE_DEBUG=1; fi -if [ -z "${ARMCOMPUTE_DEBUG}" ]; then export HAS_ARMCOMPUTE=1; fi -if [ -z "${ARMCOMPUTE_TAG}" ]; then export ARMCOMPUTE_TAG=v20.05; fi -if [ -z "${LIBND4J_BUILD_MODE}" ]; then export LIBND4J_BUILD_MODE=Release; fi -if [ -z "${ANDROID_VERSION}" ]; then export ANDROID_VERSION=21; fi -if [ -z "${HAS_ARMCOMPUTE}" ]; then export HAS_ARMCOMPUTE=1; fi - -OTHER_ARGS=() -while [[ $# -gt 0 ]] -do -key="$1" - -case $key in - -a|--arch) - CURRENT_TARGET="$2" - shift - shift - ;; - -m|--mvn) - BUILD_USING_MAVEN="mvn" - shift - ;; - *) - OTHER_ARGS+=("$1") - shift - ;; -esac -done - -CC_URL32="https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf.tar.xz?revision=e09a1c45-0ed3-4a8e-b06b-db3978fd8d56&la=en&hash=93ED4444B8B3A812B893373B490B90BBB28FD2E3" -CC_URL64="https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en&hash=167687FADA00B73D20EED2A67D0939A197504ACD" -CC_ANDROID="https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip" -TARGET_ARRS=( arm32 arm64 android-arm android-arm64 ) -COMPILER_ARRS=( "${CC_URL32}" "${CC_URL64}" "${CC_ANDROID}" "${CC_ANDROID}" ) -COMPILER_DOWNLOAD_CMD_LIST=( download_extract_xz download_extract_xz download_extract_unzip download_extract_unzip ) -COMPILER_DESTDIR=( "arm32" "arm64" "android" "android" ) - -OPENBLAS_TARGETS=( ARMV7 ARMV8 ARMV7 ARMV8) -ARMCOMPUTE_TARGETS=( armv7a arm64-v8a armv7a arm64-v8a) -OS_LIST=( linux linux android android) -LIBND4J_PLATFORM_EXT_LIST=( armhf arm64 arm arm64 ) -PREFIXES=( arm-linux-gnueabihf aarch64-linux-gnu arm-linux-androideabi aarch64-linux-android ) -TARGET_INDEX=-1 - -for i in "${!TARGET_ARRS[@]}"; do - if [[ "${TARGET_ARRS[$i]}" = "${CURRENT_TARGET}" ]]; then - TARGET_INDEX=${i} - fi -done - -if [ ${TARGET_INDEX} -eq -1 ];then - message "could not find ${CURRENT_TARGET} in ${TARGET_ARRS[@]}" - exit -1 -fi - -#BASE_DIR=${HOME}/pi -#https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -BASE_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - -export CROSS_COMPILER_URL=${COMPILER_ARRS[$TARGET_INDEX]} -export CROSS_COMPILER_DIR=${BASE_DIR}/compile_tools/cross_compiler_${COMPILER_DESTDIR[$TARGET_INDEX]} -export COMPILER_DOWNLOAD_CMD=${COMPILER_DOWNLOAD_CMD_LIST[$TARGET_INDEX]} -export DETECT=${DETECT_LIST[$TARGET_INDEX]} -export LIBND4J_PLATFORM_EXT=${LIBND4J_PLATFORM_EXT_LIST[$TARGET_INDEX]} -export BLAS_TARGET_NAME=${OPENBLAS_TARGETS[$TARGET_INDEX]} -export ARMCOMPUTE_TARGET=${ARMCOMPUTE_TARGETS[$TARGET_INDEX]} -export TARGET_OS=${OS_LIST[$TARGET_INDEX]} -export LIBND4J_PLATFORM=${TARGET_OS}-${LIBND4J_PLATFORM_EXT} -export PREFIX=${PREFIXES[$TARGET_INDEX]} - -export CMAKE=cmake #/snap/bin/cmake -mkdir -p ${BASE_DIR}/compile_tools/ - -SCONS_LOCAL_URL=http://prdownloads.sourceforge.net/scons/scons-local-3.1.1.tar.gz -SCONS_LOCAL_DIR=${BASE_DIR}/compile_tools/scons_local - -THIRD_PARTY=${BASE_DIR}/third_party_libs${TARGET_INDEX} - -ARMCOMPUTE_GIT_URL=https://github.com/ARM-software/ComputeLibrary.git -ARMCOMPUTE_DIR=${THIRD_PARTY}/arm_compute_dir - -OPENBLAS_GIT_URL="https://github.com/xianyi/OpenBLAS.git" -OPENBLAS_DIR=${THIRD_PARTY}/OpenBLAS - - -mkdir -p ${BASE_DIR} -mkdir -p ${THIRD_PARTY} - -#change directory to base -cd $BASE_DIR - -function check_requirements { - for i in "${@}" - do - if [ ! -e "$i" ]; then - message "missing: ${i}" - exit -2 - fi - done -} - -function rename_top_folder { - for dir in ${1}/* - do - if [ -d "$dir" ] - then - mv "${dir}" "${1}/folder/" - message "${dir} => ${1}/folder/" - break - fi - done -} - -function download_extract_base { - #$1 is url #2 is dir $3 is extract argument - if [ ! -f ${3}_file ]; then - message "download" - wget --quiet --show-progress -O ${3}_file ${2} - fi - - message "extract $@" - #extract - mkdir -p ${3} - if [ ${1} = "-unzip" ]; then - command="unzip -qq ${3}_file -d ${3} " - else - command="tar ${1} ${3}_file --directory=${3} " - fi - message $command - $command - check_requirements "${3}" -} - -function download_extract { - download_extract_base -xzf $@ -} - -function download_extract_xz { - download_extract_base -xf $@ -} - -function download_extract_unzip { - download_extract_base -unzip $@ -} - -function git_check { - #$1 is url #$2 is dir #$3 is tag or branch if optional - command= - if [ -n "$3" ]; then - command="git clone --quiet --depth 1 --branch ${3} ${1} ${2}" - else - command="git clone --quiet ${1} ${2}" - fi - message "$command" - $command - check_requirements "${2}" -} - -#fix py debug linkage manually and also makes it use gold -function fix_pi_linker { - #$1 BINUTILS folder - if [ ! -f ${1}/ld.original ]; then - mv ${1}/ld ${1}/ld.original - fi - rm -f ${1}/ld - printf '#!/usr/bin/env bash\n'"${1}/ld.gold --long-plt \$*">${1}/ld - chmod +x ${1}/ld -} - -if [ ! -d ${CROSS_COMPILER_DIR}/folder ]; then - #out file - message "download CROSS_COMPILER" - ${COMPILER_DOWNLOAD_CMD} ${CROSS_COMPILER_URL} ${CROSS_COMPILER_DIR} - message "rename top folder (instead of --strip-components=1)" - rename_top_folder ${CROSS_COMPILER_DIR} -fi - -export CROSS_COMPILER_DIR=${CROSS_COMPILER_DIR}/folder - -if [ "${TARGET_OS}" = "android" ];then - export ANDROID_TOOLCHAIN=${CROSS_COMPILER_DIR}/toolchains/llvm/prebuilt/linux-x86_64 - export COMPILER_PREFIX="${ANDROID_TOOLCHAIN}/bin/${PREFIX}${ANDROID_VERSION}" - export TOOLCHAIN_PREFIX="${ANDROID_TOOLCHAIN}/bin/${PREFIX}" - if [ "$BLAS_TARGET_NAME" = "ARMV7" ];then - BLAS_XTRA="ARM_SOFTFP_ABI=1 " - COMPILER_PREFIX="${ANDROID_TOOLCHAIN}/bin/armv7a-linux-androideabi${ANDROID_VERSION}" - fi - export CC_EXE="clang" - export CXX_EXE="clang++" - export AR="${TOOLCHAIN_PREFIX}-ar" - export RANLIB="${TOOLCHAIN_PREFIX}-ranlib" - export BLAS_XTRA="CC=${COMPILER_PREFIX}-${CC_EXE} AR=${AR} RANLIB=${RANLIB} ${BLAS_XTRA}" -else - export BINUTILS_BIN=${CROSS_COMPILER_DIR}/${PREFIX}/bin - export COMPILER_PREFIX=${CROSS_COMPILER_DIR}/bin/${PREFIX} - export TOOLCHAIN_PREFIX=${COMPILER_PREFIX} - export SYS_ROOT=${CROSS_COMPILER_DIR}/${PREFIX}/libc - #LD_LIBRARY_PATH=${CROSS_COMPILER_DIR}/lib:$LD_LIBRARY_PATH - export CC_EXE="gcc" - export CXX_EXE="g++" - export RANLIB="${BINUTILS_BIN}/ranlib" - export LD="${BINUTILS_BIN}/ld" - export AR="${BINUTILS_BIN}/ar" - export BLAS_XTRA="CC=${COMPILER_PREFIX}-${CC_EXE} AR=${AR} RANLIB=${RANLIB} CFLAGS=--sysroot=${SYS_ROOT} LDFLAGS=\"-L${SYS_ROOT}/../lib/ -lm\"" -fi - -check_requirements ${CC} - -if [ -z "${BUILD_USING_MAVEN}" ] ;then -#lets build OpenBlas -if [ ! -d "${OPENBLAS_DIR}" ]; then - message "download OpenBLAS" - git_check "${OPENBLAS_GIT_URL}" "${OPENBLAS_DIR}" "v0.3.10" -fi - -if [ ! -f "${THIRD_PARTY}/lib/libopenblas.so" ]; then - message "build and install OpenBLAS" - cd ${OPENBLAS_DIR} - - command="make TARGET=${BLAS_TARGET_NAME} HOSTCC=gcc NOFORTRAN=1 ${BLAS_XTRA} " - message $command - eval $command &>/dev/null - message "install it" - command="make TARGET=${BLAS_TARGET_NAME} PREFIX=${THIRD_PARTY} install &>/dev/null" - message $command - $command - cd $BASE_DIR - -fi -check_requirements ${THIRD_PARTY}/lib/libopenblas.so - -export OPENBLAS_PATH=${THIRD_PARTY} - -fi # end if [ -z "${BUILD_USING_MAVEN}"];then - -if [ ! -d ${SCONS_LOCAL_DIR} ]; then - #out file - message "download Scons local" - download_extract ${SCONS_LOCAL_URL} ${SCONS_LOCAL_DIR} -fi -check_requirements ${SCONS_LOCAL_DIR}/scons.py - -if [ ! -d "${ARMCOMPUTE_DIR}" ]; then - message "download ArmCompute Source" - git_check ${ARMCOMPUTE_GIT_URL} "${ARMCOMPUTE_DIR}" "${ARMCOMPUTE_TAG}" -fi - -#build armcompute -if [ ! -f "${ARMCOMPUTE_DIR}/build/libarm_compute-static.a" ]; then -message "build arm compute" -cd ${ARMCOMPUTE_DIR} -command="CC=${CC_EXE} CXX=${CXX_EXE} python3 ${SCONS_LOCAL_DIR}/scons.py Werror=1 -j$(nproc) toolchain_prefix=${TOOLCHAIN_PREFIX}- compiler_prefix=${COMPILER_PREFIX}- debug=${ARMCOMPUTE_DEBUG} neon=1 opencl=0 extra_cxx_flags=-fPIC os=${TARGET_OS} build=cross_compile arch=${ARMCOMPUTE_TARGET} " -message $command -eval $command &>/dev/null -cd ${BASE_DIR} -fi -check_requirements "${ARMCOMPUTE_DIR}/build/libarm_compute-static.a" "${ARMCOMPUTE_DIR}/build/libarm_compute_core-static.a" - -export ARMCOMPUTE_ROOT="${ARMCOMPUTE_DIR}" - -if [ "${TARGET_OS}" = "android" ];then - export ANDROID_NDK=${CROSS_COMPILER_DIR} -else - export RPI_BIN=${CROSS_COMPILER_DIR}/bin/${PREFIX} - export JAVA_LIBRARY_PATH=${CROSS_COMPILER_DIR}/${PREFIX}/lib - fix_pi_linker ${BINUTILS_BIN} -fi - - -#because of the toolchain passive detection we have to delete build folder manually -detect=$(cat ${BASE_DIR}/blasbuild/cpu/CMakeCache.txt | grep -o ${PREFIX}) -if [ -z "${detect}" ] ;then -message "remove blasbuild folder " -rm -rf $BASE_DIR/blasbuild/ -else -message "keep blasbuild folder" -fi - -if [ -z "${BUILD_USING_MAVEN}" ] ;then -message "lets build just library" -DHELPER=" -h armcompute " -bash ./buildnativeoperations.sh -o ${LIBND4J_PLATFORM} -t ${DHELPER} -j $(nproc) -else -message "cd $BASE_DIR/.. " -cd $BASE_DIR/.. -message "lets build jars" -export DHELPER=" -Dlibnd4j.helper=armcompute " -if [ "${DEPLOY}" ]; then - echo "Deploying to maven" - mvn -P"${PUBLISH_TO}" deploy --batch-mode -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER} -pl ":libnd4j,:nd4j-native" --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true - else - echo "Installing to local repo" - mvn install -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER} -pl ":libnd4j" --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -fi - -fi +#!/usr/bin/env bash +# +# /* ****************************************************************************** +# * +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ +# + +function message { + echo "BUILDER:::: ${@}" +} +if [ -z "${BUILD_USING_MAVEN}" ]; then export BUILD_USING_MAVEN=; fi +if [ -z "${CURRENT_TARGET}" ]; then export CURRENT_TARGET=arm32; fi +if [ -z "${HAS_ARMCOMPUTE}" ]; then export ARMCOMPUTE_DEBUG=1; fi +if [ -z "${ARMCOMPUTE_DEBUG}" ]; then export HAS_ARMCOMPUTE=1; fi +if [ -z "${ARMCOMPUTE_TAG}" ]; then export ARMCOMPUTE_TAG=v20.05; fi +if [ -z "${LIBND4J_BUILD_MODE}" ]; then export LIBND4J_BUILD_MODE=Release; fi +if [ -z "${ANDROID_VERSION}" ]; then export ANDROID_VERSION=21; fi +if [ -z "${HAS_ARMCOMPUTE}" ]; then export HAS_ARMCOMPUTE=1; fi + +OTHER_ARGS=() +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + -a|--arch) + CURRENT_TARGET="$2" + shift + shift + ;; + -m|--mvn) + BUILD_USING_MAVEN="mvn" + shift + ;; + *) + OTHER_ARGS+=("$1") + shift + ;; +esac +done + +CC_URL32="https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf.tar.xz?revision=e09a1c45-0ed3-4a8e-b06b-db3978fd8d56&la=en&hash=93ED4444B8B3A812B893373B490B90BBB28FD2E3" +CC_URL64="https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en&hash=167687FADA00B73D20EED2A67D0939A197504ACD" +CC_ANDROID="https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip" +TARGET_ARRS=( arm32 arm64 android-arm android-arm64 ) +COMPILER_ARRS=( "${CC_URL32}" "${CC_URL64}" "${CC_ANDROID}" "${CC_ANDROID}" ) +COMPILER_DOWNLOAD_CMD_LIST=( download_extract_xz download_extract_xz download_extract_unzip download_extract_unzip ) +COMPILER_DESTDIR=( "arm32" "arm64" "android" "android" ) + +OPENBLAS_TARGETS=( ARMV7 ARMV8 ARMV7 ARMV8) +ARMCOMPUTE_TARGETS=( armv7a arm64-v8a armv7a arm64-v8a) +OS_LIST=( linux linux android android) +LIBND4J_PLATFORM_EXT_LIST=( armhf arm64 arm arm64 ) +PREFIXES=( arm-linux-gnueabihf aarch64-linux-gnu arm-linux-androideabi aarch64-linux-android ) +TARGET_INDEX=-1 + +for i in "${!TARGET_ARRS[@]}"; do + if [[ "${TARGET_ARRS[$i]}" = "${CURRENT_TARGET}" ]]; then + TARGET_INDEX=${i} + fi +done + +if [ ${TARGET_INDEX} -eq -1 ];then + message "could not find ${CURRENT_TARGET} in ${TARGET_ARRS[@]}" + exit -1 +fi + +#BASE_DIR=${HOME}/pi +#https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +BASE_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + +export CROSS_COMPILER_URL=${COMPILER_ARRS[$TARGET_INDEX]} +export CROSS_COMPILER_DIR=${BASE_DIR}/compile_tools/cross_compiler_${COMPILER_DESTDIR[$TARGET_INDEX]} +export COMPILER_DOWNLOAD_CMD=${COMPILER_DOWNLOAD_CMD_LIST[$TARGET_INDEX]} +export DETECT=${DETECT_LIST[$TARGET_INDEX]} +export LIBND4J_PLATFORM_EXT=${LIBND4J_PLATFORM_EXT_LIST[$TARGET_INDEX]} +export BLAS_TARGET_NAME=${OPENBLAS_TARGETS[$TARGET_INDEX]} +export ARMCOMPUTE_TARGET=${ARMCOMPUTE_TARGETS[$TARGET_INDEX]} +export TARGET_OS=${OS_LIST[$TARGET_INDEX]} +export LIBND4J_PLATFORM=${TARGET_OS}-${LIBND4J_PLATFORM_EXT} +export PREFIX=${PREFIXES[$TARGET_INDEX]} + +export CMAKE=cmake #/snap/bin/cmake +mkdir -p ${BASE_DIR}/compile_tools/ + +SCONS_LOCAL_URL=http://prdownloads.sourceforge.net/scons/scons-local-3.1.1.tar.gz +SCONS_LOCAL_DIR=${BASE_DIR}/compile_tools/scons_local + +THIRD_PARTY=${BASE_DIR}/third_party_libs${TARGET_INDEX} + +ARMCOMPUTE_GIT_URL=https://github.com/ARM-software/ComputeLibrary.git +ARMCOMPUTE_DIR=${THIRD_PARTY}/arm_compute_dir + +OPENBLAS_GIT_URL="https://github.com/xianyi/OpenBLAS.git" +OPENBLAS_DIR=${THIRD_PARTY}/OpenBLAS + + +mkdir -p ${BASE_DIR} +mkdir -p ${THIRD_PARTY} + +#change directory to base +cd $BASE_DIR + +function check_requirements { + for i in "${@}" + do + if [ ! -e "$i" ]; then + message "missing: ${i}" + exit -2 + fi + done +} + +function rename_top_folder { + for dir in ${1}/* + do + if [ -d "$dir" ] + then + mv "${dir}" "${1}/folder/" + message "${dir} => ${1}/folder/" + break + fi + done +} + +function download_extract_base { + #$1 is url #2 is dir $3 is extract argument + if [ ! -f ${3}_file ]; then + message "download" + wget --quiet --show-progress -O ${3}_file ${2} + fi + + message "extract $@" + #extract + mkdir -p ${3} + if [ ${1} = "-unzip" ]; then + command="unzip -qq ${3}_file -d ${3} " + else + command="tar ${1} ${3}_file --directory=${3} " + fi + message $command + $command + check_requirements "${3}" +} + +function download_extract { + download_extract_base -xzf $@ +} + +function download_extract_xz { + download_extract_base -xf $@ +} + +function download_extract_unzip { + download_extract_base -unzip $@ +} + +function git_check { + #$1 is url #$2 is dir #$3 is tag or branch if optional + command= + if [ -n "$3" ]; then + command="git clone --quiet --depth 1 --branch ${3} ${1} ${2}" + else + command="git clone --quiet ${1} ${2}" + fi + message "$command" + $command + check_requirements "${2}" +} + +#fix py debug linkage manually and also makes it use gold +function fix_pi_linker { + #$1 BINUTILS folder + if [ ! -f ${1}/ld.original ]; then + mv ${1}/ld ${1}/ld.original + fi + rm -f ${1}/ld + printf '#!/usr/bin/env bash\n'"${1}/ld.gold --long-plt \$*">${1}/ld + chmod +x ${1}/ld +} + +if [ ! -d ${CROSS_COMPILER_DIR}/folder ]; then + #out file + message "download CROSS_COMPILER" + ${COMPILER_DOWNLOAD_CMD} ${CROSS_COMPILER_URL} ${CROSS_COMPILER_DIR} + message "rename top folder (instead of --strip-components=1)" + rename_top_folder ${CROSS_COMPILER_DIR} +fi + +export CROSS_COMPILER_DIR=${CROSS_COMPILER_DIR}/folder + +if [ "${TARGET_OS}" = "android" ];then + export ANDROID_TOOLCHAIN=${CROSS_COMPILER_DIR}/toolchains/llvm/prebuilt/linux-x86_64 + export COMPILER_PREFIX="${ANDROID_TOOLCHAIN}/bin/${PREFIX}${ANDROID_VERSION}" + export TOOLCHAIN_PREFIX="${ANDROID_TOOLCHAIN}/bin/${PREFIX}" + if [ "$BLAS_TARGET_NAME" = "ARMV7" ];then + BLAS_XTRA="ARM_SOFTFP_ABI=1 " + COMPILER_PREFIX="${ANDROID_TOOLCHAIN}/bin/armv7a-linux-androideabi${ANDROID_VERSION}" + fi + export CC_EXE="clang" + export CXX_EXE="clang++" + export AR="${TOOLCHAIN_PREFIX}-ar" + export RANLIB="${TOOLCHAIN_PREFIX}-ranlib" + export BLAS_XTRA="CC=${COMPILER_PREFIX}-${CC_EXE} AR=${AR} RANLIB=${RANLIB} ${BLAS_XTRA}" +else + export BINUTILS_BIN=${CROSS_COMPILER_DIR}/${PREFIX}/bin + export COMPILER_PREFIX=${CROSS_COMPILER_DIR}/bin/${PREFIX} + export TOOLCHAIN_PREFIX=${COMPILER_PREFIX} + export SYS_ROOT=${CROSS_COMPILER_DIR}/${PREFIX}/libc + #LD_LIBRARY_PATH=${CROSS_COMPILER_DIR}/lib:$LD_LIBRARY_PATH + export CC_EXE="gcc" + export CXX_EXE="g++" + export RANLIB="${BINUTILS_BIN}/ranlib" + export LD="${BINUTILS_BIN}/ld" + export AR="${BINUTILS_BIN}/ar" + export BLAS_XTRA="CC=${COMPILER_PREFIX}-${CC_EXE} AR=${AR} RANLIB=${RANLIB} CFLAGS=--sysroot=${SYS_ROOT} LDFLAGS=\"-L${SYS_ROOT}/../lib/ -lm\"" +fi + +check_requirements ${CC} + +if [ -z "${BUILD_USING_MAVEN}" ] ;then +#lets build OpenBlas +if [ ! -d "${OPENBLAS_DIR}" ]; then + message "download OpenBLAS" + git_check "${OPENBLAS_GIT_URL}" "${OPENBLAS_DIR}" "v0.3.10" +fi + +if [ ! -f "${THIRD_PARTY}/lib/libopenblas.so" ]; then + message "build and install OpenBLAS" + cd ${OPENBLAS_DIR} + + command="make TARGET=${BLAS_TARGET_NAME} HOSTCC=gcc NOFORTRAN=1 ${BLAS_XTRA} " + message $command + eval $command &>/dev/null + message "install it" + command="make TARGET=${BLAS_TARGET_NAME} PREFIX=${THIRD_PARTY} install &>/dev/null" + message $command + $command + cd $BASE_DIR + +fi +check_requirements ${THIRD_PARTY}/lib/libopenblas.so + +export OPENBLAS_PATH=${THIRD_PARTY} + +fi # end if [ -z "${BUILD_USING_MAVEN}"];then + +if [ ! -d ${SCONS_LOCAL_DIR} ]; then + #out file + message "download Scons local" + download_extract ${SCONS_LOCAL_URL} ${SCONS_LOCAL_DIR} +fi +check_requirements ${SCONS_LOCAL_DIR}/scons.py + +if [ ! -d "${ARMCOMPUTE_DIR}" ]; then + message "download ArmCompute Source" + git_check ${ARMCOMPUTE_GIT_URL} "${ARMCOMPUTE_DIR}" "${ARMCOMPUTE_TAG}" +fi + +#build armcompute +if [ ! -f "${ARMCOMPUTE_DIR}/build/libarm_compute-static.a" ]; then +message "build arm compute" +cd ${ARMCOMPUTE_DIR} +command="CC=${CC_EXE} CXX=${CXX_EXE} python3 ${SCONS_LOCAL_DIR}/scons.py Werror=1 -j$(nproc) toolchain_prefix=${TOOLCHAIN_PREFIX}- compiler_prefix=${COMPILER_PREFIX}- debug=${ARMCOMPUTE_DEBUG} neon=1 opencl=0 extra_cxx_flags=-fPIC os=${TARGET_OS} build=cross_compile arch=${ARMCOMPUTE_TARGET} " +message $command +eval $command &>/dev/null +cd ${BASE_DIR} +fi +check_requirements "${ARMCOMPUTE_DIR}/build/libarm_compute-static.a" "${ARMCOMPUTE_DIR}/build/libarm_compute_core-static.a" + +export ARMCOMPUTE_ROOT="${ARMCOMPUTE_DIR}" + +if [ "${TARGET_OS}" = "android" ];then + export ANDROID_NDK=${CROSS_COMPILER_DIR} +else + export RPI_BIN=${CROSS_COMPILER_DIR}/bin/${PREFIX} + export JAVA_LIBRARY_PATH=${CROSS_COMPILER_DIR}/${PREFIX}/lib + fix_pi_linker ${BINUTILS_BIN} +fi + + +#because of the toolchain passive detection we have to delete build folder manually +detect=$(cat ${BASE_DIR}/blasbuild/cpu/CMakeCache.txt | grep -o ${PREFIX}) +if [ -z "${detect}" ] ;then +message "remove blasbuild folder " +rm -rf $BASE_DIR/blasbuild/ +else +message "keep blasbuild folder" +fi + +if [ -z "${BUILD_USING_MAVEN}" ] ;then +message "lets build just library" +DHELPER=" -h armcompute " +bash ./buildnativeoperations.sh -o ${LIBND4J_PLATFORM} -t ${DHELPER} -j $(nproc) +else +message "cd $BASE_DIR/.. " +cd $BASE_DIR/.. +message "lets build jars" +export DHELPER=" -Dlibnd4j.helper=armcompute " +if [ "${DEPLOY}" ]; then + echo "Deploying to maven" + mvn -P"${PUBLISH_TO}" deploy --batch-mode -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER} -pl ":libnd4j,:nd4j-native" --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true + else + echo "Installing to local repo" + mvn install -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER} -pl ":libnd4j" --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true +fi + +fi