From f09cfca302dc14eddfe3f39ab295e3423e8cbca8 Mon Sep 17 00:00:00 2001 From: agibsonccc Date: Fri, 5 Mar 2021 09:37:29 +0900 Subject: [PATCH 01/11] Add workflows --- .../action.yml | 11 ++++ .../action.yml | 12 +++++ .../install-arm-cross-compile/action.yml | 12 +++++ .../actions/install-protobuf-linux/action.yml | 16 ++++++ .github/actions/msys2-base-setup/action.yml | 10 ++++ .../actions/publish-gh-packages/action.yml | 9 ++++ .github/workflows/build-android-x86_64.yml | 39 ++++++++++++++ .../workflows/build-deploy-android-arm32.yml | 43 +++++++++++++++ .../workflows/build-deploy-android-arm64.yml | 43 +++++++++++++++ .../workflows/build-deploy-linux-arm32.yml | 43 +++++++++++++++ .../workflows/build-deploy-linux-arm64.yml | 40 ++++++++++++++ .../build-deploy-linux-cuda-11.0.yml | 54 +++++++++++++++++++ .../build-deploy-linux-cuda-11.2.yml | 51 ++++++++++++++++++ ...ild-deploy-linux-jetsonnano-selfhosted.yml | 50 +++++++++++++++++ .../workflows/build-deploy-linux-x86_64.yml | 40 ++++++++++++++ .github/workflows/build-deploy-mac-arm64.yml | 33 ++++++++++++ .github/workflows/build-deploy-mac.yml | 33 ++++++++++++ .../build-deploy-windows-cuda-11.0.yml | 45 ++++++++++++++++ .../build-deploy-windows-cuda-11.2.yml | 47 ++++++++++++++++ .github/workflows/build-deploy-windows.yml | 34 ++++++++++++ 20 files changed, 665 insertions(+) create mode 100644 .github/actions/download-dl4j-test-resources-linux/action.yml create mode 100644 .github/actions/download-dl4j-test-resources-windows/action.yml create mode 100644 .github/actions/install-arm-cross-compile/action.yml create mode 100644 .github/actions/install-protobuf-linux/action.yml create mode 100644 .github/actions/msys2-base-setup/action.yml create mode 100644 .github/actions/publish-gh-packages/action.yml create mode 100644 .github/workflows/build-android-x86_64.yml create mode 100644 .github/workflows/build-deploy-android-arm32.yml create mode 100644 .github/workflows/build-deploy-android-arm64.yml create mode 100644 .github/workflows/build-deploy-linux-arm32.yml create mode 100644 .github/workflows/build-deploy-linux-arm64.yml create mode 100644 .github/workflows/build-deploy-linux-cuda-11.0.yml create mode 100644 .github/workflows/build-deploy-linux-cuda-11.2.yml create mode 100644 .github/workflows/build-deploy-linux-jetsonnano-selfhosted.yml create mode 100644 .github/workflows/build-deploy-linux-x86_64.yml create mode 100644 .github/workflows/build-deploy-mac-arm64.yml create mode 100644 .github/workflows/build-deploy-mac.yml create mode 100644 .github/workflows/build-deploy-windows-cuda-11.0.yml create mode 100644 .github/workflows/build-deploy-windows-cuda-11.2.yml create mode 100644 .github/workflows/build-deploy-windows.yml diff --git a/.github/actions/download-dl4j-test-resources-linux/action.yml b/.github/actions/download-dl4j-test-resources-linux/action.yml new file mode 100644 index 000000000..81d8266b7 --- /dev/null +++ b/.github/actions/download-dl4j-test-resources-linux/action.yml @@ -0,0 +1,11 @@ +name: Download dl4j test resources +runs: + using: composite + steps: + - name: Initial install + shell: bash + run: | + wget https://github.com/KonduitAI/dl4j-test-resources/archive/master.zip && unzip master.zip + cd dl4j-test-resources-master + mvn clean install -DskipTests + echo "Extracted test resources" \ No newline at end of file diff --git a/.github/actions/download-dl4j-test-resources-windows/action.yml b/.github/actions/download-dl4j-test-resources-windows/action.yml new file mode 100644 index 000000000..5683619e3 --- /dev/null +++ b/.github/actions/download-dl4j-test-resources-windows/action.yml @@ -0,0 +1,12 @@ +name: Download dl4j test resources +runs: + using: composite + steps: + - name: Initial install + shell: cmd + run: | + set "PATH=C:\msys64\usr\bin;%PATH%" + wget https://github.com/KonduitAI/dl4j-test-resources/archive/master.zip && unzip master.zip + cd dl4j-test-resources-master + mvn clean install -DskipTests + echo "Extracted test resources" \ No newline at end of file diff --git a/.github/actions/install-arm-cross-compile/action.yml b/.github/actions/install-arm-cross-compile/action.yml new file mode 100644 index 000000000..dcf52c4ab --- /dev/null +++ b/.github/actions/install-arm-cross-compile/action.yml @@ -0,0 +1,12 @@ +name: Download dl4j test resources +runs: + using: composite + steps: + - name: Initial install + shell: bash + run: | + sudo apt install git gcc-8-aarch64-linux-gnu g++-8-aarch64-linux-gnu libc6-armel-cross libc6-dev-armel-cross binutils-arm-linux-gnueabi libncurses5-dev build-essential bison flex libssl-dev bc \ + gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf crossbuild-essential-arm64 + mkdir -p /opt/raspberrypi && \ + cd /opt/raspberrypi && \ + git clone git://github.com/raspberrypi/tools.git diff --git a/.github/actions/install-protobuf-linux/action.yml b/.github/actions/install-protobuf-linux/action.yml new file mode 100644 index 000000000..f48a953b1 --- /dev/null +++ b/.github/actions/install-protobuf-linux/action.yml @@ -0,0 +1,16 @@ +name: Install protobuf linux +runs: + using: composite + steps: + - name: Install protobuf linux + shell: bash + run: | + curl -fsSL https://github.com/google/protobuf/releases/download/v3.5.1/protobuf-cpp-3.5.1.tar.gz \ + | tar xz && \ + cd protobuf-3.5.1 && \ + ./configure --prefix=/opt/protobuf && \ + make -j2 && \ + make install && \ + cd .. && \ + rm -rf protobuf-3.5.1 + echo "/opt/protobuf/bin" >> $GITHUB_PATH \ No newline at end of file diff --git a/.github/actions/msys2-base-setup/action.yml b/.github/actions/msys2-base-setup/action.yml new file mode 100644 index 000000000..272478d81 --- /dev/null +++ b/.github/actions/msys2-base-setup/action.yml @@ -0,0 +1,10 @@ +name: Setup for msys2 +runs: + using: composite + steps: + - name: Initial install + shell: cmd + run: | + C:\msys64\usr\bin\bash -lc "pacman -S --needed --noconfirm base-devel git tar pkg-config unzip p7zip zip autoconf autoconf-archive automake make patch gnupg" + C:\msys64\usr\bin\bash -lc "pacman -S --needed --noconfirm mingw-w64-x86_64-nasm mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool mingw-w64-x86_64-gcc mingw-w64-i686-gcc mingw-w64-x86_64-gcc-fortran mingw-w64-i686-gcc-fortran mingw-w64-x86_64-libwinpthread-git mingw-w64-i686-libwinpthread-git mingw-w64-x86_64-SDL mingw-w64-i686-SDL mingw-w64-x86_64-ragel" + echo "C:\msys64\usr\bin" >> $GITHUB_PATH \ No newline at end of file diff --git a/.github/actions/publish-gh-packages/action.yml b/.github/actions/publish-gh-packages/action.yml new file mode 100644 index 000000000..cb6f1424c --- /dev/null +++ b/.github/actions/publish-gh-packages/action.yml @@ -0,0 +1,9 @@ +name: Publish to github packages +runs: + using: composite + steps: + - name: Publish to GitHub Packages + run: mvn -Pgithub --batch-mode deploy + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/build-android-x86_64.yml b/.github/workflows/build-android-x86_64.yml new file mode 100644 index 000000000..b7ed65a4d --- /dev/null +++ b/.github/workflows/build-android-x86_64.yml @@ -0,0 +1,39 @@ +on: + push: +jobs: + android-x86_64: + runs-on: ubuntu-18.04 + steps: + - uses: AutoModality/action-clean@v1 + - uses: ./.github/actions/install-protobuf-linux + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: nttld/setup-ndk@v1 + with: + ndk-version: r18b + - uses: actions/checkout@v2 + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build on linux-x86_64 + env: + ANDROID_NDK: "/opt/hostedcache/ndk/r18b/x64" + OPENBLAS_PATH: "/opt/openblas/" + LIBND4J_HOME: "${GITHUB_WORKSPACE}/libnd4j" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "Verifying programs on path. Path is $PATH" + echo "Path post update is $PATH. Maven is at `which mvn` cmake is at `which cmake` protoc is at `which protoc`" + mvn --version + cmake --version + protoc --version + clang --version + mvn -X -Dorg.bytedeco.javacpp.logger.debug=true -Pgithub -pl ":nd4j-native,:libnd4j" --also-make \ + -Djavacpp.platform=android-x86_64 \ + -Dlibnd4j.platform=android-x86_64 -Dlibnd4j.chip=cpu \ + --batch-mode clean deploy -DskipTests + + diff --git a/.github/workflows/build-deploy-android-arm32.yml b/.github/workflows/build-deploy-android-arm32.yml new file mode 100644 index 000000000..9cd45c767 --- /dev/null +++ b/.github/workflows/build-deploy-android-arm32.yml @@ -0,0 +1,43 @@ +on: + push: +jobs: + #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. + android-arm32: + runs-on: ubuntu-18.04 + steps: + - uses: AutoModality/action-clean@v1 + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Build on android-arm32 + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEPLOY: 1 + BUILD_USING_MAVEN: 1 + TARGET_OS: android + CURRENT_TARGET: arm32 + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + mvn --version + cmake --version + protoc --version + ${GITHUB_WORKSPACE}/libnd4j/pi_build.sh + + diff --git a/.github/workflows/build-deploy-android-arm64.yml b/.github/workflows/build-deploy-android-arm64.yml new file mode 100644 index 000000000..794a1c3ad --- /dev/null +++ b/.github/workflows/build-deploy-android-arm64.yml @@ -0,0 +1,43 @@ +on: + push: +jobs: + #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. + android-arm64: + runs-on: self-hosted + steps: + - uses: AutoModality/action-clean@v1 + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + + - name: Build on android-arm64 + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEPLOY: 1 + BUILD_USING_MAVEN: 1 + TARGET_OS: android + CURRENT_TARGET: arm64 + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + mvn --version + cmake --version + protoc --version + ${GITHUB_WORKSPACE}/libnd4j/pi_build.sh + + diff --git a/.github/workflows/build-deploy-linux-arm32.yml b/.github/workflows/build-deploy-linux-arm32.yml new file mode 100644 index 000000000..b51bf385b --- /dev/null +++ b/.github/workflows/build-deploy-linux-arm32.yml @@ -0,0 +1,43 @@ +on: + push: +jobs: + #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. + linux-arm32: + runs-on: ubuntu-18.04 + steps: + - uses: AutoModality/action-clean@v1 + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Build on linux-arm32 + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEPLOY: 1 + BUILD_USING_MAVEN: 1 + TARGET_OS: linux + CURRENT_TARGET: arm32 + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + mvn --version + cmake --version + protoc --version + ${GITHUB_WORKSPACE}/libnd4j/pi_build.sh + + diff --git a/.github/workflows/build-deploy-linux-arm64.yml b/.github/workflows/build-deploy-linux-arm64.yml new file mode 100644 index 000000000..ff9ef5ae3 --- /dev/null +++ b/.github/workflows/build-deploy-linux-arm64.yml @@ -0,0 +1,40 @@ +on: + push: +jobs: + #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. + linux-arm64: + runs-on: ubuntu-18.04 + steps: + - uses: AutoModality/action-clean@v1 + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Build on linux-arm64 + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEPLOY: 1 + BUILD_USING_MAVEN: 1 + TARGET_OS: linux + CURRENT_TARGET: arm64 + PUBLISH_TO: ossrh + run: | + mvn --version + cmake --version + protoc --version + ${GITHUB_WORKSPACE}/libnd4j/pi_build.sh + + diff --git a/.github/workflows/build-deploy-linux-cuda-11.0.yml b/.github/workflows/build-deploy-linux-cuda-11.0.yml new file mode 100644 index 000000000..5e2effd39 --- /dev/null +++ b/.github/workflows/build-deploy-linux-cuda-11.0.yml @@ -0,0 +1,54 @@ +on: + push: +jobs: + + linux-x86_64-cuda_11-0: + runs-on: ubuntu-18.04 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + root-reserve-mb: 512 + swap-size-mb: 8192 + remove-dotnet: 'true' + remove-haskell: 'true' + - uses: actions/checkout@v2 + - uses: konduitai/cuda-install/.github/actions/install-cuda-ubuntu@master + env: + cuda: 11.0.167 + GCC: 9 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + + - name: Build cuda + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + export PATH="/usr/local/cuda-11.0/bin:$PATH" + mvn --version + cmake --version + protoc --version + nvcc --version + sudo apt-get autoremove + sudo apt-get clean + mvn -Possrh -Djavacpp.platform=linux-x86_64 -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -Dlibnd4j.chip=cuda -pl ":nd4j-cuda-11.0,:deeplearning4j-cuda-11.0,:libnd4j" --also-make -Pcuda clean --batch-mode deploy -DskipTests + + diff --git a/.github/workflows/build-deploy-linux-cuda-11.2.yml b/.github/workflows/build-deploy-linux-cuda-11.2.yml new file mode 100644 index 000000000..a1b11d9d7 --- /dev/null +++ b/.github/workflows/build-deploy-linux-cuda-11.2.yml @@ -0,0 +1,51 @@ +on: + push: +jobs: + linux-x86_64-cuda-11-2: + runs-on: ubuntu-18.04 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + root-reserve-mb: 512 + swap-size-mb: 8192 + remove-dotnet: 'true' + remove-haskell: 'true' + - uses: actions/checkout@v2 + - uses: konduitai/cuda-install/.github/actions/install-cuda-ubuntu@master + env: + cuda: 11.2.1_461 + GCC: 9 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Run cuda compilation on linux-x86_64 + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + export PATH="/usr/local/cuda-11.2/bin:$PATH" + nvcc --version + mvn --version + cmake --version + protoc --version + sudo apt-get autoremove + sudo apt-get clean + bash ./change-cuda-versions.sh 11.2 + mvn -Possrh -Djavacpp.platform=linux-x86_64 -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -pl ":nd4j-cuda-11.2,:deeplearning4j-cuda-11.2,:libnd4j" --also-make -Dlibnd4j.chip=cuda --batch-mode deploy -DskipTests diff --git a/.github/workflows/build-deploy-linux-jetsonnano-selfhosted.yml b/.github/workflows/build-deploy-linux-jetsonnano-selfhosted.yml new file mode 100644 index 000000000..ef0b02ca0 --- /dev/null +++ b/.github/workflows/build-deploy-linux-jetsonnano-selfhosted.yml @@ -0,0 +1,50 @@ +on: + push: +jobs: + #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. + linux-arm64: + runs-on: self-hosted + steps: + - uses: AutoModality/action-clean@v1 + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Build on linux-x86_64 + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEPLOY: 1 + BUILD_USING_MAVEN: 1 + TARGET_OS: linux + CURRENT_TARGET: arm64 + CROSS_ROOT: ${GITHUB_WORKSPACE}/rootfs-nano + CROSS_COMPILE: ${GITHUB_WORKSPACE}/toolchain/gcc-4.8.5-aarch64/bin + CC: ${CROSS_COMPILE}/aarch64-unknown-linux-gnu-gcc + CXX: ${CROSS_COMPILE}/aarch64-unknown-linux-gnu-g++ + LD: ${CROSS_COMPILE}/aarch64-unknown-linux-gnu-ld + AR: ${CROSS_COMPILE}/aarch64-unknown-linux-gnu-ar + AS: ${CROSS_COMPILE}/aarch64-unknown-linux-gnu-as + RANLIB: ${CROSS_COMPILE}/aarch64-unknown-linux-gnu-ranlib + NVCC: ${GITHUB_WORKSPACE}/cuda-10.2/bin/nvcc + + run: | + wget https://developer.nvidia.com/embedded/dlc/l4t-gcc-toolchain-64-bit-28-3 + tar xpvf l4t-gcc-toolchain-64-bit-28-3 + mv install gcc-4.8.5-aarch64 + ${GITHUB_WORKSPACE}/change-cuda-versions.sh 10.2 + wget https://github.com/KonduitAI/deeplearning4j/releases/download/cuda-10.2/cuda-10.2.tar + tar xvf cuda-10.2.tar + mvn --version + cmake --version + protoc --version + ${GITHUB_WORKSPACE}/libnd4j/pi_build.sh + + diff --git a/.github/workflows/build-deploy-linux-x86_64.yml b/.github/workflows/build-deploy-linux-x86_64.yml new file mode 100644 index 000000000..e53fcd174 --- /dev/null +++ b/.github/workflows/build-deploy-linux-x86_64.yml @@ -0,0 +1,40 @@ +on: + push: +jobs: + #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. + linux-x86_64: + runs-on: ubuntu-18.04 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - uses: ./.github/actions/install-protobuf-linux + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Build on linux-x86_64 + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + mvn --version + cmake --version + protoc --version + sudo apt-get autoremove + sudo apt-get clean + mvn -X -Possrh -Djavacpp.platform=linux-x86_64 -Dlibnd4j.chip=cpu -Pcpu --batch-mode deploy -DskipTests + + diff --git a/.github/workflows/build-deploy-mac-arm64.yml b/.github/workflows/build-deploy-mac-arm64.yml new file mode 100644 index 000000000..bb1b81888 --- /dev/null +++ b/.github/workflows/build-deploy-mac-arm64.yml @@ -0,0 +1,33 @@ +on: + push: +jobs: + mac-x86_64: + runs-on: macos-10.15 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Build and install + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + brew install unzip ccache gcc swig autoconf-archive automake cmake libomp libtool libusb ant maven nasm xz pkg-config sdl gpg1 bison flex perl ragel binutils gradle gmp isl libmpc mpfr wget python + mvn -Possrh -Djavacpp.platform=macosx-arm64 -Djavacpp.platform=macosx-arm64 -pl ":nd4j-native,:libnd4j" --also-make -Dlibnd4j.platform=macosx-x86_64 -Dlibnd4j.chip=cpu clean --batch-mode deploy -DskipTests + + diff --git a/.github/workflows/build-deploy-mac.yml b/.github/workflows/build-deploy-mac.yml new file mode 100644 index 000000000..e1fc56bd9 --- /dev/null +++ b/.github/workflows/build-deploy-mac.yml @@ -0,0 +1,33 @@ +on: + push: +jobs: + mac-x86_64: + runs-on: macos-10.15 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Build and install + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + brew install unzip ccache gcc swig autoconf-archive automake cmake libomp libtool libusb ant maven nasm xz pkg-config sdl gpg1 bison flex perl ragel binutils gradle gmp isl libmpc mpfr wget python + mvn -Possrh -Djavacpp.platform=macosx-x86_64 -Djavacpp.platform=macosx-x86_64 -pl ":nd4j-native,:libnd4j" --also-make -Dlibnd4j.platform=macosx-x86_64 -Dlibnd4j.chip=cpu clean --batch-mode deploy -DskipTests + + diff --git a/.github/workflows/build-deploy-windows-cuda-11.0.yml b/.github/workflows/build-deploy-windows-cuda-11.0.yml new file mode 100644 index 000000000..117d6e161 --- /dev/null +++ b/.github/workflows/build-deploy-windows-cuda-11.0.yml @@ -0,0 +1,45 @@ +on: + push: +jobs: + windows-x86_64-cuda-11-0: + runs-on: windows-2019 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - name: Use existing msys2 to setup environment + uses: ./.github/actions/msys2-base-setup + - uses: konduitai/cuda-install/.github/actions/install-cuda-windows@master + env: + cuda: 11.0.167 + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Run windows build + shell: cmd + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + set MSYSTEM=MINGW64 + set "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0" + which cmake + dir "%CUDA_PATH%" + dir "%CUDA_PATH%\lib" + set "PATH=C:\msys64\usr\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0\lib\x64;%PATH%" + echo "Running cuda build" + mvn -Possrh -Djavacpp.platform=windows-x86_64 -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -Djavacpp.platform=windows-x86_64 -pl ":nd4j-cuda-11.0,:deeplearning4j-cuda-11.0,:libnd4j" --also-make -Dlibnd4j.platform=windows-x86_64 -Pcuda -Dlibnd4j.chip=cuda -Pcuda clean --batch-mode deploy -DskipTests + + diff --git a/.github/workflows/build-deploy-windows-cuda-11.2.yml b/.github/workflows/build-deploy-windows-cuda-11.2.yml new file mode 100644 index 000000000..d46cb094f --- /dev/null +++ b/.github/workflows/build-deploy-windows-cuda-11.2.yml @@ -0,0 +1,47 @@ +on: + push: +jobs: + windows-x86_64-cuda-11-2: + runs-on: windows-2019 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - name: Use existing msys2 to setup environment + uses: ./.github/actions/msys2-base-setup + - uses: konduitai/cuda-install/.github/actions/install-cuda-windows@master + env: + cuda: 11.2.1 + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Run cuda build + shell: cmd + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + set MSYSTEM=MINGW64 + set "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2" + dir "%CUDA_PATH%" + dir "%CUDA_PATH%\lib" + which cmake + set "PATH=C:\msys64\usr\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\lib\x64;%PATH%" + echo "Running cuda build" + bash ./change-cuda-versions.sh 11.2 + sudo apt-get autoremove + sudo apt-get clean + mvn -Possrh -Djavacpp.platform=linux-x86_64 -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -Djavacpp.platform=windows-x86_64 -pl ":nd4j-cuda-11.2,:libnd4j,:deeplearning4j-cuda-11.2" --also-make -Dlibnd4j.platform=windows-x86_64 -Pcuda -Dlibnd4j.chip=cuda -Pcuda clean --batch-mode deploy -DskipTests + diff --git a/.github/workflows/build-deploy-windows.yml b/.github/workflows/build-deploy-windows.yml new file mode 100644 index 000000000..e55099e66 --- /dev/null +++ b/.github/workflows/build-deploy-windows.yml @@ -0,0 +1,34 @@ +on: + push: +jobs: + + windows-x86_64: + runs-on: windows-2019 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.8.0 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + - uses: ./.github/actions/msys2-base-setup + - name: Set up Java for publishing to GitHub Packages + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Run windows cpu build + shell: cmd + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_TO: ossrh + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} + run: | + set MSYSTEM=MINGW64 + set "PATH=C:\msys64\usr\bin;%PATH%" + mvn -Possrh -Djavacpp.platform=windows-x86_64 -pl ":nd4j-native,:libnd4j" --also-make -Dlibnd4j.platform=windows-x86_64 -Dlibnd4j.chip=cpu deploy -DskipTests From 5963a15d47bc7206bcbd81cb1e40c7d910a5b150 Mon Sep 17 00:00:00 2001 From: agibsonccc Date: Fri, 5 Mar 2021 16:41:12 +0900 Subject: [PATCH 02/11] Update line endings --- ...ild-deploy-linux-jetsonnano-selfhosted.yml | 50 ------------------- .github/workflows/build-deploy-mac-arm64.yml | 33 ------------ 2 files changed, 83 deletions(-) delete mode 100644 .github/workflows/build-deploy-linux-jetsonnano-selfhosted.yml delete mode 100644 .github/workflows/build-deploy-mac-arm64.yml diff --git a/.github/workflows/build-deploy-linux-jetsonnano-selfhosted.yml b/.github/workflows/build-deploy-linux-jetsonnano-selfhosted.yml deleted file mode 100644 index ef0b02ca0..000000000 --- a/.github/workflows/build-deploy-linux-jetsonnano-selfhosted.yml +++ /dev/null @@ -1,50 +0,0 @@ -on: - push: -jobs: - #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. - linux-arm64: - runs-on: self-hosted - steps: - - uses: AutoModality/action-clean@v1 - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.8.0 - with: - access_token: ${{ github.token }} - - uses: actions/checkout@v2 - - name: Set up Java for publishing to GitHub Packages - uses: actions/setup-java@v1 - with: - java-version: 1.8 - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Build on linux-x86_64 - shell: bash - env: - DEBIAN_FRONTEND: noninteractive - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DEPLOY: 1 - BUILD_USING_MAVEN: 1 - TARGET_OS: linux - CURRENT_TARGET: arm64 - CROSS_ROOT: ${GITHUB_WORKSPACE}/rootfs-nano - CROSS_COMPILE: ${GITHUB_WORKSPACE}/toolchain/gcc-4.8.5-aarch64/bin - CC: ${CROSS_COMPILE}/aarch64-unknown-linux-gnu-gcc - CXX: ${CROSS_COMPILE}/aarch64-unknown-linux-gnu-g++ - LD: ${CROSS_COMPILE}/aarch64-unknown-linux-gnu-ld - AR: ${CROSS_COMPILE}/aarch64-unknown-linux-gnu-ar - AS: ${CROSS_COMPILE}/aarch64-unknown-linux-gnu-as - RANLIB: ${CROSS_COMPILE}/aarch64-unknown-linux-gnu-ranlib - NVCC: ${GITHUB_WORKSPACE}/cuda-10.2/bin/nvcc - - run: | - wget https://developer.nvidia.com/embedded/dlc/l4t-gcc-toolchain-64-bit-28-3 - tar xpvf l4t-gcc-toolchain-64-bit-28-3 - mv install gcc-4.8.5-aarch64 - ${GITHUB_WORKSPACE}/change-cuda-versions.sh 10.2 - wget https://github.com/KonduitAI/deeplearning4j/releases/download/cuda-10.2/cuda-10.2.tar - tar xvf cuda-10.2.tar - mvn --version - cmake --version - protoc --version - ${GITHUB_WORKSPACE}/libnd4j/pi_build.sh - - diff --git a/.github/workflows/build-deploy-mac-arm64.yml b/.github/workflows/build-deploy-mac-arm64.yml deleted file mode 100644 index bb1b81888..000000000 --- a/.github/workflows/build-deploy-mac-arm64.yml +++ /dev/null @@ -1,33 +0,0 @@ -on: - push: -jobs: - mac-x86_64: - runs-on: macos-10.15 - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.8.0 - with: - access_token: ${{ github.token }} - - uses: actions/checkout@v2 - - name: Set up Java for publishing to GitHub Packages - uses: actions/setup-java@v1 - with: - java-version: 1.8 - server-id: sonatype-nexus-snapshots - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} - gpg-passphrase: MAVEN_GPG_PASSPHRASE - - name: Build and install - shell: bash - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PUBLISH_TO: ossrh - MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} - MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} - MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} - run: | - brew install unzip ccache gcc swig autoconf-archive automake cmake libomp libtool libusb ant maven nasm xz pkg-config sdl gpg1 bison flex perl ragel binutils gradle gmp isl libmpc mpfr wget python - mvn -Possrh -Djavacpp.platform=macosx-arm64 -Djavacpp.platform=macosx-arm64 -pl ":nd4j-native,:libnd4j" --also-make -Dlibnd4j.platform=macosx-x86_64 -Dlibnd4j.chip=cpu clean --batch-mode deploy -DskipTests - - From ec37e0438cede53aaed4665fe972fe6138d9af32 Mon Sep 17 00:00:00 2001 From: agibsonccc Date: Fri, 5 Mar 2021 17:13:48 +0900 Subject: [PATCH 03/11] Update line endings --- .../workflows/build-deploy-linux-arm32.yml | 20 +++++++++---------- .../workflows/build-deploy-linux-arm64.yml | 20 +++++++++---------- .../build-deploy-linux-cuda-11.0.yml | 16 +++++++-------- .github/workflows/build-deploy-windows.yml | 2 +- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build-deploy-linux-arm32.yml b/.github/workflows/build-deploy-linux-arm32.yml index b51bf385b..343661552 100644 --- a/.github/workflows/build-deploy-linux-arm32.yml +++ b/.github/workflows/build-deploy-linux-arm32.yml @@ -15,12 +15,12 @@ jobs: - name: Set up Java for publishing to GitHub Packages uses: actions/setup-java@v1 with: - java-version: 1.8 - server-id: sonatype-nexus-snapshots - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} - gpg-passphrase: MAVEN_GPG_PASSPHRASE + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE - name: Build on linux-arm32 shell: bash env: @@ -35,9 +35,9 @@ jobs: MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} run: | - mvn --version - cmake --version - protoc --version - ${GITHUB_WORKSPACE}/libnd4j/pi_build.sh + mvn --version + cmake --version + protoc --version + ${GITHUB_WORKSPACE}/libnd4j/pi_build.sh diff --git a/.github/workflows/build-deploy-linux-arm64.yml b/.github/workflows/build-deploy-linux-arm64.yml index ff9ef5ae3..19db59629 100644 --- a/.github/workflows/build-deploy-linux-arm64.yml +++ b/.github/workflows/build-deploy-linux-arm64.yml @@ -15,12 +15,12 @@ jobs: - name: Set up Java for publishing to GitHub Packages uses: actions/setup-java@v1 with: - java-version: 1.8 - server-id: sonatype-nexus-snapshots - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} - gpg-passphrase: MAVEN_GPG_PASSPHRASE + java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE - name: Build on linux-arm64 shell: bash env: @@ -32,9 +32,9 @@ jobs: CURRENT_TARGET: arm64 PUBLISH_TO: ossrh run: | - mvn --version - cmake --version - protoc --version - ${GITHUB_WORKSPACE}/libnd4j/pi_build.sh + mvn --version + cmake --version + protoc --version + ${GITHUB_WORKSPACE}/libnd4j/pi_build.sh diff --git a/.github/workflows/build-deploy-linux-cuda-11.0.yml b/.github/workflows/build-deploy-linux-cuda-11.0.yml index 5e2effd39..598e2e6ea 100644 --- a/.github/workflows/build-deploy-linux-cuda-11.0.yml +++ b/.github/workflows/build-deploy-linux-cuda-11.0.yml @@ -42,13 +42,13 @@ jobs: MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} run: | - export PATH="/usr/local/cuda-11.0/bin:$PATH" - mvn --version - cmake --version - protoc --version - nvcc --version - sudo apt-get autoremove - sudo apt-get clean - mvn -Possrh -Djavacpp.platform=linux-x86_64 -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -Dlibnd4j.chip=cuda -pl ":nd4j-cuda-11.0,:deeplearning4j-cuda-11.0,:libnd4j" --also-make -Pcuda clean --batch-mode deploy -DskipTests + export PATH="/usr/local/cuda-11.0/bin:$PATH" + mvn --version + cmake --version + protoc --version + nvcc --version + sudo apt-get autoremove + sudo apt-get clean + mvn -Possrh -Djavacpp.platform=linux-x86_64 -Dlibnd4j.compute="5.0 5.2 5.3 6.0 6.2 8.0" -Dlibnd4j.chip=cuda -pl ":nd4j-cuda-11.0,:deeplearning4j-cuda-11.0,:libnd4j" --also-make -Pcuda clean --batch-mode deploy -DskipTests diff --git a/.github/workflows/build-deploy-windows.yml b/.github/workflows/build-deploy-windows.yml index e55099e66..daa6839dc 100644 --- a/.github/workflows/build-deploy-windows.yml +++ b/.github/workflows/build-deploy-windows.yml @@ -1,7 +1,7 @@ on: push: jobs: - + windows-x86_64: runs-on: windows-2019 steps: From 92b9c54833bb9284357d1b42e9c6e506976f3039 Mon Sep 17 00:00:00 2001 From: agibsonccc Date: Fri, 5 Mar 2021 17:16:11 +0900 Subject: [PATCH 04/11] Update build-android-x86_64.yml --- .github/workflows/build-android-x86_64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-android-x86_64.yml b/.github/workflows/build-android-x86_64.yml index b7ed65a4d..1236c569d 100644 --- a/.github/workflows/build-android-x86_64.yml +++ b/.github/workflows/build-android-x86_64.yml @@ -5,7 +5,6 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: AutoModality/action-clean@v1 - - uses: ./.github/actions/install-protobuf-linux - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.8.0 with: @@ -14,6 +13,7 @@ jobs: with: ndk-version: r18b - uses: actions/checkout@v2 + - uses: ./.github/actions/install-protobuf-linux - name: Set up Java for publishing to GitHub Packages uses: actions/setup-java@v1 with: From d54a91505b482fcd551cf2468041f0a927940e2f Mon Sep 17 00:00:00 2001 From: agibsonccc Date: Fri, 5 Mar 2021 17:26:06 +0900 Subject: [PATCH 05/11] Update pi_build.sh --- libnd4j/pi_build.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/libnd4j/pi_build.sh b/libnd4j/pi_build.sh index d1e3ba1fc..712609447 100755 --- a/libnd4j/pi_build.sh +++ b/libnd4j/pi_build.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh # # /* ****************************************************************************** # * @@ -148,10 +148,10 @@ function download_extract_base { message "download" wget --quiet --show-progress -O ${3}_file ${2} fi - + message "extract $@" #extract - mkdir -p ${3} + mkdir -p ${3} if [ ${1} = "-unzip" ]; then command="unzip -qq ${3}_file -d ${3} " else @@ -178,12 +178,12 @@ function git_check { #$1 is url #$2 is dir #$3 is tag or branch if optional command= if [ -n "$3" ]; then - command="git clone --quiet --depth 1 --branch ${3} ${1} ${2}" - else + command="git clone --quiet --depth 1 --branch ${3} ${1} ${2}" + else command="git clone --quiet ${1} ${2}" fi message "$command" - $command + $command check_requirements "${2}" } @@ -195,7 +195,7 @@ function fix_pi_linker { fi rm -f ${1}/ld printf '#!/usr/bin/env bash\n'"${1}/ld.gold --long-plt \$*">${1}/ld - chmod +x ${1}/ld + chmod +x ${1}/ld } if [ ! -d ${CROSS_COMPILER_DIR}/folder ]; then @@ -238,14 +238,14 @@ fi check_requirements ${CC} if [ -z "${BUILD_USING_MAVEN}" ] ;then -#lets build OpenBlas +#lets build OpenBlas if [ ! -d "${OPENBLAS_DIR}" ]; then message "download OpenBLAS" git_check "${OPENBLAS_GIT_URL}" "${OPENBLAS_DIR}" "v0.3.10" fi if [ ! -f "${THIRD_PARTY}/lib/libopenblas.so" ]; then - message "build and install OpenBLAS" + message "build and install OpenBLAS" cd ${OPENBLAS_DIR} command="make TARGET=${BLAS_TARGET_NAME} HOSTCC=gcc NOFORTRAN=1 ${BLAS_XTRA} " @@ -271,9 +271,9 @@ if [ ! -d ${SCONS_LOCAL_DIR} ]; then fi check_requirements ${SCONS_LOCAL_DIR}/scons.py -if [ ! -d "${ARMCOMPUTE_DIR}" ]; then - message "download ArmCompute Source" - git_check ${ARMCOMPUTE_GIT_URL} "${ARMCOMPUTE_DIR}" "${ARMCOMPUTE_TAG}" +if [ ! -d "${ARMCOMPUTE_DIR}" ]; then + message "download ArmCompute Source" + git_check ${ARMCOMPUTE_GIT_URL} "${ARMCOMPUTE_DIR}" "${ARMCOMPUTE_TAG}" fi #build armcompute @@ -283,7 +283,7 @@ cd ${ARMCOMPUTE_DIR} command="CC=${CC_EXE} CXX=${CXX_EXE} python3 ${SCONS_LOCAL_DIR}/scons.py Werror=1 -j$(nproc) toolchain_prefix=${TOOLCHAIN_PREFIX}- compiler_prefix=${COMPILER_PREFIX}- debug=${ARMCOMPUTE_DEBUG} neon=1 opencl=0 extra_cxx_flags=-fPIC os=${TARGET_OS} build=cross_compile arch=${ARMCOMPUTE_TARGET} " message $command eval $command &>/dev/null -cd ${BASE_DIR} +cd ${BASE_DIR} fi check_requirements "${ARMCOMPUTE_DIR}/build/libarm_compute-static.a" "${ARMCOMPUTE_DIR}/build/libarm_compute_core-static.a" From c6b1dfa79e73e4993e08f342e558e96bfc6fe9ab Mon Sep 17 00:00:00 2001 From: agibsonccc Date: Fri, 5 Mar 2021 17:34:24 +0900 Subject: [PATCH 06/11] Update pi_build.sh --- libnd4j/pi_build.sh | 640 ++++++++++++++++++++++---------------------- 1 file changed, 320 insertions(+), 320 deletions(-) diff --git a/libnd4j/pi_build.sh b/libnd4j/pi_build.sh index 712609447..6f7d9965b 100755 --- a/libnd4j/pi_build.sh +++ b/libnd4j/pi_build.sh @@ -1,320 +1,320 @@ -#!/bin/sh -# -# /* ****************************************************************************** -# * -# * -# * This program and the accompanying materials are made available under the -# * terms of the Apache License, Version 2.0 which is available at -# * https://www.apache.org/licenses/LICENSE-2.0. -# * -# * See the NOTICE file distributed with this work for additional -# * information regarding copyright ownership. -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# * License for the specific language governing permissions and limitations -# * under the License. -# * -# * SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************/ -# - -function message { - echo "BUILDER:::: ${@}" -} - -BUILD_USING_MAVEN= -CURRENT_TARGET=arm32 -HAS_ARMCOMPUTE=1 -ARMCOMPUTE_DEBUG=0 -ARMCOMPUTE_TAG=v20.05 -LIBND4J_BUILD_MODE=Release -export ANDROID_VERSION=21 -OTHER_ARGS=() -while [[ $# -gt 0 ]] -do -key="$1" - -case $key in - -a|--arch) - CURRENT_TARGET="$2" - shift - shift - ;; - -m|--mvn) - BUILD_USING_MAVEN="mvn" - shift - ;; - *) - OTHER_ARGS+=("$1") - shift - ;; -esac -done - -CC_URL32="https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf.tar.xz?revision=e09a1c45-0ed3-4a8e-b06b-db3978fd8d56&la=en&hash=93ED4444B8B3A812B893373B490B90BBB28FD2E3" -CC_URL64="https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en&hash=167687FADA00B73D20EED2A67D0939A197504ACD" -CC_ANDROID="https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip" -TARGET_ARRS=( arm32 arm64 android-arm android-arm64 ) -COMPILER_ARRS=( "${CC_URL32}" "${CC_URL64}" "${CC_ANDROID}" "${CC_ANDROID}" ) -COMPILER_DOWNLOAD_CMD_LIST=( download_extract_xz download_extract_xz download_extract_unzip download_extract_unzip ) -COMPILER_DESTDIR=( "arm32" "arm64" "android" "android" ) - -OPENBLAS_TARGETS=( ARMV7 ARMV8 ARMV7 ARMV8) -ARMCOMPUTE_TARGETS=( armv7a arm64-v8a armv7a arm64-v8a) -OS_LIST=( linux linux android android) -LIBND4J_PLATFORM_EXT_LIST=( armhf arm64 arm arm64 ) -PREFIXES=( arm-linux-gnueabihf aarch64-linux-gnu arm-linux-androideabi aarch64-linux-android ) -TARGET_INDEX=-1 - -for i in "${!TARGET_ARRS[@]}"; do - if [[ "${TARGET_ARRS[$i]}" = "${CURRENT_TARGET}" ]]; then - TARGET_INDEX=${i} - fi -done - -if [ ${TARGET_INDEX} -eq -1 ];then - message "could not find ${CURRENT_TARGET} in ${TARGET_ARRS[@]}" - exit -1 -fi - -#BASE_DIR=${HOME}/pi -#https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -BASE_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - -CROSS_COMPILER_URL=${COMPILER_ARRS[$TARGET_INDEX]} -CROSS_COMPILER_DIR=${BASE_DIR}/compile_tools/cross_compiler_${COMPILER_DESTDIR[$TARGET_INDEX]} -COMPILER_DOWNLOAD_CMD=${COMPILER_DOWNLOAD_CMD_LIST[$TARGET_INDEX]} -DETECT=${DETECT_LIST[$TARGET_INDEX]} -LIBND4J_PLATFORM_EXT=${LIBND4J_PLATFORM_EXT_LIST[$TARGET_INDEX]} -BLAS_TARGET_NAME=${OPENBLAS_TARGETS[$TARGET_INDEX]} -ARMCOMPUTE_TARGET=${ARMCOMPUTE_TARGETS[$TARGET_INDEX]} -TARGET_OS=${OS_LIST[$TARGET_INDEX]} -LIBND4J_PLATFORM=${TARGET_OS}-${LIBND4J_PLATFORM_EXT} -PREFIX=${PREFIXES[$TARGET_INDEX]} - -CMAKE=cmake #/snap/bin/cmake -mkdir -p ${BASE_DIR}/compile_tools/ - -SCONS_LOCAL_URL=http://prdownloads.sourceforge.net/scons/scons-local-3.1.1.tar.gz -SCONS_LOCAL_DIR=${BASE_DIR}/compile_tools/scons_local - -THIRD_PARTY=${BASE_DIR}/third_party_libs${TARGET_INDEX} - -ARMCOMPUTE_GIT_URL=https://github.com/ARM-software/ComputeLibrary.git -ARMCOMPUTE_DIR=${THIRD_PARTY}/arm_compute_dir - -OPENBLAS_GIT_URL="https://github.com/xianyi/OpenBLAS.git" -OPENBLAS_DIR=${THIRD_PARTY}/OpenBLAS - - -mkdir -p ${BASE_DIR} -mkdir -p ${THIRD_PARTY} - -#change directory to base -cd $BASE_DIR - -function check_requirements { - for i in "${@}" - do - if [ ! -e "$i" ]; then - message "missing: ${i}" - exit -2 - fi - done -} - -function rename_top_folder { - for dir in ${1}/* - do - if [ -d "$dir" ] - then - mv "${dir}" "${1}/folder/" - message "${dir} => ${1}/folder/" - break - fi - done -} - -function download_extract_base { - #$1 is url #2 is dir $3 is extract argument - if [ ! -f ${3}_file ]; then - message "download" - wget --quiet --show-progress -O ${3}_file ${2} - fi - - message "extract $@" - #extract - mkdir -p ${3} - if [ ${1} = "-unzip" ]; then - command="unzip -qq ${3}_file -d ${3} " - else - command="tar ${1} ${3}_file --directory=${3} " - fi - message $command - $command - check_requirements "${3}" -} - -function download_extract { - download_extract_base -xzf $@ -} - -function download_extract_xz { - download_extract_base -xf $@ -} - -function download_extract_unzip { - download_extract_base -unzip $@ -} - -function git_check { - #$1 is url #$2 is dir #$3 is tag or branch if optional - command= - if [ -n "$3" ]; then - command="git clone --quiet --depth 1 --branch ${3} ${1} ${2}" - else - command="git clone --quiet ${1} ${2}" - fi - message "$command" - $command - check_requirements "${2}" -} - -#fix py debug linkage manually and also makes it use gold -function fix_pi_linker { - #$1 BINUTILS folder - if [ ! -f ${1}/ld.original ]; then - mv ${1}/ld ${1}/ld.original - fi - rm -f ${1}/ld - printf '#!/usr/bin/env bash\n'"${1}/ld.gold --long-plt \$*">${1}/ld - chmod +x ${1}/ld -} - -if [ ! -d ${CROSS_COMPILER_DIR}/folder ]; then - #out file - message "download CROSS_COMPILER" - ${COMPILER_DOWNLOAD_CMD} ${CROSS_COMPILER_URL} ${CROSS_COMPILER_DIR} - message "rename top folder (instead of --strip-components=1)" - rename_top_folder ${CROSS_COMPILER_DIR} -fi - -CROSS_COMPILER_DIR=${CROSS_COMPILER_DIR}/folder - -if [ "${TARGET_OS}" = "android" ];then - ANDROID_TOOLCHAIN=${CROSS_COMPILER_DIR}/toolchains/llvm/prebuilt/linux-x86_64 - COMPILER_PREFIX="${ANDROID_TOOLCHAIN}/bin/${PREFIX}${ANDROID_VERSION}" - TOOLCHAIN_PREFIX="${ANDROID_TOOLCHAIN}/bin/${PREFIX}" - if [ "$BLAS_TARGET_NAME" = "ARMV7" ];then - BLAS_XTRA="ARM_SOFTFP_ABI=1 " - COMPILER_PREFIX="${ANDROID_TOOLCHAIN}/bin/armv7a-linux-androideabi${ANDROID_VERSION}" - fi - CC_EXE="clang" - CXX_EXE="clang++" - AR="${TOOLCHAIN_PREFIX}-ar" - RANLIB="${TOOLCHAIN_PREFIX}-ranlib" - BLAS_XTRA="CC=${COMPILER_PREFIX}-${CC_EXE} AR=${AR} RANLIB=${RANLIB} ${BLAS_XTRA}" -else - BINUTILS_BIN=${CROSS_COMPILER_DIR}/${PREFIX}/bin - COMPILER_PREFIX=${CROSS_COMPILER_DIR}/bin/${PREFIX} - TOOLCHAIN_PREFIX=${COMPILER_PREFIX} - SYS_ROOT=${CROSS_COMPILER_DIR}/${PREFIX}/libc - #LD_LIBRARY_PATH=${CROSS_COMPILER_DIR}/lib:$LD_LIBRARY_PATH - CC_EXE="gcc" - CXX_EXE="g++" - RANLIB="${BINUTILS_BIN}/ranlib" - export LD="${BINUTILS_BIN}/ld" - AR="${BINUTILS_BIN}/ar" - BLAS_XTRA="CC=${COMPILER_PREFIX}-${CC_EXE} AR=${AR} RANLIB=${RANLIB} CFLAGS=--sysroot=${SYS_ROOT} LDFLAGS=\"-L${SYS_ROOT}/../lib/ -lm\"" -fi - -check_requirements ${CC} - -if [ -z "${BUILD_USING_MAVEN}" ] ;then -#lets build OpenBlas -if [ ! -d "${OPENBLAS_DIR}" ]; then - message "download OpenBLAS" - git_check "${OPENBLAS_GIT_URL}" "${OPENBLAS_DIR}" "v0.3.10" -fi - -if [ ! -f "${THIRD_PARTY}/lib/libopenblas.so" ]; then - message "build and install OpenBLAS" - cd ${OPENBLAS_DIR} - - command="make TARGET=${BLAS_TARGET_NAME} HOSTCC=gcc NOFORTRAN=1 ${BLAS_XTRA} " - message $command - eval $command &>/dev/null - message "install it" - command="make TARGET=${BLAS_TARGET_NAME} PREFIX=${THIRD_PARTY} install &>/dev/null" - message $command - $command - cd $BASE_DIR - -fi -check_requirements ${THIRD_PARTY}/lib/libopenblas.so - -export OPENBLAS_PATH=${THIRD_PARTY} - -fi # end if [ -z "${BUILD_USING_MAVEN}"];then - -if [ ! -d ${SCONS_LOCAL_DIR} ]; then - #out file - message "download Scons local" - download_extract ${SCONS_LOCAL_URL} ${SCONS_LOCAL_DIR} -fi -check_requirements ${SCONS_LOCAL_DIR}/scons.py - -if [ ! -d "${ARMCOMPUTE_DIR}" ]; then - message "download ArmCompute Source" - git_check ${ARMCOMPUTE_GIT_URL} "${ARMCOMPUTE_DIR}" "${ARMCOMPUTE_TAG}" -fi - -#build armcompute -if [ ! -f "${ARMCOMPUTE_DIR}/build/libarm_compute-static.a" ]; then -message "build arm compute" -cd ${ARMCOMPUTE_DIR} -command="CC=${CC_EXE} CXX=${CXX_EXE} python3 ${SCONS_LOCAL_DIR}/scons.py Werror=1 -j$(nproc) toolchain_prefix=${TOOLCHAIN_PREFIX}- compiler_prefix=${COMPILER_PREFIX}- debug=${ARMCOMPUTE_DEBUG} neon=1 opencl=0 extra_cxx_flags=-fPIC os=${TARGET_OS} build=cross_compile arch=${ARMCOMPUTE_TARGET} " -message $command -eval $command &>/dev/null -cd ${BASE_DIR} -fi -check_requirements "${ARMCOMPUTE_DIR}/build/libarm_compute-static.a" "${ARMCOMPUTE_DIR}/build/libarm_compute_core-static.a" - -export ARMCOMPUTE_ROOT="${ARMCOMPUTE_DIR}" - -if [ "${TARGET_OS}" = "android" ];then - export ANDROID_NDK=${CROSS_COMPILER_DIR} -else - export RPI_BIN=${CROSS_COMPILER_DIR}/bin/${PREFIX} - export JAVA_LIBRARY_PATH=${CROSS_COMPILER_DIR}/${PREFIX}/lib - fix_pi_linker ${BINUTILS_BIN} -fi - - -#because of the toolchain passive detection we have to delete build folder manually -detect=$(cat ${BASE_DIR}/blasbuild/cpu/CMakeCache.txt | grep -o ${PREFIX}) -if [ -z "${detect}" ] ;then -message "remove blasbuild folder " -rm -rf $BASE_DIR/blasbuild/ -else -message "keep blasbuild folder" -fi - -if [ -z "${BUILD_USING_MAVEN}" ] ;then -message "lets build just library" -DHELPER=" -h armcompute " -bash ./buildnativeoperations.sh -o ${LIBND4J_PLATFORM} -t ${DHELPER} -j $(nproc) -else -message "cd $BASE_DIR/.. " -cd $BASE_DIR/.. -message "lets build jars" -DHELPER=" -Dlibnd4j.helper=armcompute " -mvn install -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER} -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -fi +#!/bin/sh +# +# /* ****************************************************************************** +# * +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * See the NOTICE file distributed with this work for additional +# * information regarding copyright ownership. +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ +# + +function message { + echo "BUILDER:::: ${@}" +} + +BUILD_USING_MAVEN= +CURRENT_TARGET=arm32 +HAS_ARMCOMPUTE=1 +ARMCOMPUTE_DEBUG=0 +ARMCOMPUTE_TAG=v20.05 +LIBND4J_BUILD_MODE=Release +export ANDROID_VERSION=21 +OTHER_ARGS=() +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + -a|--arch) + CURRENT_TARGET="$2" + shift + shift + ;; + -m|--mvn) + BUILD_USING_MAVEN="mvn" + shift + ;; + *) + OTHER_ARGS+=("$1") + shift + ;; +esac +done + +CC_URL32="https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf.tar.xz?revision=e09a1c45-0ed3-4a8e-b06b-db3978fd8d56&la=en&hash=93ED4444B8B3A812B893373B490B90BBB28FD2E3" +CC_URL64="https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en&hash=167687FADA00B73D20EED2A67D0939A197504ACD" +CC_ANDROID="https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip" +TARGET_ARRS=( arm32 arm64 android-arm android-arm64 ) +COMPILER_ARRS=( "${CC_URL32}" "${CC_URL64}" "${CC_ANDROID}" "${CC_ANDROID}" ) +COMPILER_DOWNLOAD_CMD_LIST=( download_extract_xz download_extract_xz download_extract_unzip download_extract_unzip ) +COMPILER_DESTDIR=( "arm32" "arm64" "android" "android" ) + +OPENBLAS_TARGETS=( ARMV7 ARMV8 ARMV7 ARMV8) +ARMCOMPUTE_TARGETS=( armv7a arm64-v8a armv7a arm64-v8a) +OS_LIST=( linux linux android android) +LIBND4J_PLATFORM_EXT_LIST=( armhf arm64 arm arm64 ) +PREFIXES=( arm-linux-gnueabihf aarch64-linux-gnu arm-linux-androideabi aarch64-linux-android ) +TARGET_INDEX=-1 + +for i in "${!TARGET_ARRS[@]}"; do + if [[ "${TARGET_ARRS[$i]}" = "${CURRENT_TARGET}" ]]; then + TARGET_INDEX=${i} + fi +done + +if [ ${TARGET_INDEX} -eq -1 ];then + message "could not find ${CURRENT_TARGET} in ${TARGET_ARRS[@]}" + exit -1 +fi + +#BASE_DIR=${HOME}/pi +#https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +BASE_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + +CROSS_COMPILER_URL=${COMPILER_ARRS[$TARGET_INDEX]} +CROSS_COMPILER_DIR=${BASE_DIR}/compile_tools/cross_compiler_${COMPILER_DESTDIR[$TARGET_INDEX]} +COMPILER_DOWNLOAD_CMD=${COMPILER_DOWNLOAD_CMD_LIST[$TARGET_INDEX]} +DETECT=${DETECT_LIST[$TARGET_INDEX]} +LIBND4J_PLATFORM_EXT=${LIBND4J_PLATFORM_EXT_LIST[$TARGET_INDEX]} +BLAS_TARGET_NAME=${OPENBLAS_TARGETS[$TARGET_INDEX]} +ARMCOMPUTE_TARGET=${ARMCOMPUTE_TARGETS[$TARGET_INDEX]} +TARGET_OS=${OS_LIST[$TARGET_INDEX]} +LIBND4J_PLATFORM=${TARGET_OS}-${LIBND4J_PLATFORM_EXT} +PREFIX=${PREFIXES[$TARGET_INDEX]} + +CMAKE=cmake #/snap/bin/cmake +mkdir -p ${BASE_DIR}/compile_tools/ + +SCONS_LOCAL_URL=http://prdownloads.sourceforge.net/scons/scons-local-3.1.1.tar.gz +SCONS_LOCAL_DIR=${BASE_DIR}/compile_tools/scons_local + +THIRD_PARTY=${BASE_DIR}/third_party_libs${TARGET_INDEX} + +ARMCOMPUTE_GIT_URL=https://github.com/ARM-software/ComputeLibrary.git +ARMCOMPUTE_DIR=${THIRD_PARTY}/arm_compute_dir + +OPENBLAS_GIT_URL="https://github.com/xianyi/OpenBLAS.git" +OPENBLAS_DIR=${THIRD_PARTY}/OpenBLAS + + +mkdir -p ${BASE_DIR} +mkdir -p ${THIRD_PARTY} + +#change directory to base +cd $BASE_DIR + +function check_requirements { + for i in "${@}" + do + if [ ! -e "$i" ]; then + message "missing: ${i}" + exit -2 + fi + done +} + +function rename_top_folder { + for dir in ${1}/* + do + if [ -d "$dir" ] + then + mv "${dir}" "${1}/folder/" + message "${dir} => ${1}/folder/" + break + fi + done +} + +function download_extract_base { + #$1 is url #2 is dir $3 is extract argument + if [ ! -f ${3}_file ]; then + message "download" + wget --quiet --show-progress -O ${3}_file ${2} + fi + + message "extract $@" + #extract + mkdir -p ${3} + if [ ${1} = "-unzip" ]; then + command="unzip -qq ${3}_file -d ${3} " + else + command="tar ${1} ${3}_file --directory=${3} " + fi + message $command + $command + check_requirements "${3}" +} + +function download_extract { + download_extract_base -xzf $@ +} + +function download_extract_xz { + download_extract_base -xf $@ +} + +function download_extract_unzip { + download_extract_base -unzip $@ +} + +function git_check { + #$1 is url #$2 is dir #$3 is tag or branch if optional + command= + if [ -n "$3" ]; then + command="git clone --quiet --depth 1 --branch ${3} ${1} ${2}" + else + command="git clone --quiet ${1} ${2}" + fi + message "$command" + $command + check_requirements "${2}" +} + +#fix py debug linkage manually and also makes it use gold +function fix_pi_linker { + #$1 BINUTILS folder + if [ ! -f ${1}/ld.original ]; then + mv ${1}/ld ${1}/ld.original + fi + rm -f ${1}/ld + printf '#!/usr/bin/env bash\n'"${1}/ld.gold --long-plt \$*">${1}/ld + chmod +x ${1}/ld +} + +if [ ! -d ${CROSS_COMPILER_DIR}/folder ]; then + #out file + message "download CROSS_COMPILER" + ${COMPILER_DOWNLOAD_CMD} ${CROSS_COMPILER_URL} ${CROSS_COMPILER_DIR} + message "rename top folder (instead of --strip-components=1)" + rename_top_folder ${CROSS_COMPILER_DIR} +fi + +CROSS_COMPILER_DIR=${CROSS_COMPILER_DIR}/folder + +if [ "${TARGET_OS}" = "android" ];then + ANDROID_TOOLCHAIN=${CROSS_COMPILER_DIR}/toolchains/llvm/prebuilt/linux-x86_64 + COMPILER_PREFIX="${ANDROID_TOOLCHAIN}/bin/${PREFIX}${ANDROID_VERSION}" + TOOLCHAIN_PREFIX="${ANDROID_TOOLCHAIN}/bin/${PREFIX}" + if [ "$BLAS_TARGET_NAME" = "ARMV7" ];then + BLAS_XTRA="ARM_SOFTFP_ABI=1 " + COMPILER_PREFIX="${ANDROID_TOOLCHAIN}/bin/armv7a-linux-androideabi${ANDROID_VERSION}" + fi + CC_EXE="clang" + CXX_EXE="clang++" + AR="${TOOLCHAIN_PREFIX}-ar" + RANLIB="${TOOLCHAIN_PREFIX}-ranlib" + BLAS_XTRA="CC=${COMPILER_PREFIX}-${CC_EXE} AR=${AR} RANLIB=${RANLIB} ${BLAS_XTRA}" +else + BINUTILS_BIN=${CROSS_COMPILER_DIR}/${PREFIX}/bin + COMPILER_PREFIX=${CROSS_COMPILER_DIR}/bin/${PREFIX} + TOOLCHAIN_PREFIX=${COMPILER_PREFIX} + SYS_ROOT=${CROSS_COMPILER_DIR}/${PREFIX}/libc + #LD_LIBRARY_PATH=${CROSS_COMPILER_DIR}/lib:$LD_LIBRARY_PATH + CC_EXE="gcc" + CXX_EXE="g++" + RANLIB="${BINUTILS_BIN}/ranlib" + export LD="${BINUTILS_BIN}/ld" + AR="${BINUTILS_BIN}/ar" + BLAS_XTRA="CC=${COMPILER_PREFIX}-${CC_EXE} AR=${AR} RANLIB=${RANLIB} CFLAGS=--sysroot=${SYS_ROOT} LDFLAGS=\"-L${SYS_ROOT}/../lib/ -lm\"" +fi + +check_requirements ${CC} + +if [ -z "${BUILD_USING_MAVEN}" ] ;then +#lets build OpenBlas +if [ ! -d "${OPENBLAS_DIR}" ]; then + message "download OpenBLAS" + git_check "${OPENBLAS_GIT_URL}" "${OPENBLAS_DIR}" "v0.3.10" +fi + +if [ ! -f "${THIRD_PARTY}/lib/libopenblas.so" ]; then + message "build and install OpenBLAS" + cd ${OPENBLAS_DIR} + + command="make TARGET=${BLAS_TARGET_NAME} HOSTCC=gcc NOFORTRAN=1 ${BLAS_XTRA} " + message $command + eval $command &>/dev/null + message "install it" + command="make TARGET=${BLAS_TARGET_NAME} PREFIX=${THIRD_PARTY} install &>/dev/null" + message $command + $command + cd $BASE_DIR + +fi +check_requirements ${THIRD_PARTY}/lib/libopenblas.so + +export OPENBLAS_PATH=${THIRD_PARTY} + +fi # end if [ -z "${BUILD_USING_MAVEN}"];then + +if [ ! -d ${SCONS_LOCAL_DIR} ]; then + #out file + message "download Scons local" + download_extract ${SCONS_LOCAL_URL} ${SCONS_LOCAL_DIR} +fi +check_requirements ${SCONS_LOCAL_DIR}/scons.py + +if [ ! -d "${ARMCOMPUTE_DIR}" ]; then + message "download ArmCompute Source" + git_check ${ARMCOMPUTE_GIT_URL} "${ARMCOMPUTE_DIR}" "${ARMCOMPUTE_TAG}" +fi + +#build armcompute +if [ ! -f "${ARMCOMPUTE_DIR}/build/libarm_compute-static.a" ]; then +message "build arm compute" +cd ${ARMCOMPUTE_DIR} +command="CC=${CC_EXE} CXX=${CXX_EXE} python3 ${SCONS_LOCAL_DIR}/scons.py Werror=1 -j$(nproc) toolchain_prefix=${TOOLCHAIN_PREFIX}- compiler_prefix=${COMPILER_PREFIX}- debug=${ARMCOMPUTE_DEBUG} neon=1 opencl=0 extra_cxx_flags=-fPIC os=${TARGET_OS} build=cross_compile arch=${ARMCOMPUTE_TARGET} " +message $command +eval $command &>/dev/null +cd ${BASE_DIR} +fi +check_requirements "${ARMCOMPUTE_DIR}/build/libarm_compute-static.a" "${ARMCOMPUTE_DIR}/build/libarm_compute_core-static.a" + +export ARMCOMPUTE_ROOT="${ARMCOMPUTE_DIR}" + +if [ "${TARGET_OS}" = "android" ];then + export ANDROID_NDK=${CROSS_COMPILER_DIR} +else + export RPI_BIN=${CROSS_COMPILER_DIR}/bin/${PREFIX} + export JAVA_LIBRARY_PATH=${CROSS_COMPILER_DIR}/${PREFIX}/lib + fix_pi_linker ${BINUTILS_BIN} +fi + + +#because of the toolchain passive detection we have to delete build folder manually +detect=$(cat ${BASE_DIR}/blasbuild/cpu/CMakeCache.txt | grep -o ${PREFIX}) +if [ -z "${detect}" ] ;then +message "remove blasbuild folder " +rm -rf $BASE_DIR/blasbuild/ +else +message "keep blasbuild folder" +fi + +if [ -z "${BUILD_USING_MAVEN}" ] ;then +message "lets build just library" +DHELPER=" -h armcompute " +bash ./buildnativeoperations.sh -o ${LIBND4J_PLATFORM} -t ${DHELPER} -j $(nproc) +else +message "cd $BASE_DIR/.. " +cd $BASE_DIR/.. +message "lets build jars" +DHELPER=" -Dlibnd4j.helper=armcompute " +mvn install -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER} -Dmaven.test.skip=true -Dmaven.javadoc.skip=true +fi From c35984b7ea75411a65e61223700a4371e4334008 Mon Sep 17 00:00:00 2001 From: agibsonccc Date: Fri, 5 Mar 2021 17:41:57 +0900 Subject: [PATCH 07/11] Update pi_build.sh --- libnd4j/pi_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libnd4j/pi_build.sh b/libnd4j/pi_build.sh index 6f7d9965b..5bc3e0109 100755 --- a/libnd4j/pi_build.sh +++ b/libnd4j/pi_build.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # /* ****************************************************************************** # * From 94c7dbc462b7d2a3d0d9fcd5ece5059f0bc0029b Mon Sep 17 00:00:00 2001 From: agibsonccc Date: Fri, 5 Mar 2021 17:52:24 +0900 Subject: [PATCH 08/11] Update build-android-x86_64.yml --- .github/workflows/build-android-x86_64.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-android-x86_64.yml b/.github/workflows/build-android-x86_64.yml index 1236c569d..ed9e15a8e 100644 --- a/.github/workflows/build-android-x86_64.yml +++ b/.github/workflows/build-android-x86_64.yml @@ -10,6 +10,7 @@ jobs: with: access_token: ${{ github.token }} - uses: nttld/setup-ndk@v1 + id: setup-ndk with: ndk-version: r18b - uses: actions/checkout@v2 @@ -20,7 +21,7 @@ jobs: java-version: 1.8 - name: Build on linux-x86_64 env: - ANDROID_NDK: "/opt/hostedcache/ndk/r18b/x64" + ANDROID_NDK: ${{ steps.setup-ndk.outputs.ndk-path }} OPENBLAS_PATH: "/opt/openblas/" LIBND4J_HOME: "${GITHUB_WORKSPACE}/libnd4j" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From edac746504108b065a782fa5ca5f13de141a47fc Mon Sep 17 00:00:00 2001 From: agibsonccc Date: Sat, 6 Mar 2021 08:01:05 +0900 Subject: [PATCH 09/11] Update build-android-x86_64.yml --- .github/workflows/build-android-x86_64.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-android-x86_64.yml b/.github/workflows/build-android-x86_64.yml index ed9e15a8e..65167f0d2 100644 --- a/.github/workflows/build-android-x86_64.yml +++ b/.github/workflows/build-android-x86_64.yml @@ -19,12 +19,20 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 + server-id: sonatype-nexus-snapshots + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE - name: Build on linux-x86_64 env: ANDROID_NDK: ${{ steps.setup-ndk.outputs.ndk-path }} OPENBLAS_PATH: "/opt/openblas/" LIBND4J_HOME: "${GITHUB_WORKSPACE}/libnd4j" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} + MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} run: | echo "Verifying programs on path. Path is $PATH" echo "Path post update is $PATH. Maven is at `which mvn` cmake is at `which cmake` protoc is at `which protoc`" @@ -32,7 +40,7 @@ jobs: cmake --version protoc --version clang --version - mvn -X -Dorg.bytedeco.javacpp.logger.debug=true -Pgithub -pl ":nd4j-native,:libnd4j" --also-make \ + mvn -X -Dorg.bytedeco.javacpp.logger.debug=true -Possrh -pl ":nd4j-native,:libnd4j" --also-make \ -Djavacpp.platform=android-x86_64 \ -Dlibnd4j.platform=android-x86_64 -Dlibnd4j.chip=cpu \ --batch-mode clean deploy -DskipTests From a43695cd627dde060b1377ca89deabaff066e7d8 Mon Sep 17 00:00:00 2001 From: agibsonccc Date: Sat, 6 Mar 2021 08:02:20 +0900 Subject: [PATCH 10/11] Update build-deploy-android-arm64.yml --- .github/workflows/build-deploy-android-arm64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-deploy-android-arm64.yml b/.github/workflows/build-deploy-android-arm64.yml index 794a1c3ad..76d88223d 100644 --- a/.github/workflows/build-deploy-android-arm64.yml +++ b/.github/workflows/build-deploy-android-arm64.yml @@ -3,7 +3,7 @@ on: jobs: #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. android-arm64: - runs-on: self-hosted + runs-on: ubuntu-18.04 steps: - uses: AutoModality/action-clean@v1 - name: Cancel Previous Runs From bae856cd712a86e09d62d49542d00ec50217464b Mon Sep 17 00:00:00 2001 From: agibsonccc Date: Sat, 6 Mar 2021 09:24:24 +0900 Subject: [PATCH 11/11] Run snapshots every 12 hours --- .github/workflows/build-android-x86_64.yml | 3 ++- .github/workflows/build-deploy-android-arm32.yml | 3 ++- .github/workflows/build-deploy-android-arm64.yml | 3 ++- .github/workflows/build-deploy-linux-arm32.yml | 3 ++- .github/workflows/build-deploy-linux-arm64.yml | 3 ++- .github/workflows/build-deploy-linux-cuda-11.0.yml | 3 ++- .github/workflows/build-deploy-linux-cuda-11.2.yml | 3 ++- .github/workflows/build-deploy-linux-x86_64.yml | 3 ++- .github/workflows/build-deploy-mac.yml | 3 ++- .github/workflows/build-deploy-windows-cuda-11.0.yml | 3 ++- .github/workflows/build-deploy-windows-cuda-11.2.yml | 3 ++- .github/workflows/build-deploy-windows.yml | 3 ++- 12 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-android-x86_64.yml b/.github/workflows/build-android-x86_64.yml index 65167f0d2..fa79452e2 100644 --- a/.github/workflows/build-android-x86_64.yml +++ b/.github/workflows/build-android-x86_64.yml @@ -1,5 +1,6 @@ on: - push: + schedule: + - cron: "0 */12 * * *" jobs: android-x86_64: runs-on: ubuntu-18.04 diff --git a/.github/workflows/build-deploy-android-arm32.yml b/.github/workflows/build-deploy-android-arm32.yml index 9cd45c767..73e96ac0a 100644 --- a/.github/workflows/build-deploy-android-arm32.yml +++ b/.github/workflows/build-deploy-android-arm32.yml @@ -1,5 +1,6 @@ on: - push: + schedule: + - cron: "0 */12 * * *" jobs: #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. android-arm32: diff --git a/.github/workflows/build-deploy-android-arm64.yml b/.github/workflows/build-deploy-android-arm64.yml index 76d88223d..586806163 100644 --- a/.github/workflows/build-deploy-android-arm64.yml +++ b/.github/workflows/build-deploy-android-arm64.yml @@ -1,5 +1,6 @@ on: - push: + schedule: + - cron: "0 */12 * * *" jobs: #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. android-arm64: diff --git a/.github/workflows/build-deploy-linux-arm32.yml b/.github/workflows/build-deploy-linux-arm32.yml index 343661552..90aafc1c4 100644 --- a/.github/workflows/build-deploy-linux-arm32.yml +++ b/.github/workflows/build-deploy-linux-arm32.yml @@ -1,5 +1,6 @@ on: - push: + schedule: + - cron: "0 */12 * * *" jobs: #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. linux-arm32: diff --git a/.github/workflows/build-deploy-linux-arm64.yml b/.github/workflows/build-deploy-linux-arm64.yml index 19db59629..f6090c52b 100644 --- a/.github/workflows/build-deploy-linux-arm64.yml +++ b/.github/workflows/build-deploy-linux-arm64.yml @@ -1,5 +1,6 @@ on: - push: + schedule: + - cron: "0 */12 * * *" jobs: #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. linux-arm64: diff --git a/.github/workflows/build-deploy-linux-cuda-11.0.yml b/.github/workflows/build-deploy-linux-cuda-11.0.yml index 598e2e6ea..cc179305d 100644 --- a/.github/workflows/build-deploy-linux-cuda-11.0.yml +++ b/.github/workflows/build-deploy-linux-cuda-11.0.yml @@ -1,5 +1,6 @@ on: - push: + schedule: + - cron: "0 */12 * * *" jobs: linux-x86_64-cuda_11-0: diff --git a/.github/workflows/build-deploy-linux-cuda-11.2.yml b/.github/workflows/build-deploy-linux-cuda-11.2.yml index a1b11d9d7..958b9e0e5 100644 --- a/.github/workflows/build-deploy-linux-cuda-11.2.yml +++ b/.github/workflows/build-deploy-linux-cuda-11.2.yml @@ -1,5 +1,6 @@ on: - push: + schedule: + - cron: "0 */12 * * *" jobs: linux-x86_64-cuda-11-2: runs-on: ubuntu-18.04 diff --git a/.github/workflows/build-deploy-linux-x86_64.yml b/.github/workflows/build-deploy-linux-x86_64.yml index e53fcd174..6e9951dd9 100644 --- a/.github/workflows/build-deploy-linux-x86_64.yml +++ b/.github/workflows/build-deploy-linux-x86_64.yml @@ -1,5 +1,6 @@ on: - push: + schedule: + - cron: "0 */12 * * *" jobs: #Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads. linux-x86_64: diff --git a/.github/workflows/build-deploy-mac.yml b/.github/workflows/build-deploy-mac.yml index e1fc56bd9..d21646bda 100644 --- a/.github/workflows/build-deploy-mac.yml +++ b/.github/workflows/build-deploy-mac.yml @@ -1,5 +1,6 @@ on: - push: + schedule: + - cron: "0 */12 * * *" jobs: mac-x86_64: runs-on: macos-10.15 diff --git a/.github/workflows/build-deploy-windows-cuda-11.0.yml b/.github/workflows/build-deploy-windows-cuda-11.0.yml index 117d6e161..1c1f53691 100644 --- a/.github/workflows/build-deploy-windows-cuda-11.0.yml +++ b/.github/workflows/build-deploy-windows-cuda-11.0.yml @@ -1,5 +1,6 @@ on: - push: + schedule: + - cron: "0 */12 * * *" jobs: windows-x86_64-cuda-11-0: runs-on: windows-2019 diff --git a/.github/workflows/build-deploy-windows-cuda-11.2.yml b/.github/workflows/build-deploy-windows-cuda-11.2.yml index d46cb094f..d89ecfcec 100644 --- a/.github/workflows/build-deploy-windows-cuda-11.2.yml +++ b/.github/workflows/build-deploy-windows-cuda-11.2.yml @@ -1,5 +1,6 @@ on: - push: + schedule: + - cron: "0 */12 * * *" jobs: windows-x86_64-cuda-11-2: runs-on: windows-2019 diff --git a/.github/workflows/build-deploy-windows.yml b/.github/workflows/build-deploy-windows.yml index daa6839dc..5c7f1926d 100644 --- a/.github/workflows/build-deploy-windows.yml +++ b/.github/workflows/build-deploy-windows.yml @@ -1,5 +1,6 @@ on: - push: + schedule: + - cron: "0 */12 * * *" jobs: windows-x86_64: