From becdc100eeb334a99ab1ffa075afa1bbb821e664 Mon Sep 17 00:00:00 2001 From: AbdelRauf Date: Tue, 9 Mar 2021 02:20:28 +0100 Subject: [PATCH 1/2] cross_build: added android {x86, x86_64} and jetson {cross arm64 + host cuda + target cuda aarch64} jetson note: host cuda toolkit should be installed to setup cross environment Signed-off-by: AbdelRauf --- libnd4j/buildnativeoperations.sh | 41 ++--------- libnd4j/cmake/rpi.cmake | 41 +++++++++-- libnd4j/pi_build.sh | 120 ++++++++++++++++++++++--------- 3 files changed, 127 insertions(+), 75 deletions(-) diff --git a/libnd4j/buildnativeoperations.sh b/libnd4j/buildnativeoperations.sh index ec20f400e..0719a910a 100755 --- a/libnd4j/buildnativeoperations.sh +++ b/libnd4j/buildnativeoperations.sh @@ -82,14 +82,11 @@ OPERATIONS= CLEAN="false" MINIFIER="false" TESTS="false" -VERBOSE="false" +VERBOSE="true" VERBOSE_ARG="VERBOSE=1" HELPER= CHECK_VECTORIZATION="OFF" -SYS_ROOT= -EXTRA_LINK_FLAGS= NAME= -EXTRA_CUDA_FLAGS= while [[ $# -gt 0 ]] do key="$1" @@ -402,11 +399,6 @@ if [ -z "$BUILD" ]; then fi -if [ -z "$SYS_ROOT" ]; then - export SYS_ROOT="" -fi - - if [ -z "$CHIP" ]; then CHIP="cpu" fi @@ -518,12 +510,6 @@ if [ "$TESTS" == "true" ]; then fi -if [ "$SYS_ROOT" != "" ]; then - EXTRA_SYSROOT="-DCMAKE_SYSROOT=$SYS_ROOT" - else - EXTRA_SYSROOT="" -fi - ARCH_ARG="-DSD_ARCH=$ARCH -DSD_EXTENSION=$CHIP_EXTENSION" CUDA_COMPUTE="-DCOMPUTE=\"$COMPUTE\"" @@ -531,16 +517,8 @@ CUDA_COMPUTE="-DCOMPUTE=\"$COMPUTE\"" if [ "$CHIP" == "cuda" ] && [ -n "$CHIP_VERSION" ]; then case $OS in linux*) - export CUDA_PATH="/usr/local/cuda-$CHIP_VERSION/" - # Cross compilation for jetson nano - if [ "$ARCH" != "x86-64" ]; then - if [ "$ARCH" == "armv8-a" ]; then - export EXTRA_CUDA_FLAGS="-DCUDA_TARGET_CPU_ARCH=AARCH64" - else - export EXTRA_CUDA_FLAGS="-DCUDA_TARGET_CPU_ARCH=ARM" - fi - else - export EXTRA_CUDA_FLAGS="" + if [ "${CUDA_PATH-}" == "" ]; then + export CUDA_PATH="/usr/local/cuda-$CHIP_VERSION/" fi ;; macosx*) @@ -609,13 +587,6 @@ else IFS=' ' fi -LINKER_FLAGS="" -if [ "$EXTRA_LINK_FLAGS" != "" ]; then - LINKER_FLAGS="-DCMAKE_CXX_LINK_FLAGS=$EXTRA_LINK_FLAGS -DCMAKE_EXE_LINKER_FLAGS=$EXTRA_LINK_FLAGS -DCMAKE_CUDA_FLAGS=$EXTRA_LINK_FLAGS" -fi - - - echo PACKAGING = "${PACKAGING}" echo BUILD = "${BUILD}" echo CHIP = "${CHIP}" @@ -632,12 +603,9 @@ echo NAME = "${NAME_ARG}" echo OPENBLAS_PATH = "$OPENBLAS_PATH" echo CHECK_VECTORIZATION = "$CHECK_VECTORIZATION" echo HELPERS = "$HELPERS" -echo EXTRA_LINK_FLAGS = "$EXTRA_LINK_FLAGS" -echo EXTRA_CUDA_FLAGS = "$EXTRA_CUDA_FLAGS" -echo EXTRA_SYSROOT = "$EXTRA_SYSROOT" mkbuilddir pwd -eval "$CMAKE_COMMAND" "$EXTRA_SYSROOT" "$LINKER_FLAGS" "$EXTRA_CUDA_FLAGS" "$BLAS_ARG" "$ARCH_ARG" "$NAME_ARG" -DSD_CHECK_VECTORIZATION="${CHECK_VECTORIZATION}" "$HELPERS" "$SHARED_LIBS_ARG" "$MINIFIER_ARG" "$OPERATIONS_ARG" "$BUILD_TYPE" "$PACKAGING_ARG" "$EXPERIMENTAL_ARG" "$TESTS_ARG" "$CUDA_COMPUTE" -DOPENBLAS_PATH="$OPENBLAS_PATH" -DDEV=FALSE -DCMAKE_NEED_RESPONSE=YES -DMKL_MULTI_THREADED=TRUE ../.. +eval "$CMAKE_COMMAND" "$BLAS_ARG" "$ARCH_ARG" "$NAME_ARG" -DSD_CHECK_VECTORIZATION="${CHECK_VECTORIZATION}" "$HELPERS" "$SHARED_LIBS_ARG" "$MINIFIER_ARG" "$OPERATIONS_ARG" "$BUILD_TYPE" "$PACKAGING_ARG" "$EXPERIMENTAL_ARG" "$TESTS_ARG" "$CUDA_COMPUTE" -DOPENBLAS_PATH="$OPENBLAS_PATH" -DDEV=FALSE -DCMAKE_NEED_RESPONSE=YES -DMKL_MULTI_THREADED=TRUE ../.. if [ "$PARALLEL" == "true" ]; then MAKE_ARGUMENTS="$MAKE_ARGUMENTS -j $MAKEJ" @@ -658,3 +626,4 @@ exec 3>&- else eval "$MAKE_COMMAND" "$MAKE_ARGUMENTS" && cd ../../.. fi + diff --git a/libnd4j/cmake/rpi.cmake b/libnd4j/cmake/rpi.cmake index 5329b2ce3..a7f7b0890 100644 --- a/libnd4j/cmake/rpi.cmake +++ b/libnd4j/cmake/rpi.cmake @@ -16,16 +16,43 @@ # # SPDX-License-Identifier: Apache-2.0 ################################################################################ -# CMake toolchain to build libnd4j for Raspberry PI. Sample usage: -# -# PI_BIN=/path/to/raspberrypi/tools/arm-bcm2708/arm-rpi-4.9.3-linux-gnueabihf/bin/arm-linux-gnueabihf \ -# cmake -DCMAKE_TOOLCHAIN_FILE=rpi.cmake -DCMAKE_INSTALL_PREFIX=.. SET(CMAKE_SYSTEM_NAME Linux) SET(CMAKE_SYSTEM_VERSION 1) set(CMAKE_C_COMPILER "$ENV{RPI_BIN}-gcc" CACHE STRING "" FORCE) set(CMAKE_CXX_COMPILER "$ENV{RPI_BIN}-g++" CACHE STRING "" FORCE) -SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -SET(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) \ No newline at end of file + +if (SD_CUDA) + if(${SD_ARCH} MATCHES "armv8") + set(CMAKE_SYSTEM_PROCESSOR aarch64) + else() + set(CMAKE_SYSTEM_PROCESSOR ${SD_ARCH}) + endif() + set(CUDA_TARGET_CPU_ARCH ${CMAKE_SYSTEM_PROCESSOR}) + set(CUDA_TARGET_OS_VARIANT "linux") + if (SD_CUDA) + set(ENV{CUDAHOSTCXX} "${CMAKE_CXX_COMPILER}") + endif() +endif() + +set(CMAKE_FIND_ROOT_PATH "") + +if(DEFINED ENV{SYSROOT}) + set(CMAKE_SYSROOT "$ENV{SYSROOT}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --sysroot=$ENV{SYSROOT}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --sysroot=$ENV{SYSROOT}") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --sysroot=$ENV{SYSROOT}") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} --sysroot=$ENV{SYSROOT}") + list(APPEND CMAKE_FIND_ROOT_PATH "$ENV{SYSROOT}") +endif() + +if(DEFINED ENV{CUDNN_ROOT_DIR}) + list(APPEND CMAKE_FIND_ROOT_PATH "$ENV{CUDNN_ROOT_DIR}") +endif() + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +#search only in target path +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) \ No newline at end of file diff --git a/libnd4j/pi_build.sh b/libnd4j/pi_build.sh index 8a536d155..fcd85ddab 100755 --- a/libnd4j/pi_build.sh +++ b/libnd4j/pi_build.sh @@ -24,12 +24,12 @@ function message { } if [ -z "${BUILD_USING_MAVEN}" ]; then export BUILD_USING_MAVEN=; fi if [ -z "${CURRENT_TARGET}" ]; then export CURRENT_TARGET=arm32; fi -if [ -z "${HAS_ARMCOMPUTE}" ]; then export ARMCOMPUTE_DEBUG=1; fi -if [ -z "${ARMCOMPUTE_DEBUG}" ]; then export HAS_ARMCOMPUTE=1; fi +if [ -z "${ARMCOMPUTE_DEBUG}" ]; then export ARMCOMPUTE_DEBUG=1; fi +if [ -z "${HAS_ARMCOMPUTE}" ]; then export HAS_ARMCOMPUTE=1; fi if [ -z "${ARMCOMPUTE_TAG}" ]; then export ARMCOMPUTE_TAG=v20.05; fi if [ -z "${LIBND4J_BUILD_MODE}" ]; then export LIBND4J_BUILD_MODE=Release; fi if [ -z "${ANDROID_VERSION}" ]; then export ANDROID_VERSION=21; fi -if [ -z "${HAS_ARMCOMPUTE}" ]; then export HAS_ARMCOMPUTE=1; fi +if [ -z "${CUDA_VER}" ]; then export CUDA_VER=10.2; fi OTHER_ARGS=() while [[ $# -gt 0 ]] @@ -56,16 +56,17 @@ done CC_URL32="https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf.tar.xz?revision=e09a1c45-0ed3-4a8e-b06b-db3978fd8d56&la=en&hash=93ED4444B8B3A812B893373B490B90BBB28FD2E3" CC_URL64="https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en&hash=167687FADA00B73D20EED2A67D0939A197504ACD" CC_ANDROID="https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip" -TARGET_ARRS=( arm32 arm64 android-arm android-arm64 ) -COMPILER_ARRS=( "${CC_URL32}" "${CC_URL64}" "${CC_ANDROID}" "${CC_ANDROID}" ) -COMPILER_DOWNLOAD_CMD_LIST=( download_extract_xz download_extract_xz download_extract_unzip download_extract_unzip ) -COMPILER_DESTDIR=( "arm32" "arm64" "android" "android" ) +CC_L4T64="https://developer.nvidia.com/embedded/dlc/l4t-gcc-7-3-1-toolchain-64-bit" +TARGET_ARRS=( arm32 arm64 android-arm android-arm64 android-x86 android-x86_64 jetson_arm64) +COMPILER_ARRS=( "${CC_URL32}" "${CC_URL64}" "${CC_ANDROID}" "${CC_ANDROID}" "${CC_ANDROID}" "${CC_ANDROID}" "${CC_L4T64}" ) +COMPILER_DOWNLOAD_CMD_LIST=( download_extract_xz download_extract_xz download_extract_unzip download_extract_unzip download_extract_unzip download_extract_unzip download_extract_xz) +COMPILER_DESTDIR=( "arm32" "arm64" "android" "android" "android" "android" "l4t" ) -OPENBLAS_TARGETS=( ARMV7 ARMV8 ARMV7 ARMV8) -ARMCOMPUTE_TARGETS=( armv7a arm64-v8a armv7a arm64-v8a) -OS_LIST=( linux linux android android) -LIBND4J_PLATFORM_EXT_LIST=( armhf arm64 arm arm64 ) -PREFIXES=( arm-linux-gnueabihf aarch64-linux-gnu arm-linux-androideabi aarch64-linux-android ) +OPENBLAS_TARGETS=( ARMV7 ARMV8 ARMV7 ARMV8 ATOM ATOM ARMV8) +ARMCOMPUTE_TARGETS=( armv7a arm64-v8a armv7a arm64-v8a None None None) +OS_LIST=( linux linux android android android android linux) +LIBND4J_PLATFORM_EXT_LIST=( armhf arm64 arm arm64 x86 x86_64 arm64) +PREFIXES=( arm-linux-gnueabihf aarch64-linux-gnu arm-linux-androideabi aarch64-linux-android i686-linux-android x86_64-linux-android aarch64-linux-gnu) TARGET_INDEX=-1 for i in "${!TARGET_ARRS[@]}"; do @@ -144,23 +145,27 @@ function rename_top_folder { } function download_extract_base { - #$1 is url #2 is dir $3 is extract argument - if [ ! -f ${3}_file ]; then - message "download" - wget --quiet --show-progress -O ${3}_file ${2} + #$1 is extract arg, $2 is url, $3 is dir + xtract_arg=${1} + down_url=${2} + down_dir=$(dirname "${3}/__") + down_file="${down_dir}_file" + if [ ! -f ${down_file} ]; then + message "download ${down_url}" + wget --quiet --show-progress -O ${down_file} ${down_url} fi message "extract $@" #extract - mkdir -p ${3} - if [ ${1} = "-unzip" ]; then - command="unzip -qq ${3}_file -d ${3} " + mkdir -p ${down_dir} + if [ ${xtract_arg} = "-unzip" ]; then + command="unzip -qq ${down_file} -d ${down_dir} " else - command="tar ${1} ${3}_file --directory=${3} " + command="tar ${xtract_arg} ${down_file} --directory=${down_dir} " fi message $command $command - check_requirements "${3}" + check_requirements "${down_dir}" } function download_extract { @@ -199,6 +204,36 @@ function fix_pi_linker { chmod +x ${1}/ld } +function cuda_cross_setup { + # $1 is local cuda toolkit version + # $2 the folder where cross cuda toolkit will be + loc_VER=${1-10.2} + loc_DIR=${2-/tmp} + CUDA_DIR=${CUDA_DIR-/usr/local/cuda-${loc_VER}} + CUDA_TARGET_STUBS=${loc_DIR}/target_stub${loc_VER} + CUDA_TARGET_STUB_URL=https://github.com/quickwritereader/temp_jars/releases/download/1.2/aarch64_linux_cuda_${loc_VER}_cudnn8.tar.gz + download_extract ${CUDA_TARGET_STUB_URL} ${CUDA_TARGET_STUBS} + message "lets setup cuda toolkit by combining local cuda-${loc_VER} and target ${CUDA_TARGET_STUBS}" + message "cuda cross folder: ${loc_DIR}" + check_requirements ${CUDA_DIR} ${CUDA_TARGET_STUBS}/aarch64-linux/ + if [ ! -d ${loc_DIR}/cuda/bin ];then + mkdir -p ${loc_DIR}/cuda/bin + cd ${CUDA_DIR}/bin/ + # we are obliged to symlink inner files and folders to avoid errors happening + # when relative folders are used from symlinks + for i in $(find . -maxdepth 1) + do + ln -s ${CUDA_DIR}/bin/${i} ${loc_DIR}/cuda/bin/${i} + done + cd - + ln -s ${CUDA_DIR}/nvvm ${loc_DIR}/cuda/nvvm + ln -s ${CUDA_TARGET_STUBS} ${loc_DIR}/cuda/targets + fi + export CUDACXX=${loc_DIR}/cuda/bin/nvcc + export CUDNN_ROOT_DIR=${CUDA_TARGET_STUBS}/aarch64-linux + export CUDA_TOOLKIT_ROOT=${loc_DIR}/cuda +} + if [ ! -d ${CROSS_COMPILER_DIR}/folder ]; then #out file message "download CROSS_COMPILER" @@ -265,6 +300,15 @@ export OPENBLAS_PATH=${THIRD_PARTY} fi # end if [ -z "${BUILD_USING_MAVEN}"];then + +XTRA_ARGS="" +XTRA_MVN_ARGS="" + +if [ "${ARMCOMPUTE_TARGET}" != "None" ];then +message "~~~ARMCOMPUTE~~~" +XTRA_ARGS=" -h armcompute " +XTRA_MVN_ARGS=" -Dlibnd4j.helper=armcompute " + if [ ! -d ${SCONS_LOCAL_DIR} ]; then #out file message "download Scons local" @@ -290,11 +334,25 @@ check_requirements "${ARMCOMPUTE_DIR}/build/libarm_compute-static.a" "${ARMCOMPU export ARMCOMPUTE_ROOT="${ARMCOMPUTE_DIR}" +fi #if armcompute + + if [ "${TARGET_OS}" = "android" ];then export ANDROID_NDK=${CROSS_COMPILER_DIR} + XTRA_MVN_ARGS="${XTRA_MVN_ARGS} -pl \":libnd4j,:nd4j-native\" " else - export RPI_BIN=${CROSS_COMPILER_DIR}/bin/${PREFIX} - export JAVA_LIBRARY_PATH=${CROSS_COMPILER_DIR}/${PREFIX}/lib + if [ "${CURRENT_TARGET}" == "jetson_arm64" ];then + message "jetson cuda build " + cuda_cross_setup ${CUDA_VER} + XTRA_ARGS="${XTRA_ARGS} -c cuda -h cudnn " + XTRA_MVN_ARGS="${XTRA_MVN_ARGS} -Dcuda.version=${CUDA_VER} -Dlibnd4j.cuda=${CUDA_VER} -Dlibnd4j.chip=cuda " + XTRA_MVN_ARGS="${XTRA_MVN_ARGS} -Dlibnd4j.helper=cudnn " + export SYSROOT=${CROSS_COMPILER_DIR}/${PREFIX}/libc + else + XTRA_MVN_ARGS="${XTRA_MVN_ARGS} -pl \":libnd4j,:nd4j-native\" " + fi + export RPI_BIN=${CROSS_COMPILER_DIR}/bin/${PREFIX} + export JAVA_LIBRARY_PATH=${CROSS_COMPILER_DIR}/${PREFIX}/lib fix_pi_linker ${BINUTILS_BIN} fi @@ -310,19 +368,17 @@ fi if [ -z "${BUILD_USING_MAVEN}" ] ;then message "lets build just library" -DHELPER=" -h armcompute " -bash ./buildnativeoperations.sh -o ${LIBND4J_PLATFORM} -t ${DHELPER} -j $(nproc) +bash ./buildnativeoperations.sh -o ${LIBND4J_PLATFORM} -t -j $(nproc) ${XTRA_ARGS} else message "cd $BASE_DIR/.. " cd $BASE_DIR/.. message "lets build jars" -export DHELPER=" -Dlibnd4j.helper=armcompute " -if [ "${DEPLOY}" ]; then - echo "Deploying to maven" - mvn -P"${PUBLISH_TO}" deploy --batch-mode -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER} -pl ":libnd4j,:nd4j-native" --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true - else - echo "Installing to local repo" - mvn install -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER} -pl ":libnd4j" --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true +if [ "${DEPLOY-}" != "" ]; then + message "Deploying to maven" + mvn -P"${PUBLISH_TO}" deploy --batch-mode -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} ${XTRA_MVN_ARGS} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true +else + message "Installing to local repo" + mvn install -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} ${XTRA_MVN_ARGS} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true fi fi From f7d74c98ebaab28005173bd5dd866c7845167802 Mon Sep 17 00:00:00 2001 From: AbdelRauf Date: Tue, 9 Mar 2021 02:51:29 +0100 Subject: [PATCH 2/2] cross_build: set jetson compute 5.3. small modifications Signed-off-by: AbdelRauf --- libnd4j/cmake/rpi.cmake | 4 ++-- libnd4j/pi_build.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libnd4j/cmake/rpi.cmake b/libnd4j/cmake/rpi.cmake index a7f7b0890..6e4438bff 100644 --- a/libnd4j/cmake/rpi.cmake +++ b/libnd4j/cmake/rpi.cmake @@ -20,8 +20,8 @@ SET(CMAKE_SYSTEM_NAME Linux) SET(CMAKE_SYSTEM_VERSION 1) -set(CMAKE_C_COMPILER "$ENV{RPI_BIN}-gcc" CACHE STRING "" FORCE) -set(CMAKE_CXX_COMPILER "$ENV{RPI_BIN}-g++" CACHE STRING "" FORCE) +set(CMAKE_C_COMPILER "$ENV{RPI_BIN}-gcc" ) +set(CMAKE_CXX_COMPILER "$ENV{RPI_BIN}-g++" ) if (SD_CUDA) if(${SD_ARCH} MATCHES "armv8") diff --git a/libnd4j/pi_build.sh b/libnd4j/pi_build.sh index fcd85ddab..178b4810a 100755 --- a/libnd4j/pi_build.sh +++ b/libnd4j/pi_build.sh @@ -345,8 +345,8 @@ else message "jetson cuda build " cuda_cross_setup ${CUDA_VER} XTRA_ARGS="${XTRA_ARGS} -c cuda -h cudnn " - XTRA_MVN_ARGS="${XTRA_MVN_ARGS} -Dcuda.version=${CUDA_VER} -Dlibnd4j.cuda=${CUDA_VER} -Dlibnd4j.chip=cuda " - XTRA_MVN_ARGS="${XTRA_MVN_ARGS} -Dlibnd4j.helper=cudnn " + XTRA_MVN_ARGS="${XTRA_MVN_ARGS} -Djavacpp.version=1.5.3 -Dcuda.version=${CUDA_VER} -Dlibnd4j.cuda=${CUDA_VER} -Dlibnd4j.chip=cuda -Dlibnd4j.compute=5.3 " + XTRA_MVN_ARGS="${XTRA_MVN_ARGS} -Dlibnd4j.helper=cudnn " export SYSROOT=${CROSS_COMPILER_DIR}/${PREFIX}/libc else XTRA_MVN_ARGS="${XTRA_MVN_ARGS} -pl \":libnd4j,:nd4j-native\" "