cross_build: added android {x86, x86_64} and jetson {cross arm64 + host cuda + target cuda aarch64}

jetson note: host cuda toolkit should be installed to setup cross environment

Signed-off-by: AbdelRauf <rauf@konduit.ai>

libnd4j/buildnativeoperations.sh

@@ -82,14 +82,11 @@ OPERATIONS=
 CLEAN="false"
 MINIFIER="false"
 TESTS="false"
-VERBOSE="false"
+VERBOSE="true"
 VERBOSE_ARG="VERBOSE=1"
 HELPER=
 CHECK_VECTORIZATION="OFF"
-SYS_ROOT=
-EXTRA_LINK_FLAGS=
 NAME=
-EXTRA_CUDA_FLAGS=
 while [[ $# -gt 0 ]]
 do
 key="$1"
@@ -402,11 +399,6 @@ if [ -z "$BUILD" ]; then
 
 fi
 
-if [ -z "$SYS_ROOT" ]; then
- export SYS_ROOT=""
-fi
-
-
 if [ -z "$CHIP" ]; then
  CHIP="cpu"
 fi
@@ -518,12 +510,6 @@ if [ "$TESTS" == "true" ]; then
 fi
 
 
-if [ "$SYS_ROOT" != "" ]; then
-    EXTRA_SYSROOT="-DCMAKE_SYSROOT=$SYS_ROOT"
-  else
-      EXTRA_SYSROOT=""
-fi
-
 ARCH_ARG="-DSD_ARCH=$ARCH -DSD_EXTENSION=$CHIP_EXTENSION"
 
 CUDA_COMPUTE="-DCOMPUTE=\"$COMPUTE\""
@@ -531,16 +517,8 @@ CUDA_COMPUTE="-DCOMPUTE=\"$COMPUTE\""
 if [ "$CHIP" == "cuda" ] && [ -n "$CHIP_VERSION" ]; then
     case $OS in
         linux*)
-        export CUDA_PATH="/usr/local/cuda-$CHIP_VERSION/"
+        if [ "${CUDA_PATH-}" == "" ]; then
-        # Cross compilation for jetson nano
+            export CUDA_PATH="/usr/local/cuda-$CHIP_VERSION/"
-        if [ "$ARCH" != "x86-64" ]; then
-                 if [ "$ARCH" == "armv8-a" ]; then
-                       export EXTRA_CUDA_FLAGS="-DCUDA_TARGET_CPU_ARCH=AARCH64"
-                     else
-                        export EXTRA_CUDA_FLAGS="-DCUDA_TARGET_CPU_ARCH=ARM"
-                 fi
-          else
-              export EXTRA_CUDA_FLAGS=""
         fi
         ;;
         macosx*)
@@ -609,13 +587,6 @@ else
   IFS=' '
 fi
 
-LINKER_FLAGS=""
-if [ "$EXTRA_LINK_FLAGS" != "" ]; then
-   LINKER_FLAGS="-DCMAKE_CXX_LINK_FLAGS=$EXTRA_LINK_FLAGS -DCMAKE_EXE_LINKER_FLAGS=$EXTRA_LINK_FLAGS -DCMAKE_CUDA_FLAGS=$EXTRA_LINK_FLAGS"
-fi
-
-
-
 echo PACKAGING  = "${PACKAGING}"
 echo BUILD  = "${BUILD}"
 echo CHIP     = "${CHIP}"
@@ -632,12 +603,9 @@ echo NAME = "${NAME_ARG}"
 echo OPENBLAS_PATH = "$OPENBLAS_PATH"
 echo CHECK_VECTORIZATION = "$CHECK_VECTORIZATION"
 echo HELPERS = "$HELPERS"
-echo EXTRA_LINK_FLAGS = "$EXTRA_LINK_FLAGS"
-echo EXTRA_CUDA_FLAGS = "$EXTRA_CUDA_FLAGS"
-echo EXTRA_SYSROOT = "$EXTRA_SYSROOT"
 mkbuilddir
 pwd
-eval "$CMAKE_COMMAND" "$EXTRA_SYSROOT" "$LINKER_FLAGS" "$EXTRA_CUDA_FLAGS" "$BLAS_ARG" "$ARCH_ARG" "$NAME_ARG" -DSD_CHECK_VECTORIZATION="${CHECK_VECTORIZATION}"  "$HELPERS" "$SHARED_LIBS_ARG" "$MINIFIER_ARG" "$OPERATIONS_ARG" "$BUILD_TYPE" "$PACKAGING_ARG" "$EXPERIMENTAL_ARG" "$TESTS_ARG" "$CUDA_COMPUTE" -DOPENBLAS_PATH="$OPENBLAS_PATH" -DDEV=FALSE -DCMAKE_NEED_RESPONSE=YES -DMKL_MULTI_THREADED=TRUE ../..
+eval "$CMAKE_COMMAND"  "$BLAS_ARG" "$ARCH_ARG" "$NAME_ARG" -DSD_CHECK_VECTORIZATION="${CHECK_VECTORIZATION}"  "$HELPERS" "$SHARED_LIBS_ARG" "$MINIFIER_ARG" "$OPERATIONS_ARG" "$BUILD_TYPE" "$PACKAGING_ARG" "$EXPERIMENTAL_ARG" "$TESTS_ARG" "$CUDA_COMPUTE" -DOPENBLAS_PATH="$OPENBLAS_PATH" -DDEV=FALSE -DCMAKE_NEED_RESPONSE=YES -DMKL_MULTI_THREADED=TRUE ../..
 
 if [ "$PARALLEL" == "true" ]; then
     MAKE_ARGUMENTS="$MAKE_ARGUMENTS -j $MAKEJ"
@@ -658,3 +626,4 @@ exec 3>&-
 else
 eval "$MAKE_COMMAND" "$MAKE_ARGUMENTS"  && cd ../../..
 fi
+

libnd4j/cmake/rpi.cmake

@@ -16,16 +16,43 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 ################################################################################
-# CMake toolchain to build libnd4j for Raspberry PI. Sample usage:
-#
-# PI_BIN=/path/to/raspberrypi/tools/arm-bcm2708/arm-rpi-4.9.3-linux-gnueabihf/bin/arm-linux-gnueabihf \
-# cmake -DCMAKE_TOOLCHAIN_FILE=rpi.cmake -DCMAKE_INSTALL_PREFIX=..
 
 SET(CMAKE_SYSTEM_NAME Linux)
 SET(CMAKE_SYSTEM_VERSION 1)
 
 set(CMAKE_C_COMPILER   "$ENV{RPI_BIN}-gcc" CACHE STRING "" FORCE)
 set(CMAKE_CXX_COMPILER "$ENV{RPI_BIN}-g++" CACHE STRING "" FORCE)
-SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+
-SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+if (SD_CUDA)
-SET(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+    if(${SD_ARCH} MATCHES "armv8")
+        set(CMAKE_SYSTEM_PROCESSOR aarch64)
+    else()
+        set(CMAKE_SYSTEM_PROCESSOR ${SD_ARCH})
+    endif()
+    set(CUDA_TARGET_CPU_ARCH ${CMAKE_SYSTEM_PROCESSOR})
+    set(CUDA_TARGET_OS_VARIANT "linux")
+    if (SD_CUDA)
+        set(ENV{CUDAHOSTCXX} "${CMAKE_CXX_COMPILER}")
+    endif()
+endif()
+
+set(CMAKE_FIND_ROOT_PATH "")
+
+if(DEFINED ENV{SYSROOT})
+    set(CMAKE_SYSROOT "$ENV{SYSROOT}")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --sysroot=$ENV{SYSROOT}")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --sysroot=$ENV{SYSROOT}")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --sysroot=$ENV{SYSROOT}")
+    set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} --sysroot=$ENV{SYSROOT}")
+    list(APPEND CMAKE_FIND_ROOT_PATH "$ENV{SYSROOT}")
+endif()
+
+if(DEFINED ENV{CUDNN_ROOT_DIR})
+    list(APPEND CMAKE_FIND_ROOT_PATH "$ENV{CUDNN_ROOT_DIR}")
+endif()
+
+set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+#search only in target path
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

libnd4j/pi_build.sh

@@ -24,12 +24,12 @@ function message {
 }
 if [ -z "${BUILD_USING_MAVEN}" ]; then export BUILD_USING_MAVEN=; fi
 if [ -z "${CURRENT_TARGET}" ]; then export  CURRENT_TARGET=arm32; fi
-if [ -z "${HAS_ARMCOMPUTE}" ]; then export  ARMCOMPUTE_DEBUG=1; fi
+if [ -z "${ARMCOMPUTE_DEBUG}" ]; then export  ARMCOMPUTE_DEBUG=1; fi
-if [ -z "${ARMCOMPUTE_DEBUG}" ]; then export  HAS_ARMCOMPUTE=1; fi
+if [ -z "${HAS_ARMCOMPUTE}" ]; then export  HAS_ARMCOMPUTE=1; fi
 if [ -z "${ARMCOMPUTE_TAG}" ]; then export  ARMCOMPUTE_TAG=v20.05; fi
 if [ -z "${LIBND4J_BUILD_MODE}" ]; then export  LIBND4J_BUILD_MODE=Release; fi
 if [ -z "${ANDROID_VERSION}" ]; then export  ANDROID_VERSION=21; fi
-if [ -z "${HAS_ARMCOMPUTE}" ]; then export  HAS_ARMCOMPUTE=1; fi
+if [ -z "${CUDA_VER}" ]; then export  CUDA_VER=10.2; fi
 
 OTHER_ARGS=()
 while [[ $# -gt 0 ]]
@@ -56,16 +56,17 @@ done
 CC_URL32="https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf.tar.xz?revision=e09a1c45-0ed3-4a8e-b06b-db3978fd8d56&la=en&hash=93ED4444B8B3A812B893373B490B90BBB28FD2E3"
 CC_URL64="https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en&hash=167687FADA00B73D20EED2A67D0939A197504ACD"
 CC_ANDROID="https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip"
-TARGET_ARRS=( arm32 arm64 android-arm android-arm64 )
+CC_L4T64="https://developer.nvidia.com/embedded/dlc/l4t-gcc-7-3-1-toolchain-64-bit"
-COMPILER_ARRS=( "${CC_URL32}" "${CC_URL64}" "${CC_ANDROID}" "${CC_ANDROID}" )
+TARGET_ARRS=( arm32 arm64 android-arm android-arm64 android-x86 android-x86_64 jetson_arm64)
-COMPILER_DOWNLOAD_CMD_LIST=( download_extract_xz download_extract_xz download_extract_unzip download_extract_unzip )
+COMPILER_ARRS=( "${CC_URL32}" "${CC_URL64}" "${CC_ANDROID}" "${CC_ANDROID}" "${CC_ANDROID}" "${CC_ANDROID}" "${CC_L4T64}" )
-COMPILER_DESTDIR=( "arm32" "arm64" "android" "android" )
+COMPILER_DOWNLOAD_CMD_LIST=( download_extract_xz download_extract_xz download_extract_unzip download_extract_unzip download_extract_unzip download_extract_unzip download_extract_xz)
+COMPILER_DESTDIR=( "arm32" "arm64" "android" "android" "android" "android" "l4t" )
 
-OPENBLAS_TARGETS=( ARMV7 ARMV8 ARMV7 ARMV8)
+OPENBLAS_TARGETS=( ARMV7 ARMV8 ARMV7 ARMV8 ATOM ATOM ARMV8)
-ARMCOMPUTE_TARGETS=( armv7a arm64-v8a armv7a arm64-v8a)
+ARMCOMPUTE_TARGETS=( armv7a arm64-v8a armv7a arm64-v8a None None None)
-OS_LIST=( linux linux android android)
+OS_LIST=( linux linux android android android android linux)
-LIBND4J_PLATFORM_EXT_LIST=( armhf arm64 arm arm64 )
+LIBND4J_PLATFORM_EXT_LIST=( armhf arm64 arm arm64 x86 x86_64 arm64)
-PREFIXES=( arm-linux-gnueabihf aarch64-linux-gnu arm-linux-androideabi aarch64-linux-android )
+PREFIXES=( arm-linux-gnueabihf aarch64-linux-gnu arm-linux-androideabi aarch64-linux-android  i686-linux-android x86_64-linux-android aarch64-linux-gnu)
 TARGET_INDEX=-1
 
 for i in "${!TARGET_ARRS[@]}"; do
@@ -144,23 +145,27 @@ function rename_top_folder {
 }
 
 function download_extract_base {
-	#$1 is url #2 is dir $3 is extract argument
+	#$1 is extract arg, $2 is url, $3 is dir
-	if [ ! -f ${3}_file ]; then
+	xtract_arg=${1}
-		message "download"
+	down_url=${2}
-		wget --quiet --show-progress -O ${3}_file ${2}
+	down_dir=$(dirname "${3}/__")
+	down_file="${down_dir}_file"
+	if [ ! -f ${down_file} ]; then
+		message "download ${down_url}"
+		wget --quiet --show-progress -O ${down_file} ${down_url}
 	fi
  
 	message "extract $@"
     #extract
-	mkdir -p ${3} 
+	mkdir -p ${down_dir} 
-	if [ ${1} = "-unzip" ]; then
+	if [ ${xtract_arg} = "-unzip" ]; then
-		command="unzip -qq ${3}_file -d ${3} "
+		command="unzip -qq ${down_file} -d ${down_dir} "
 	else
-		command="tar ${1}  ${3}_file --directory=${3} "
+		command="tar ${xtract_arg} ${down_file} --directory=${down_dir} "
 	fi
 	message $command
 	$command
-	check_requirements "${3}"
+	check_requirements "${down_dir}"
 }
 
 function download_extract {
@@ -199,6 +204,36 @@ function fix_pi_linker {
   chmod +x ${1}/ld 
 }
 
+function cuda_cross_setup {
+		# $1 is local  cuda toolkit version
+		# $2 the folder where cross cuda toolkit will be
+		loc_VER=${1-10.2}
+		loc_DIR=${2-/tmp}
+		CUDA_DIR=${CUDA_DIR-/usr/local/cuda-${loc_VER}}
+		CUDA_TARGET_STUBS=${loc_DIR}/target_stub${loc_VER}
+		CUDA_TARGET_STUB_URL=https://github.com/quickwritereader/temp_jars/releases/download/1.2/aarch64_linux_cuda_${loc_VER}_cudnn8.tar.gz
+		download_extract ${CUDA_TARGET_STUB_URL} ${CUDA_TARGET_STUBS}
+		message "lets setup cuda toolkit by combining local cuda-${loc_VER} and target ${CUDA_TARGET_STUBS}"
+		message "cuda cross folder: ${loc_DIR}"
+		check_requirements ${CUDA_DIR} ${CUDA_TARGET_STUBS}/aarch64-linux/
+		if [ ! -d ${loc_DIR}/cuda/bin ];then
+			mkdir -p ${loc_DIR}/cuda/bin
+			cd ${CUDA_DIR}/bin/
+			# we are obliged to symlink inner files and folders to avoid errors happening 
+			# when  relative folders are used from symlinks 
+			for i in $(find . -maxdepth 1)
+			do 
+				ln -s ${CUDA_DIR}/bin/${i} ${loc_DIR}/cuda/bin/${i}
+			done
+			cd -
+			ln -s ${CUDA_DIR}/nvvm ${loc_DIR}/cuda/nvvm
+			ln -s ${CUDA_TARGET_STUBS} ${loc_DIR}/cuda/targets
+		fi
+		export CUDACXX=${loc_DIR}/cuda/bin/nvcc
+		export CUDNN_ROOT_DIR=${CUDA_TARGET_STUBS}/aarch64-linux
+		export CUDA_TOOLKIT_ROOT=${loc_DIR}/cuda
+}
+
 if [ ! -d ${CROSS_COMPILER_DIR}/folder ]; then
 	#out file
 	message "download CROSS_COMPILER"
@@ -265,6 +300,15 @@ export OPENBLAS_PATH=${THIRD_PARTY}
 
 fi # end if [ -z "${BUILD_USING_MAVEN}"];then
 
+
+XTRA_ARGS=""
+XTRA_MVN_ARGS=""
+
+if [ "${ARMCOMPUTE_TARGET}" != "None" ];then
+message "~~~ARMCOMPUTE~~~"
+XTRA_ARGS=" -h armcompute "
+XTRA_MVN_ARGS=" -Dlibnd4j.helper=armcompute "
+
 if [ ! -d ${SCONS_LOCAL_DIR} ]; then
 	#out file
 	message "download Scons local"
@@ -290,11 +334,25 @@ check_requirements "${ARMCOMPUTE_DIR}/build/libarm_compute-static.a" "${ARMCOMPU
 
 export ARMCOMPUTE_ROOT="${ARMCOMPUTE_DIR}"
 
+fi #if armcompute
+
+
 if [ "${TARGET_OS}" = "android" ];then
 	export ANDROID_NDK=${CROSS_COMPILER_DIR}
+	XTRA_MVN_ARGS="${XTRA_MVN_ARGS} -pl \":libnd4j,:nd4j-native\" "
 else
-    export RPI_BIN=${CROSS_COMPILER_DIR}/bin/${PREFIX}
+	 if [ "${CURRENT_TARGET}" == "jetson_arm64" ];then
-	  export JAVA_LIBRARY_PATH=${CROSS_COMPILER_DIR}/${PREFIX}/lib
+	 	message  "jetson cuda build "
+		cuda_cross_setup ${CUDA_VER}
+		XTRA_ARGS="${XTRA_ARGS} -c cuda  -h cudnn  "
+		XTRA_MVN_ARGS="${XTRA_MVN_ARGS} -Dcuda.version=${CUDA_VER} -Dlibnd4j.cuda=${CUDA_VER} -Dlibnd4j.chip=cuda  "
+		XTRA_MVN_ARGS="${XTRA_MVN_ARGS}  -Dlibnd4j.helper=cudnn "
+		export SYSROOT=${CROSS_COMPILER_DIR}/${PREFIX}/libc
+	else
+		XTRA_MVN_ARGS="${XTRA_MVN_ARGS} -pl \":libnd4j,:nd4j-native\" "
+	fi
+	export RPI_BIN=${CROSS_COMPILER_DIR}/bin/${PREFIX}
+	export JAVA_LIBRARY_PATH=${CROSS_COMPILER_DIR}/${PREFIX}/lib
 	fix_pi_linker ${BINUTILS_BIN}
 fi
 
@@ -310,19 +368,17 @@ fi
 
 if [ -z "${BUILD_USING_MAVEN}" ] ;then
 message "lets build just library"
-DHELPER="  -h armcompute "
+bash ./buildnativeoperations.sh -o ${LIBND4J_PLATFORM} -t -j $(nproc) ${XTRA_ARGS} 
-bash ./buildnativeoperations.sh -o ${LIBND4J_PLATFORM} -t ${DHELPER} -j $(nproc)
 else
 message "cd $BASE_DIR/.. "
 cd $BASE_DIR/..
 message "lets build jars"
-export DHELPER=" -Dlibnd4j.helper=armcompute "
+if [ "${DEPLOY-}" != "" ]; then
-if [ "${DEPLOY}" ]; then
+  message "Deploying to maven"
-  echo "Deploying to maven"
+  mvn  -P"${PUBLISH_TO}" deploy  --batch-mode  -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} ${XTRA_MVN_ARGS} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH}  --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true
-  mvn  -P"${PUBLISH_TO}" deploy  --batch-mode  -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER}  -pl ":libnd4j,:nd4j-native" --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true
+else
- else
+  message "Installing to local repo"
-     echo "Installing to local repo"
+  mvn  install  -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM}  ${XTRA_MVN_ARGS}  -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH}  --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true
-     mvn  install  -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER}  -pl ":libnd4j" --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true
 fi
 
 fi