Brutex/cavis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #9206 from eclipse/ag_libnd4j_build_changes

Browse Source 
Libnd4j build changes
This commit is contained in:
Adam Gibson 2021-03-06 09:20:35 +09:00 committed by GitHub
commit 5541edadc2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 453 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
libnd4j/CMakeLists.txt
View File

@ -1,8 +1,11 @@
cmake_minimum_required(VERSION 3.15) cmake_minimum_required(VERSION 3.15)
project(libnd4j) project(libnd4j)
set(CMAKE_VERBOSE_MAKEFILE OFF) set(CMAKE_VERBOSE_MAKEFILE ON)
set (CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
message("CMAKE MODULE PATH ${CMAKE_MODULE_PATH}")
set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
#ensure we create lib files #ensure we create lib files
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS OFF) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS OFF)
@ -18,8 +21,99 @@ set(FLATBUFFERS_BUILD_FLATC "OFF" CACHE STRING "Hack to disable flatc build" FOR
set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD 11)
#///////////////////////////////////////////////////////////////////////////////
# genCompilation: Generates cpp, cu files
# INPUT:
# $FILE_ITEM template-configuration that utilizes libnd4j type, macros helpers
# defined inside { include/types/types.h, include/system/type_boilerplate.h}
# OUTPUT:
# $CUSTOMOPS_GENERIC_SOURCES generated files will be added into this List
#////////////////////////////////////////////////////////////////////////////////
# A simple template-configuration file example:
# // hints and defines what types will be generated
# #cmakedefine LIBND4J_TYPE_GEN
# #cmakedefine FLOAT_TYPE_GEN
# // below if defines blocks are needed for correctly handling multiple types
# #if defined(LIBND4J_TYPE_GEN)
# BUILD_DOUBLE_TEMPLATE(template void someFunc, (arg_list,..),
# LIBND4J_TYPES_@FL_TYPE_INDEX@, INDEXING_TYPES);
# #endif
# #if defined(FLOAT_TYPE_GEN)
# BUILD_SINGLE_TEMPLATE(template class SomeClass,, FLOAT_TYPES_@FL_TYPE_INDEX@);
# #endif
#////////////////////////////////////////////////////////////////////////////////
set_property(GLOBAL PROPERTY JOB_POOLS one_jobs=1 two_jobs=2)
function(genCompilation FILE_ITEM)
get_filename_component(FILE_ITEM_WE ${FL_ITEM} NAME_WE)
set(EXTENSION "cpp")
if(FL_ITEM MATCHES "cu.in$")
set(EXTENSION "cu")
endif()
file(READ ${FL_ITEM} CONTENT_FL)
#check content for types
#set all to false
set (FLOAT_TYPE_GEN 0)
set (INT_TYPE_GEN 0)
set (LIBND4J_TYPE_GEN 0)
set (PAIRWISE_TYPE_GEN 0)
set (RANGE_STOP -1)
string(REGEX MATCHALL "#cmakedefine[ \t]+[^_]+_TYPE_GEN" TYPE_MATCHES ${CONTENT_FL})
foreach(TYPEX ${TYPE_MATCHES})
set(STOP -1)
if(TYPEX MATCHES "INT_TYPE_GEN$")
set (INT_TYPE_GEN 1)
set(STOP 7)
endif()
if(TYPEX MATCHES "LIBND4J_TYPE_GEN$")
set (LIBND4J_TYPE_GEN 1)
set(STOP 9)
endif()
if(TYPEX MATCHES "FLOAT_TYPE_GEN$")
set (FLOAT_TYPE_GEN 1)
set(STOP 3)
endif()
if(TYPEX MATCHES "PAIRWISE_TYPE_GEN$")
set (PAIRWISE_TYPE_GEN 1)
set(STOP 12)
endif()
if(STOP GREATER RANGE_STOP)
set(RANGE_STOP ${STOP})
endif()
endforeach()
if(RANGE_STOP GREATER -1)
foreach(FL_TYPE_INDEX RANGE 0 ${RANGE_STOP})
# set OFF if the index is above
if(FL_TYPE_INDEX GREATER 3)
set (FLOAT_TYPE_GEN 0)
endif()
if(FL_TYPE_INDEX GREATER 7)
set (INT_TYPE_GEN 0)
endif()
if(FL_TYPE_INDEX GREATER 9)
set (LIBND4J_TYPE_GEN 0)
endif()
set(GENERATED_SOURCE "${CMAKE_BINARY_DIR}/compilation_units/${FILE_ITEM_WE}_${FL_TYPE_INDEX}.${EXTENSION}")
configure_file( "${FL_ITEM}" "${GENERATED_SOURCE}" @ONLY)
LIST(APPEND CUSTOMOPS_GENERIC_SOURCES ${GENERATED_SOURCE} )
endforeach()
endif()
set(CUSTOMOPS_GENERIC_SOURCES ${CUSTOMOPS_GENERIC_SOURCES} PARENT_SCOPE)
endfunction()
include(GenCompilation)
if (SD_CUDA) if (SD_CUDA)
enable_language(CUDA) enable_language(CUDA)
@ -42,6 +136,7 @@ endif()
# -fsanitize=address # -fsanitize=address
# -fsanitize=leak # -fsanitize=leak
if (SD_ANDROID_BUILD) if (SD_ANDROID_BUILD)
set_property(GLOBAL PROPERTY JOB_POOLS one_job=1 two_jobs=2)
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -fPIC -Wno-braced-scalar-init -Wno-delete-non-virtual-dtor -Wno-unused-command-line-argument -Wno-dangling-else -D_RELEASE=true") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -fPIC -Wno-braced-scalar-init -Wno-delete-non-virtual-dtor -Wno-unused-command-line-argument -Wno-dangling-else -D_RELEASE=true")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g -fPIC -Wno-braced-scalar-init -Wno-delete-non-virtual-dtor -Wno-unused-command-line-argument -Wno-dangling-else") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g -fPIC -Wno-braced-scalar-init -Wno-delete-non-virtual-dtor -Wno-unused-command-line-argument -Wno-dangling-else")
elseif (APPLE) elseif (APPLE)
@ -315,13 +410,13 @@ elseif(DISTRIBUTION STREQUAL "CentOS")
else() else()
set(CPACK_RPM_PACKAGE_ARCHITECTURE "i686") set(CPACK_RPM_PACKAGE_ARCHITECTURE "i686")
endif() endif()
set(CPACK_PACKAGE_CONTACT "raver119") set(CPACK_PACKAGE_CONTACT "agibsonccc")
set(CPACK_RPM_PACKAGE_GROUP "Development/Tools") set(CPACK_RPM_PACKAGE_GROUP "Development/Tools")
set(CPACK_RPM_PACKAGE_LICENSE "Apache-2.0") set(CPACK_RPM_PACKAGE_LICENSE "Apache-2.0")
set(CPACK_RPM_PACKAGE_SUGGESTS "cuda") set(CPACK_RPM_PACKAGE_SUGGESTS "cuda")
# Build deps: atlas blas lapack cmake3 devtoolset-4-gcc devtoolset-4-gcc-c++ # Build deps: atlas blas lapack cmake3 devtoolset-4-gcc devtoolset-4-gcc-c++
set(CPACK_RPM_PACKAGE_REQUIRES "") set(CPACK_RPM_PACKAGE_REQUIRES "")
set(CPACK_RPM_PACKAGE_URL "https://github.com/deeplearning4j/libnd4j") set(CPACK_RPM_PACKAGE_URL "https://github.com/eclipse/deeplearning4j/libnd4j")
set(CPACK_GENERATOR "RPM") set(CPACK_GENERATOR "RPM")
set(CPACK_PACKAGE_FILE_NAME ${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}.fc${RELEASE}.${CPACK_RPM_PACKAGE_ARCHITECTURE}) set(CPACK_PACKAGE_FILE_NAME ${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}.fc${RELEASE}.${CPACK_RPM_PACKAGE_ARCHITECTURE})
set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/cmake/postinst") set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/cmake/postinst")

5
libnd4j/README.md
View File

@ -45,10 +45,9 @@ You can find the same information for the older Toolkit versions [in the CUDA ar
[Download the NDK](https://developer.android.com/ndk/downloads/), extract it somewhere, and execute the following commands, replacing `android-xxx` with either `android-arm` or `android-x86`: [Download the NDK](https://developer.android.com/ndk/downloads/), extract it somewhere, and execute the following commands, replacing `android-xxx` with either `android-arm` or `android-x86`:
```bash ```bash
git clone https://github.com/deeplearning4j/libnd4j git clone https://github.com/eclipse/deeplearning4j
git clone https://github.com/deeplearning4j/nd4j
export ANDROID_NDK=/path/to/android-ndk/ export ANDROID_NDK=/path/to/android-ndk/
cd libnd4j cd deeplearning4j/libnd4j
bash buildnativeoperations.sh -platform android-xxx bash buildnativeoperations.sh -platform android-xxx
cd ../nd4j cd ../nd4j
mvn clean install -Djavacpp.platform=android-xxx -DskipTests -pl '!:nd4j-cuda-9.0,!:nd4j-cuda-9.0-platform,!:nd4j-tests' mvn clean install -Djavacpp.platform=android-xxx -DskipTests -pl '!:nd4j-cuda-9.0,!:nd4j-cuda-9.0-platform,!:nd4j-tests'

25
libnd4j/blas/CMakeLists.txt
View File

@ -176,6 +176,8 @@ if(SD_CUDA)
set(EXPM " -D__ND4J_EXPERIMENTAL__=true") set(EXPM " -D__ND4J_EXPERIMENTAL__=true")
endif() endif()
# the only difference for debug mode here is host/device debug symbols # the only difference for debug mode here is host/device debug symbols
set(CMAKE_CUDA_FLAGS_DEBUG " -G -g") set(CMAKE_CUDA_FLAGS_DEBUG " -G -g")
@ -185,6 +187,18 @@ if(SD_CUDA)
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=-fPIC") set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=-fPIC")
endif() endif()
if(WIN32)
message("In windows, setting cublas library and cusolver library")
if(NOT DEFINED CUDA_cublas_LIBRARY)
set(CUDA_cublas_LIBRARY ${CUDA_HOME}/lib/x64/cublas.lib)
endif()
if(NOT DEFINED CUDA_cusolver_LIBRARY)
set(CUDA_cusolver_LIBRARY ${CUDA_HOME}/lib/x64/cusolver.lib)
endif()
endif()
string( TOLOWER "${COMPUTE}" COMPUTE_CMP ) string( TOLOWER "${COMPUTE}" COMPUTE_CMP )
if ("${COMPUTE_CMP}" STREQUAL "all") if ("${COMPUTE_CMP}" STREQUAL "all")
CUDA_SELECT_NVCC_ARCH_FLAGS(CUDA_ARCH_FLAGS "Common") CUDA_SELECT_NVCC_ARCH_FLAGS(CUDA_ARCH_FLAGS "Common")
@ -343,16 +357,27 @@ elseif(SD_CPU)
message("CPU BLAS") message("CPU BLAS")
add_definitions(-D__CPUBLAS__=true) add_definitions(-D__CPUBLAS__=true)
add_library(samediff_obj OBJECT ${LEGACY_SOURCES} add_library(samediff_obj OBJECT ${LEGACY_SOURCES}
${LOOPS_SOURCES} ${HELPERS_SOURCES} ${EXEC_SOURCES} ${ARRAY_SOURCES} ${TYPES_SOURCES} ${LOOPS_SOURCES} ${HELPERS_SOURCES} ${EXEC_SOURCES} ${ARRAY_SOURCES} ${TYPES_SOURCES}
${MEMORY_SOURCES} ${GRAPH_SOURCES} ${CUSTOMOPS_SOURCES} ${EXCEPTIONS_SOURCES} ${INDEXING_SOURCES} ${CUSTOMOPS_MKLDNN_SOURCES} ${MEMORY_SOURCES} ${GRAPH_SOURCES} ${CUSTOMOPS_SOURCES} ${EXCEPTIONS_SOURCES} ${INDEXING_SOURCES} ${CUSTOMOPS_MKLDNN_SOURCES}
${CUSTOMOPS_ARMCOMPUTE_SOURCES} ${CUSTOMOPS_GENERIC_SOURCES} ${OPS_SOURCES} ${PERF_SOURCES}) ${CUSTOMOPS_ARMCOMPUTE_SOURCES} ${CUSTOMOPS_GENERIC_SOURCES} ${OPS_SOURCES} ${PERF_SOURCES})
if(IOS) if(IOS)
add_library(${SD_LIBRARY_NAME} STATIC $<TARGET_OBJECTS:samediff_obj>) add_library(${SD_LIBRARY_NAME} STATIC $<TARGET_OBJECTS:samediff_obj>)
else() else()
# build shared library by default or when it's explicitly requested # build shared library by default or when it's explicitly requested
if(NOT SD_STATIC_LIB OR SD_SHARED_LIB) if(NOT SD_STATIC_LIB OR SD_SHARED_LIB)
add_library(${SD_LIBRARY_NAME} SHARED $<TARGET_OBJECTS:samediff_obj>) add_library(${SD_LIBRARY_NAME} SHARED $<TARGET_OBJECTS:samediff_obj>)
if(ANDROID)
# See: https://www.scivision.dev/cmake-ninja-job-pool-limited-memory/
# See: https://cmake.org/cmake/help/v3.0/command/cmake_host_system_information.html
# See: https://cmake.org/cmake/help/latest/prop_gbl/JOB_POOLS.html
cmake_host_system_information(RESULT _logical_cores QUERY NUMBER_OF_LOGICAL_CORES)
if(_logical_cores LESS 4)
set_target_properties(${SD_LIBRARY_NAME} PROPERTIES JOB_POOL_COMPILE one_jobs)
endif()
endif()
endif() endif()
if (SD_STATIC_LIB AND SD_SHARED_LIB) if (SD_STATIC_LIB AND SD_SHARED_LIB)

49
libnd4j/buildnativeoperations.sh
View File

@ -86,7 +86,10 @@ VERBOSE="false"
VERBOSE_ARG="VERBOSE=1" VERBOSE_ARG="VERBOSE=1"
HELPER= HELPER=
CHECK_VECTORIZATION="OFF" CHECK_VECTORIZATION="OFF"
SYS_ROOT=
EXTRA_LINK_FLAGS=
NAME= NAME=
EXTRA_CUDA_FLAGS=
while [[ $# -gt 0 ]] while [[ $# -gt 0 ]]
do do
key="$1" key="$1"
@ -399,6 +402,11 @@ if [ -z "$BUILD" ]; then
fi fi
if [ -z "$SYS_ROOT" ]; then
export SYS_ROOT=""
fi
if [ -z "$CHIP" ]; then if [ -z "$CHIP" ]; then
CHIP="cpu" CHIP="cpu"
fi fi
@ -411,9 +419,7 @@ if [ -z "$PACKAGING" ]; then
PACKAGING="none" PACKAGING="none"
fi fi
if [ -z "$COMPUTE" ]; then
COMPUTE="all"
fi
if [ "$CHIP_EXTENSION" == "avx512" ] || [ "$ARCH" == "avx512" ]; then if [ "$CHIP_EXTENSION" == "avx512" ] || [ "$ARCH" == "avx512" ]; then
CHIP_EXTENSION="avx512" CHIP_EXTENSION="avx512"
@ -430,6 +436,14 @@ if [ -z "$ARCH" ]; then
ARCH="x86-64" ARCH="x86-64"
fi fi
if [ -z "$COMPUTE" ]; then
if [ "$ARCH" == "x86-64" ]; then
COMPUTE="5.0 5.2 5.3 6.0 6.2 8.0"
else
COMPUTE="5.0 5.2 5.3 6.0 6.2"
fi
fi
OPERATIONS_ARG= OPERATIONS_ARG=
if [ -z "$OPERATIONS" ]; then if [ -z "$OPERATIONS" ]; then
@ -503,6 +517,13 @@ if [ "$TESTS" == "true" ]; then
TESTS_ARG="-DSD_BUILD_TESTS=ON" TESTS_ARG="-DSD_BUILD_TESTS=ON"
fi fi
if [ "$SYS_ROOT" != "" ]; then
EXTRA_SYSROOT="-DCMAKE_SYSROOT=$SYS_ROOT"
else
EXTRA_SYSROOT=""
fi
ARCH_ARG="-DSD_ARCH=$ARCH -DSD_EXTENSION=$CHIP_EXTENSION" ARCH_ARG="-DSD_ARCH=$ARCH -DSD_EXTENSION=$CHIP_EXTENSION"
CUDA_COMPUTE="-DCOMPUTE=\"$COMPUTE\"" CUDA_COMPUTE="-DCOMPUTE=\"$COMPUTE\""
@ -511,6 +532,16 @@ if [ "$CHIP" == "cuda" ] && [ -n "$CHIP_VERSION" ]; then
case $OS in case $OS in
linux*) linux*)
export CUDA_PATH="/usr/local/cuda-$CHIP_VERSION/" export CUDA_PATH="/usr/local/cuda-$CHIP_VERSION/"
# Cross compilation for jetson nano
if [ "$ARCH" != "x86-64" ]; then
if [ "$ARCH" == "armv8-a" ]; then
export EXTRA_CUDA_FLAGS="-DCUDA_TARGET_CPU_ARCH=AARCH64"
else
export EXTRA_CUDA_FLAGS="-DCUDA_TARGET_CPU_ARCH=ARM"
fi
else
export EXTRA_CUDA_FLAGS=""
fi
;; ;;
macosx*) macosx*)
export CUDA_PATH="/Developer/NVIDIA/CUDA-$CHIP_VERSION/" export CUDA_PATH="/Developer/NVIDIA/CUDA-$CHIP_VERSION/"
@ -578,6 +609,13 @@ else
IFS=' ' IFS=' '
fi fi
LINKER_FLAGS=""
if [ "$EXTRA_LINK_FLAGS" != "" ]; then
LINKER_FLAGS="-DCMAKE_CXX_LINK_FLAGS=$EXTRA_LINK_FLAGS -DCMAKE_EXE_LINKER_FLAGS=$EXTRA_LINK_FLAGS -DCMAKE_CUDA_FLAGS=$EXTRA_LINK_FLAGS"
fi
echo PACKAGING = "${PACKAGING}" echo PACKAGING = "${PACKAGING}"
echo BUILD = "${BUILD}" echo BUILD = "${BUILD}"
echo CHIP = "${CHIP}" echo CHIP = "${CHIP}"
@ -594,9 +632,12 @@ echo NAME = "${NAME_ARG}"
echo OPENBLAS_PATH = "$OPENBLAS_PATH" echo OPENBLAS_PATH = "$OPENBLAS_PATH"
echo CHECK_VECTORIZATION = "$CHECK_VECTORIZATION" echo CHECK_VECTORIZATION = "$CHECK_VECTORIZATION"
echo HELPERS = "$HELPERS" echo HELPERS = "$HELPERS"
echo EXTRA_LINK_FLAGS = "$EXTRA_LINK_FLAGS"
echo EXTRA_CUDA_FLAGS = "$EXTRA_CUDA_FLAGS"
echo EXTRA_SYSROOT = "$EXTRA_SYSROOT"
mkbuilddir mkbuilddir
pwd pwd
eval "$CMAKE_COMMAND" "$BLAS_ARG" "$ARCH_ARG" "$NAME_ARG" -DSD_CHECK_VECTORIZATION="${CHECK_VECTORIZATION}" "$HELPERS" "$SHARED_LIBS_ARG" "$MINIFIER_ARG" "$OPERATIONS_ARG" "$BUILD_TYPE" "$PACKAGING_ARG" "$EXPERIMENTAL_ARG" "$TESTS_ARG" "$CUDA_COMPUTE" -DOPENBLAS_PATH="$OPENBLAS_PATH" -DDEV=FALSE -DCMAKE_NEED_RESPONSE=YES -DMKL_MULTI_THREADED=TRUE ../.. eval "$CMAKE_COMMAND" "$EXTRA_SYSROOT" "$LINKER_FLAGS" "$EXTRA_CUDA_FLAGS" "$BLAS_ARG" "$ARCH_ARG" "$NAME_ARG" -DSD_CHECK_VECTORIZATION="${CHECK_VECTORIZATION}" "$HELPERS" "$SHARED_LIBS_ARG" "$MINIFIER_ARG" "$OPERATIONS_ARG" "$BUILD_TYPE" "$PACKAGING_ARG" "$EXPERIMENTAL_ARG" "$TESTS_ARG" "$CUDA_COMPUTE" -DOPENBLAS_PATH="$OPENBLAS_PATH" -DDEV=FALSE -DCMAKE_NEED_RESPONSE=YES -DMKL_MULTI_THREADED=TRUE ../..
if [ "$PARALLEL" == "true" ]; then if [ "$PARALLEL" == "true" ]; then
MAKE_ARGUMENTS="$MAKE_ARGUMENTS -j $MAKEJ" MAKE_ARGUMENTS="$MAKE_ARGUMENTS -j $MAKEJ"

87
libnd4j/cmake/GenCompilation.cmake
View File

@ -17,90 +17,3 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
################################################################################ ################################################################################
#///////////////////////////////////////////////////////////////////////////////
# genCompilation: Generates cpp, cu files
# INPUT:
# $FILE_ITEM template-configuration that utilizes libnd4j type, macros helpers
# defined inside { include/types/types.h, include/system/type_boilerplate.h}
# OUTPUT:
# $CUSTOMOPS_GENERIC_SOURCES generated files will be added into this List
#////////////////////////////////////////////////////////////////////////////////
# A simple template-configuration file example:
# // hints and defines what types will be generated
# #cmakedefine LIBND4J_TYPE_GEN
# #cmakedefine FLOAT_TYPE_GEN
# // below if defines blocks are needed for correctly handling multiple types
# #if defined(LIBND4J_TYPE_GEN)
# BUILD_DOUBLE_TEMPLATE(template void someFunc, (arg_list,..),
# LIBND4J_TYPES_@FL_TYPE_INDEX@, INDEXING_TYPES);
# #endif
# #if defined(FLOAT_TYPE_GEN)
# BUILD_SINGLE_TEMPLATE(template class SomeClass,, FLOAT_TYPES_@FL_TYPE_INDEX@);
# #endif
#////////////////////////////////////////////////////////////////////////////////
function(genCompilation FILE_ITEM)
get_filename_component(FILE_ITEM_WE ${FL_ITEM} NAME_WE)
set(EXTENSION "cpp")
if(FL_ITEM MATCHES "cu.in$")
set(EXTENSION "cu")
endif()
file(READ ${FL_ITEM} CONTENT_FL)
#check content for types
#set all to false
set (FLOAT_TYPE_GEN 0)
set (INT_TYPE_GEN 0)
set (LIBND4J_TYPE_GEN 0)
set (PAIRWISE_TYPE_GEN 0)
set (RANGE_STOP -1)
string(REGEX MATCHALL "#cmakedefine[ \t]+[^_]+_TYPE_GEN" TYPE_MATCHES ${CONTENT_FL})
foreach(TYPEX ${TYPE_MATCHES})
set(STOP -1)
if(TYPEX MATCHES "INT_TYPE_GEN$")
set (INT_TYPE_GEN 1)
set(STOP 7)
endif()
if(TYPEX MATCHES "LIBND4J_TYPE_GEN$")
set (LIBND4J_TYPE_GEN 1)
set(STOP 9)
endif()
if(TYPEX MATCHES "FLOAT_TYPE_GEN$")
set (FLOAT_TYPE_GEN 1)
set(STOP 3)
endif()
if(TYPEX MATCHES "PAIRWISE_TYPE_GEN$")
set (PAIRWISE_TYPE_GEN 1)
set(STOP 12)
endif()
if(STOP GREATER RANGE_STOP)
set(RANGE_STOP ${STOP})
endif()
endforeach()
if(RANGE_STOP GREATER -1)
foreach(FL_TYPE_INDEX RANGE 0 ${RANGE_STOP})
# set OFF if the index is above
if(FL_TYPE_INDEX GREATER 3)
set (FLOAT_TYPE_GEN 0)
endif()
if(FL_TYPE_INDEX GREATER 7)
set (INT_TYPE_GEN 0)
endif()
if(FL_TYPE_INDEX GREATER 9)
set (LIBND4J_TYPE_GEN 0)
endif()
set(GENERATED_SOURCE "${CMAKE_BINARY_DIR}/compilation_units/${FILE_ITEM_WE}_${FL_TYPE_INDEX}.${EXTENSION}")
configure_file( "${FL_ITEM}" "${GENERATED_SOURCE}" @ONLY)
LIST(APPEND CUSTOMOPS_GENERIC_SOURCES ${GENERATED_SOURCE} )
endforeach()
endif()
set(CUSTOMOPS_GENERIC_SOURCES ${CUSTOMOPS_GENERIC_SOURCES} PARENT_SCOPE)
endfunction()

234
libnd4j/nano_build.sh Normal file
View File

@ -0,0 +1,234 @@
#!/usr/bin/env bash
#
# /* ******************************************************************************
# *
# *
# * This program and the accompanying materials are made available under the
# * terms of the Apache License, Version 2.0 which is available at
# * https://www.apache.org/licenses/LICENSE-2.0.
# *
# * See the NOTICE file distributed with this work for additional
# * information regarding copyright ownership.
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# * License for the specific language governing permissions and limitations
# * under the License.
# *
# * SPDX-License-Identifier: Apache-2.0
# ******************************************************************************/
#
function message {
echo "BUILDER:::: ${@}"
}
if [ -z "${BUILD_USING_MAVEN}" ]; then export BUILD_USING_MAVEN=; fi
if [ -z "${CURRENT_TARGET}" ]; then export CURRENT_TARGET=arm32; fi
if [ -z "${HAS_ARMCOMPUTE}" ]; then export ARMCOMPUTE_DEBUG=1; fi
if [ -z "${ARMCOMPUTE_DEBUG}" ]; then export HAS_ARMCOMPUTE=1; fi
if [ -z "${ARMCOMPUTE_TAG}" ]; then export ARMCOMPUTE_TAG=v20.05; fi
if [ -z "${LIBND4J_BUILD_MODE}" ]; then export LIBND4J_BUILD_MODE=Release; fi
if [ -z "${ANDROID_VERSION}" ]; then export ANDROID_VERSION=21; fi
if [ -z "${HAS_ARMCOMPUTE}" ]; then export HAS_ARMCOMPUTE=1; fi
OTHER_ARGS=()
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-a|--arch)
CURRENT_TARGET="$2"
shift
shift
;;
-m|--mvn)
BUILD_USING_MAVEN="mvn"
shift
;;
*)
OTHER_ARGS+=("$1")
shift
;;
esac
done
CC_URL32="https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf.tar.xz?revision=e09a1c45-0ed3-4a8e-b06b-db3978fd8d56&la=en&hash=93ED4444B8B3A812B893373B490B90BBB28FD2E3"
CC_URL64="https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en&hash=167687FADA00B73D20EED2A67D0939A197504ACD"
CC_ANDROID="https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip"
COMPILER_ARRS=( "${CC_URL32}" "${CC_URL64}" "${CC_ANDROID}" "${CC_ANDROID}" )
COMPILER_DOWNLOAD_CMD_LIST=( download_extract_xz download_extract_xz download_extract_unzip download_extract_unzip )
COMPILER_DESTDIR=( "arm32" "arm64" "android" "android" )
PREFIXES=( arm-linux-gnueabihf aarch64-linux-gnu arm-linux-androideabi aarch64-linux-android )
TARGET_INDEX=-1
for i in "${!TARGET_ARRS[@]}"; do
if [[ "${TARGET_ARRS[$i]}" = "${CURRENT_TARGET}" ]]; then
TARGET_INDEX=${i}
fi
done
if [ ${TARGET_INDEX} -eq -1 ];then
message "could not find ${CURRENT_TARGET} in ${TARGET_ARRS[@]}"
exit -1
fi
#BASE_DIR=${HOME}/pi
#https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
BASE_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
export CROSS_COMPILER_URL="https://developer.nvidia.com/embedded/dlc/l4t-gcc-toolchain-64-bit-32-5"
export CROSS_COMPILER_DIR=${BASE_DIR}/compile_tools/cross_compiler_${COMPILER_DESTDIR[$TARGET_INDEX]}
export COMPILER_DOWNLOAD_CMD=${COMPILER_DOWNLOAD_CMD_LIST[$TARGET_INDEX]}
export DETECT=${DETECT_LIST[$TARGET_INDEX]}
export LIBND4J_PLATFORM_EXT=${LIBND4J_PLATFORM_EXT_LIST[$TARGET_INDEX]}
export TARGET_OS="linux"
export LIBND4J_PLATFORM="linux-arm64"
export PREFIX=${PREFIXES[$TARGET_INDEX]}
export CMAKE=cmake #/snap/bin/cmake
mkdir -p ${BASE_DIR}/compile_tools/
mkdir -p ${BASE_DIR}
mkdir -p ${THIRD_PARTY}
#change directory to base
cd $BASE_DIR
function check_requirements {
for i in "${@}"
do
if [ ! -e "$i" ]; then
message "missing: ${i}"
exit -2
fi
done
}
function rename_top_folder {
for dir in ${1}/*
do
if [ -d "$dir" ]
then
mv "${dir}" "${1}/folder/"
message "${dir} => ${1}/folder/"
break
fi
done
}
function download_extract_base {
#$1 is url #2 is dir $3 is extract argument
if [ ! -f ${3}_file ]; then
message "download"
wget --quiet --show-progress -O ${3}_file ${2}
fi
message "extract $@"
#extract
mkdir -p ${3}
if [ ${1} = "-unzip" ]; then
command="unzip -qq ${3}_file -d ${3} "
else
command="tar ${1} ${3}_file --directory=${3} "
fi
message $command
$command
check_requirements "${3}"
}
function download_extract {
download_extract_base -xzf $@
}
function download_extract_xz {
download_extract_base -xf $@
}
function download_extract_unzip {
download_extract_base -unzip $@
}
function git_check {
#$1 is url #$2 is dir #$3 is tag or branch if optional
command=
if [ -n "$3" ]; then
command="git clone --quiet --depth 1 --branch ${3} ${1} ${2}"
else
command="git clone --quiet ${1} ${2}"
fi
message "$command"
$command
check_requirements "${2}"
}
#fix py debug linkage manually and also makes it use gold
function fix_pi_linker {
#$1 BINUTILS folder
if [ ! -f ${1}/ld.original ]; then
mv ${1}/ld ${1}/ld.original
fi
rm -f ${1}/ld
printf '#!/usr/bin/env bash\n'"${1}/ld.gold --long-plt \$*">${1}/ld
chmod +x ${1}/ld
}
if [ ! -d ${CROSS_COMPILER_DIR}/folder ]; then
#out file
message "download CROSS_COMPILER"
${COMPILER_DOWNLOAD_CMD} ${CROSS_COMPILER_URL} ${CROSS_COMPILER_DIR}
message "rename top folder (instead of --strip-components=1)"
rename_top_folder ${CROSS_COMPILER_DIR}
fi
export CROSS_COMPILER_DIR=${CROSS_COMPILER_DIR}/folder
export BINUTILS_BIN=${CROSS_COMPILER_DIR}/${PREFIX}/bin
export COMPILER_PREFIX=${CROSS_COMPILER_DIR}/bin/${PREFIX}
export TOOLCHAIN_PREFIX=${COMPILER_PREFIX}
export SYS_ROOT=${CROSS_COMPILER_DIR}/${PREFIX}/libc
#LD_LIBRARY_PATH=${CROSS_COMPILER_DIR}/lib:$LD_LIBRARY_PATH
export CC_EXE="gcc"
export CXX_EXE="g++"
export RANLIB="${BINUTILS_BIN}/ranlib"
export LD="${BINUTILS_BIN}/ld"
export AR="${BINUTILS_BIN}/ar"
export BLAS_XTRA="CC=${COMPILER_PREFIX}-${CC_EXE} AR=${AR} RANLIB=${RANLIB} CFLAGS=--sysroot=${SYS_ROOT} LDFLAGS=\"-L${SYS_ROOT}/../lib/ -lm\""
check_requirements ${CC}
#because of the toolchain passive detection we have to delete build folder manually
detect=$(cat ${BASE_DIR}/blasbuild/cpu/CMakeCache.txt | grep -o ${PREFIX})
if [ -z "${detect}" ] ;then
message "remove blasbuild folder "
rm -rf $BASE_DIR/blasbuild/
else
message "keep blasbuild folder"
fi
if [ -z "${BUILD_USING_MAVEN}" ] ;then
message "lets build just library"
DHELPER=" -h armcompute "
bash ./buildnativeoperations.sh -o ${LIBND4J_PLATFORM} -t ${DHELPER} -j $(nproc)
else
message "cd $BASE_DIR/.. "
cd $BASE_DIR/..
message "lets build jars"
export DHELPER=" -Dlibnd4j.helper=armcompute "
if [ "${DEPLOY}" ]; then
echo "Deploying to maven"
mvn -Pgithub deploy --batch-mode -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER} -pl ":libnd4j,:nd4j-native" --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true
else
echo "Installing to local repo"
mvn install -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER} -pl ":libnd4j" --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true
fi
fi

84
libnd4j/pi_build.sh
View File

@ -22,14 +22,15 @@
function message { function message {
echo "BUILDER:::: ${@}" echo "BUILDER:::: ${@}"
} }
if [ -z "${BUILD_USING_MAVEN}" ]; then export BUILD_USING_MAVEN=; fi
if [ -z "${CURRENT_TARGET}" ]; then export CURRENT_TARGET=arm32; fi
if [ -z "${HAS_ARMCOMPUTE}" ]; then export ARMCOMPUTE_DEBUG=1; fi
if [ -z "${ARMCOMPUTE_DEBUG}" ]; then export HAS_ARMCOMPUTE=1; fi
if [ -z "${ARMCOMPUTE_TAG}" ]; then export ARMCOMPUTE_TAG=v20.05; fi
if [ -z "${LIBND4J_BUILD_MODE}" ]; then export LIBND4J_BUILD_MODE=Release; fi
if [ -z "${ANDROID_VERSION}" ]; then export ANDROID_VERSION=21; fi
if [ -z "${HAS_ARMCOMPUTE}" ]; then export HAS_ARMCOMPUTE=1; fi
BUILD_USING_MAVEN=
CURRENT_TARGET=arm32
HAS_ARMCOMPUTE=1
ARMCOMPUTE_DEBUG=0
ARMCOMPUTE_TAG=v20.05
LIBND4J_BUILD_MODE=Release
export ANDROID_VERSION=21
OTHER_ARGS=() OTHER_ARGS=()
while [[ $# -gt 0 ]] while [[ $# -gt 0 ]]
do do
@ -88,18 +89,18 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
done done
BASE_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" BASE_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
CROSS_COMPILER_URL=${COMPILER_ARRS[$TARGET_INDEX]} export CROSS_COMPILER_URL=${COMPILER_ARRS[$TARGET_INDEX]}
CROSS_COMPILER_DIR=${BASE_DIR}/compile_tools/cross_compiler_${COMPILER_DESTDIR[$TARGET_INDEX]} export CROSS_COMPILER_DIR=${BASE_DIR}/compile_tools/cross_compiler_${COMPILER_DESTDIR[$TARGET_INDEX]}
COMPILER_DOWNLOAD_CMD=${COMPILER_DOWNLOAD_CMD_LIST[$TARGET_INDEX]} export COMPILER_DOWNLOAD_CMD=${COMPILER_DOWNLOAD_CMD_LIST[$TARGET_INDEX]}
DETECT=${DETECT_LIST[$TARGET_INDEX]} export DETECT=${DETECT_LIST[$TARGET_INDEX]}
LIBND4J_PLATFORM_EXT=${LIBND4J_PLATFORM_EXT_LIST[$TARGET_INDEX]} export LIBND4J_PLATFORM_EXT=${LIBND4J_PLATFORM_EXT_LIST[$TARGET_INDEX]}
BLAS_TARGET_NAME=${OPENBLAS_TARGETS[$TARGET_INDEX]} export BLAS_TARGET_NAME=${OPENBLAS_TARGETS[$TARGET_INDEX]}
ARMCOMPUTE_TARGET=${ARMCOMPUTE_TARGETS[$TARGET_INDEX]} export ARMCOMPUTE_TARGET=${ARMCOMPUTE_TARGETS[$TARGET_INDEX]}
TARGET_OS=${OS_LIST[$TARGET_INDEX]} export TARGET_OS=${OS_LIST[$TARGET_INDEX]}
LIBND4J_PLATFORM=${TARGET_OS}-${LIBND4J_PLATFORM_EXT} export LIBND4J_PLATFORM=${TARGET_OS}-${LIBND4J_PLATFORM_EXT}
PREFIX=${PREFIXES[$TARGET_INDEX]} export PREFIX=${PREFIXES[$TARGET_INDEX]}
CMAKE=cmake #/snap/bin/cmake export CMAKE=cmake #/snap/bin/cmake
mkdir -p ${BASE_DIR}/compile_tools/ mkdir -p ${BASE_DIR}/compile_tools/
SCONS_LOCAL_URL=http://prdownloads.sourceforge.net/scons/scons-local-3.1.1.tar.gz SCONS_LOCAL_URL=http://prdownloads.sourceforge.net/scons/scons-local-3.1.1.tar.gz
@ -206,33 +207,33 @@ if [ ! -d ${CROSS_COMPILER_DIR}/folder ]; then
rename_top_folder ${CROSS_COMPILER_DIR} rename_top_folder ${CROSS_COMPILER_DIR}
fi fi
CROSS_COMPILER_DIR=${CROSS_COMPILER_DIR}/folder export CROSS_COMPILER_DIR=${CROSS_COMPILER_DIR}/folder
if [ "${TARGET_OS}" = "android" ];then if [ "${TARGET_OS}" = "android" ];then
ANDROID_TOOLCHAIN=${CROSS_COMPILER_DIR}/toolchains/llvm/prebuilt/linux-x86_64 export ANDROID_TOOLCHAIN=${CROSS_COMPILER_DIR}/toolchains/llvm/prebuilt/linux-x86_64
COMPILER_PREFIX="${ANDROID_TOOLCHAIN}/bin/${PREFIX}${ANDROID_VERSION}" export COMPILER_PREFIX="${ANDROID_TOOLCHAIN}/bin/${PREFIX}${ANDROID_VERSION}"
TOOLCHAIN_PREFIX="${ANDROID_TOOLCHAIN}/bin/${PREFIX}" export TOOLCHAIN_PREFIX="${ANDROID_TOOLCHAIN}/bin/${PREFIX}"
if [ "$BLAS_TARGET_NAME" = "ARMV7" ];then if [ "$BLAS_TARGET_NAME" = "ARMV7" ];then
BLAS_XTRA="ARM_SOFTFP_ABI=1 " BLAS_XTRA="ARM_SOFTFP_ABI=1 "
COMPILER_PREFIX="${ANDROID_TOOLCHAIN}/bin/armv7a-linux-androideabi${ANDROID_VERSION}" COMPILER_PREFIX="${ANDROID_TOOLCHAIN}/bin/armv7a-linux-androideabi${ANDROID_VERSION}"
fi fi
CC_EXE="clang" export CC_EXE="clang"
CXX_EXE="clang++" export CXX_EXE="clang++"
AR="${TOOLCHAIN_PREFIX}-ar" export AR="${TOOLCHAIN_PREFIX}-ar"
RANLIB="${TOOLCHAIN_PREFIX}-ranlib" export RANLIB="${TOOLCHAIN_PREFIX}-ranlib"
BLAS_XTRA="CC=${COMPILER_PREFIX}-${CC_EXE} AR=${AR} RANLIB=${RANLIB} ${BLAS_XTRA}" export BLAS_XTRA="CC=${COMPILER_PREFIX}-${CC_EXE} AR=${AR} RANLIB=${RANLIB} ${BLAS_XTRA}"
else else
BINUTILS_BIN=${CROSS_COMPILER_DIR}/${PREFIX}/bin export BINUTILS_BIN=${CROSS_COMPILER_DIR}/${PREFIX}/bin
COMPILER_PREFIX=${CROSS_COMPILER_DIR}/bin/${PREFIX} export COMPILER_PREFIX=${CROSS_COMPILER_DIR}/bin/${PREFIX}
TOOLCHAIN_PREFIX=${COMPILER_PREFIX} export TOOLCHAIN_PREFIX=${COMPILER_PREFIX}
SYS_ROOT=${CROSS_COMPILER_DIR}/${PREFIX}/libc export SYS_ROOT=${CROSS_COMPILER_DIR}/${PREFIX}/libc
#LD_LIBRARY_PATH=${CROSS_COMPILER_DIR}/lib:$LD_LIBRARY_PATH #LD_LIBRARY_PATH=${CROSS_COMPILER_DIR}/lib:$LD_LIBRARY_PATH
CC_EXE="gcc" export CC_EXE="gcc"
CXX_EXE="g++" export CXX_EXE="g++"
RANLIB="${BINUTILS_BIN}/ranlib" export RANLIB="${BINUTILS_BIN}/ranlib"
export LD="${BINUTILS_BIN}/ld" export LD="${BINUTILS_BIN}/ld"
AR="${BINUTILS_BIN}/ar" export AR="${BINUTILS_BIN}/ar"
BLAS_XTRA="CC=${COMPILER_PREFIX}-${CC_EXE} AR=${AR} RANLIB=${RANLIB} CFLAGS=--sysroot=${SYS_ROOT} LDFLAGS=\"-L${SYS_ROOT}/../lib/ -lm\"" export BLAS_XTRA="CC=${COMPILER_PREFIX}-${CC_EXE} AR=${AR} RANLIB=${RANLIB} CFLAGS=--sysroot=${SYS_ROOT} LDFLAGS=\"-L${SYS_ROOT}/../lib/ -lm\""
fi fi
check_requirements ${CC} check_requirements ${CC}
@ -315,6 +316,13 @@ else
message "cd $BASE_DIR/.. " message "cd $BASE_DIR/.. "
cd $BASE_DIR/.. cd $BASE_DIR/..
message "lets build jars" message "lets build jars"
DHELPER=" -Dlibnd4j.helper=armcompute " export DHELPER=" -Dlibnd4j.helper=armcompute "
mvn install -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER} -Dmaven.test.skip=true -Dmaven.javadoc.skip=true if [ "${DEPLOY}" ]; then
echo "Deploying to maven"
mvn -P"${PUBLISH_TO}" deploy --batch-mode -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER} -pl ":libnd4j,:nd4j-native" --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true
else
echo "Installing to local repo"
mvn install -Dlibnd4j.platform=${LIBND4J_PLATFORM} -Djavacpp.platform=${LIBND4J_PLATFORM} -DprotocCommand=protoc -Djavacpp.platform.compiler=${COMPILER_PREFIX}-${CC_EXE} -Djava.library.path=${JAVA_LIBRARY_PATH} ${DHELPER} -pl ":libnd4j" --also-make -DskipTests -Dmaven.test.skip=true -Dmaven.javadoc.skip=true
fi
fi fi