Add jenkinsfile for pipeline build and dockerfile for build

master
Brian Rosenberger 2022-10-07 15:04:30 +02:00
parent 11ba7a59c1
commit 205252e5a9
1425 changed files with 1381 additions and 28975 deletions

View File

@ -2,7 +2,12 @@ FROM nvidia/cuda:11.4.0-cudnn8-devel-ubuntu20.04
RUN apt-get update && \ RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-11-jdk wget build-essential checkinstall zlib1g-dev libssl-dev git DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-11-jdk wget build-essential checkinstall zlib1g-dev libssl-dev git
RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2.tar.gz && \ #Build cmake version from source \
tar -xvf cmake-3.24.2.tar.gz && cd cmake-3.24.2 && \ #RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2.tar.gz && \
./bootstrap && make && make install # tar -xvf cmake-3.24.2.tar.gz && cd cmake-3.24.2 && \
# ./bootstrap && make && make install
RUN wget -nv https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.sh && \
mkdir /opt/cmake && sh ./cmake-3.24.2-linux-x86_64.sh --skip-license --prefix=/opt/cmake && ln -s /opt/cmake/bin/cmake /usr/bin/cmake && \
rm cmake-3.24.2-linux-x86_64.sh

16
.gitignore vendored
View File

@ -50,12 +50,12 @@ release.properties
*.dylib *.dylib
.vs/ .vs/
.vscode/ .vscode/
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/bin .old/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/bin
nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/test/resources/writeNumpy.csv .old/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/test/resources/writeNumpy.csv
nd4j/nd4j-backends/nd4j-tests/src/test/resources/tf_graphs/examples/**/data-all* .old/nd4j/nd4j-backends/nd4j-tests/src/test/resources/tf_graphs/examples/**/data-all*
nd4j/nd4j-backends/nd4j-tests/src/test/resources/tf_graphs/examples/**/checkpoint .old/nd4j/nd4j-backends/nd4j-tests/src/test/resources/tf_graphs/examples/**/checkpoint
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/onnx/ .old/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/onnx/
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/tensorflow/ .old/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/tensorflow/
doc_sources/ doc_sources/
doc_sources_* doc_sources_*
@ -67,8 +67,8 @@ venv/
venv2/ venv2/
# Ignore the nd4j files that are created by javacpp at build to stop merge conflicts # Ignore the nd4j files that are created by javacpp at build to stop merge conflicts
nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpu.java .old/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpu.java
nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativeblas/Nd4jCuda.java .old/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativeblas/Nd4jCuda.java
# Ignore meld temp files # Ignore meld temp files
*.orig *.orig

View File

@ -0,0 +1,82 @@
/*
*
* ******************************************************************************
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*
*/
pipeline {
agent {
label 'linux'
}
stages {
stage('prep-build-environment-linux-cpu') {
steps {
checkout scm
sh 'gcc --version'
sh 'cmake --version'
sh 'sh ./gradlew --version'
}
}
stage('build-linux-cpu') {
environment {
MAVEN = credentials('Internal Archiva')
OSSRH = credentials('OSSRH')
}
steps {
withGradle {
sh 'sh ./gradlew build --stacktrace -x test -PCAVIS_CHIP=cpu \
-Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
-PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
}
//stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
}
}
/*stage('test-linux-cpu') {
environment {
MAVEN = credentials('Internal Archiva')
OSSRH = credentials('OSSRH')
}
steps {
withGradle {
sh 'sh ./gradlew test --stacktrace -PexcludeTests=\'long-running,performance\' -PCAVIS_CHIP=cpu \
-Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
-PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
}
//stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
}
}*/
stage('publish-linux-cpu') {
environment {
MAVEN = credentials('Internal Archiva')
OSSRH = credentials('OSSRH')
}
steps {
withGradle {
sh 'sh ./gradlew publish --stacktrace -PCAVIS_CHIP=cpu \
-Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
-PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
}
//stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
}
}
}
}

View File

@ -0,0 +1,60 @@
/*
*
* ******************************************************************************
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*
*/
pipeline {
agent {
dockerfile {
filename 'Dockerfile'
dir '.docker'
label 'linux && cuda'
//additionalBuildArgs '--build-arg version=1.0.2'
//args '--gpus all' --needed for test only, you can build without GPU
}
}
stages {
stage('prep-build-environment-linux-cuda') {
steps {
checkout scm
//sh 'nvidia-smi'
sh 'nvcc --version'
sh 'gcc --version'
sh 'cmake --version'
sh 'sh ./gradlew --version'
}
}
stage('build-linux-cuda') {
environment {
MAVEN = credentials('Internal Archiva')
OSSRH = credentials('OSSRH')
}
steps {
withGradle {
sh 'sh ./gradlew build --stacktrace -x test -PCAVIS_CHIP=cuda \
-Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
-PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
}
//stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
}
}
}
}

View File

@ -0,0 +1,66 @@
/*
*
* ******************************************************************************
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*
*/
pipeline {
agent {
dockerfile {
filename 'Dockerfile'
dir '.docker'
label 'linux && docker && cuda'
//additionalBuildArgs '--build-arg version=1.0.2'
//args '--gpus all' --needed for test only, you can build without GPU
}
}
stages {
stage("Build all chip") {
parallel {
stage('prep-build-environment-linux-cuda') {
steps {
checkout scm
//sh 'nvidia-smi'
sh 'nvcc --version'
sh 'gcc --version'
sh 'cmake --version'
sh 'sh ./gradlew --version'
}
}
stage('build-linux-cuda') {
environment {
MAVEN = credentials('Internal Archiva')
OSSRH = credentials('OSSRH')
}
steps {
withGradle {
sh 'sh ./gradlew build --stacktrace -x test -PCAVIS_CHIP=cuda \
-Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
-PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
}
//stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
}
}
}
}
}
}

View File

@ -24,7 +24,7 @@ pipeline {
dockerfile { dockerfile {
filename 'Dockerfile' filename 'Dockerfile'
dir '.docker' dir '.docker'
label 'linuxdocker' label 'linux && docker'
//additionalBuildArgs '--build-arg version=1.0.2' //additionalBuildArgs '--build-arg version=1.0.2'
//args '--gpus all' //args '--gpus all'
} }
@ -62,9 +62,9 @@ pipeline {
steps { steps {
withGradle { withGradle {
sh 'sh ./gradlew test --stacktrace -PCAVIS_CHIP=cpu \ //sh 'sh ./gradlew test --stacktrace -PCAVIS_CHIP=cpu \
-Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \ // -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
-PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW' // -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
} }
//stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build' //stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
} }

View File

@ -0,0 +1,49 @@
/*
*
* ******************************************************************************
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*
*/
pipeline {
agent {
dockerfile {
filename 'Dockerfile'
dir '.docker'
label 'linux && docker'
//additionalBuildArgs '--build-arg version=1.0.2'
//args '--gpus all'
}
}
stages {
stage('publish-linux-cpu') {
environment {
MAVEN = credentials('Internal Archiva')
OSSRH = credentials('OSSRH')
}
steps {
withGradle {
sh 'sh ./gradlew publish -x test -PCAVIS_CHIP=cpu \
-Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
-PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
}
}
}
}
}

View File

@ -24,9 +24,9 @@ pipeline {
dockerfile { dockerfile {
filename 'Dockerfile' filename 'Dockerfile'
dir '.docker' dir '.docker'
label 'linuxdocker-cuda' label 'linux && docker && cuda'
//additionalBuildArgs '--build-arg version=1.0.2' //additionalBuildArgs '--build-arg version=1.0.2'
args '--gpus all' //args '--gpus all' --needed for test only, you can build without GPU
} }
} }
@ -34,7 +34,7 @@ pipeline {
stage('prep-build-environment-linux-cuda') { stage('prep-build-environment-linux-cuda') {
steps { steps {
checkout scm checkout scm
sh 'nvidia-smi' //sh 'nvidia-smi'
sh 'nvcc --version' sh 'nvcc --version'
sh 'gcc --version' sh 'gcc --version'
sh 'cmake --version' sh 'cmake --version'
@ -49,7 +49,7 @@ pipeline {
steps { steps {
withGradle { withGradle {
sh 'sh ./gradlew publish --stacktrace -x test -PCAVIS_CHIP=cuda \ sh 'sh ./gradlew build --stacktrace -x test -PCAVIS_CHIP=cuda \
-Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \ -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
-PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW' -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
} }

Some files were not shown because too many files have changed in this diff Show More