Add jenkinsfile for pipeline build and dockerfile for build
parent
a31fbb9942
commit
1ce4a0ba70
|
|
@ -1,7 +1,8 @@
|
|||
FROM nvidia/cuda:11.4.0-devel-ubuntu20.04
|
||||
FROM nvidia/cuda:11.4.0-cudnn8-devel-ubuntu20.04
|
||||
|
||||
RUN apt-get update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-11-jdk wget build-essential checkinstall zlib1g-dev libssl-dev
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-11-jdk wget build-essential checkinstall zlib1g-dev libssl-dev git
|
||||
RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2.tar.gz && \
|
||||
tar -xvf cmake-3.24.2.tar.gz && cd cmake-3.24.2 && \
|
||||
./bootstrap && make && make install
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,129 @@
|
|||
# BRUTEX CAVIS Build Requirements #
|
||||
|
||||
You can build CAVIS on either x86_64 linux or x86_64 windows
|
||||
(sometimes referred to as amd64). Support for native GPU (CUDA) or CPU (AVX2/ AVX512) support can be enabled.
|
||||
|
||||
Linux on x86_64 is the primary target platform.
|
||||
CAVIS is written in Java, C, C++ and Kotlin, therefore you will need a Java JDK and
|
||||
C/C++ compiler.
|
||||
|
||||
# Table of contents
|
||||
1. [Introduction](#introduction)
|
||||
2.
|
||||
2. [Some paragraph](#paragraph1)
|
||||
1. [Sub paragraph](#subparagraph1)
|
||||
3. [Another paragraph](#paragraph2)
|
||||
|
||||
## Build Environment Prerequisites <a id="introduction"></a>
|
||||
CAVIS uses the [Gradle Build Tool](https://gradle.org) and [CMake](https://cmake.org/) to execute the build.
|
||||
A Gradle Wrapper is provided, that will pull the right version of gradle.
|
||||
|
||||
### Linux x86_64 Prerequisites
|
||||
|
||||
Linux Kernel 5.10 or higher
|
||||
|
||||
**Required build tools:**
|
||||
|
||||
| Tools | Version | Remarks |
|
||||
|----------|------------|--------------------------------------------------|
|
||||
| Gradle | \> 7.2 | |
|
||||
| CMake | \> 2.24.10 | |
|
||||
| GNU Make | 3.82 | |
|
||||
|
||||
| Compiler | Version | Remarks |
|
||||
| -------- |----------------|---------------------------------------------------|
|
||||
| GCC | 7.3.1 - 10.2.1 | C and C++ compiler |
|
||||
| nvcc | \> 11.4 | only in case you want to compile for CUDA support |
|
||||
| javac | JDK 11 | openjdk-11 |
|
||||
|
||||
#### Amazon Linux 2 ####
|
||||
Preparation shell script
|
||||
|
||||
```shell
|
||||
sudo amazon-linux-extras install java-openjdk11 && \
|
||||
sudo yum install --assumeyes git make gcc gcc-c++ openssl-devel java-11-openjdk-devel && \
|
||||
wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2.tar.gz && \
|
||||
tar -xf cmake-3.24.2.tar.gz && cd cmake-3.24.2/ && \
|
||||
./bootstrap && make && sudo make install && cd ..
|
||||
```
|
||||
|
||||
In case you have less than 8GB RAM, a swap file should be created
|
||||
```shell
|
||||
sudo dd if=/dev/zero of=/swapfile count=8k bs=1M && sudo chmod 600 /swapfile && \
|
||||
sudo mkswap /swapfile && sudo swapon /swapfile
|
||||
```
|
||||
|
||||
You can make the swap change permanent (surviving a reboot) by putting it into /etc/fstab file.
|
||||
```shell
|
||||
echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab
|
||||
```
|
||||
|
||||
In order to resolve Brutex Network IP6 endpoints, you need to add an additional nameserver
|
||||
```shell
|
||||
echo 'nameserver 8.8.8.8' | sudo tee -a /etc/resolv.conf
|
||||
```
|
||||
## Some paragraph <a id="paragraph1"></a>
|
||||
The first paragraph text
|
||||
|
||||
### Sub paragraph <a id="subparagraph1"></a>
|
||||
This is a sub paragraph, formatted in heading 3 style
|
||||
|
||||
## Troubleshooting <a id="paragraph2"></a>
|
||||
### Gradle process just crashes ###
|
||||
This is likely an out-of-memory problem, check
|
||||
```shell
|
||||
dmesg
|
||||
```
|
||||
for
|
||||
```
|
||||
oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/,task=java,pid=16477,uid=1000
|
||||
[ 6291.694315] Out of memory: Killed process 16477 (java) total-vm:11373580kB, anon-rss:1373168kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:11220kB oom_score_adj:0
|
||||
```
|
||||
and increase i.e. swapfile size (by adding another one).
|
||||
|
||||
|
||||
|
||||
sudo apt-get -q update
|
||||
ret=$?
|
||||
while [ $ret -ne 0 ];
|
||||
do
|
||||
echo Waiting for instance to be ready ...
|
||||
sleep 20s
|
||||
sudo apt-get -q update
|
||||
ret=$?
|
||||
done
|
||||
|
||||
echo Instance is ready.
|
||||
|
||||
#Create swap
|
||||
echo Adding swap file
|
||||
sudo fallocate -l 16G /swapfile
|
||||
sudo chmod 600 /swapfile
|
||||
sudo mkswap /swapfile
|
||||
sudo swapon /swapfile
|
||||
echo Swap created.
|
||||
|
||||
#echo "deb http://deb.debian.org/debian/ buster main" | sudo tee /etc/apt/sources.list
|
||||
#echo "deb http://deb.debian.org/debian/ buster main contrib non-free" | sudo tee -a /etc/apt/sources.list
|
||||
|
||||
|
||||
sudo apt-get -q --yes install build-essential default-jdk git ninja-build gnupg software-properties-common snapd make g++-multilib g++ g++-8 gcc gcc-8 libopenblas-base libopenblas-dev
|
||||
ret=$?
|
||||
while [ $ret -ne 0 ];
|
||||
do
|
||||
echo Waiting for instance to be ready ...
|
||||
sleep 20s
|
||||
sudo apt-get -q --yes install build-essential default-jdk git ninja-build gnupg software-properties-common snapd make g++-multilib g++ g++-8 gcc gcc-8 libopenblas-base libopenblas-dev
|
||||
ret=$?
|
||||
done
|
||||
#sudo snap install core
|
||||
#sudo snap install cmake --classic
|
||||
|
||||
wget https://github.com/Kitware/CMake/releases/download/v3.20.4/cmake-3.20.4-linux-x86_64.sh
|
||||
chmod a+x cmake-3.20.4-linux-x86_64.sh
|
||||
sudo sh cmake-3.20.4-linux-x86_64.sh --skip-license
|
||||
|
||||
echo "supersede domain-name-servers 172.31.0.2, 8.8.8.8" | sudo tee -a /etc/dhcp/dhclient.conf
|
||||
echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf
|
||||
|
||||
|
||||
|
|
@ -31,7 +31,10 @@ systemProp.org.gradle.internal.publish.checksums.insecure=true
|
|||
|
||||
# Specifies the JVM arguments used for the daemon process.
|
||||
# The setting is particularly useful for tweaking memory settings.
|
||||
org.gradle.jvmargs=-Xmx8192m
|
||||
|
||||
#for whatever reason we had to add MaxMetaspace and file encoding = utf8, gradle crashed otherwise
|
||||
org.gradle.jvmargs=-Xmx8192m -XX:MaxMetaspaceSize=768m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 -XX:ErrorFile=/var/log/java/hs_err_pid%p.log
|
||||
|
||||
|
||||
# When configured, Gradle will run in incubating parallel mode.
|
||||
# This option should only be used with decoupled projects. More details, visit
|
||||
|
|
|
|||
95
jenkinsfile
95
jenkinsfile
|
|
@ -1,44 +1,85 @@
|
|||
pipeline {
|
||||
|
||||
|
||||
agent {
|
||||
|
||||
dockerfile {
|
||||
filename 'Dockerfile'
|
||||
dir '.docker'
|
||||
label 'linuxdocker'
|
||||
//additionalBuildArgs '--build-arg version=1.0.2'
|
||||
args '--gpus all'
|
||||
}
|
||||
}
|
||||
agent none
|
||||
|
||||
stages {
|
||||
stage('prep-build-environment') {
|
||||
stage('prep-build-environment-linux') {
|
||||
agent {
|
||||
label 'linux'
|
||||
}
|
||||
steps {
|
||||
//git poll: false, url: 'http://bru3-pc.fritz.box:3000/Brutex/ci-deeplearning4j.git'
|
||||
sh 'nvidia-smi'
|
||||
sh 'nvcc --version'
|
||||
checkout scm
|
||||
//sh 'nvidia-smi'
|
||||
//sh 'nvcc --version'
|
||||
sh 'gcc --version'
|
||||
sh 'cmake --version'
|
||||
checkout scm
|
||||
sh 'sh ./gradlew --version'
|
||||
}
|
||||
}
|
||||
stage('prep-build-environment-linux-cuda') {
|
||||
agent {
|
||||
dockerfile {
|
||||
filename 'Dockerfile'
|
||||
dir '.docker'
|
||||
label 'linuxdocker'
|
||||
//additionalBuildArgs '--build-arg version=1.0.2'
|
||||
args '--gpus all'
|
||||
}
|
||||
}
|
||||
steps {
|
||||
checkout scm
|
||||
sh 'nvidia-smi'
|
||||
sh 'nvcc --version'
|
||||
sh 'gcc --version'
|
||||
sh 'cmake --version'
|
||||
sh 'sh ./gradlew --version'
|
||||
}
|
||||
}
|
||||
stage('build') {
|
||||
environment {
|
||||
MAVEN = credentials('Internal Archiva')
|
||||
OSSRH = credentials('OSSRH')
|
||||
|
||||
MAVEN = credentials('Internal Archiva')
|
||||
OSSRH = credentials('OSSRH')
|
||||
}
|
||||
steps {
|
||||
|
||||
sh 'chmod u+x ./gradlew && ls -la'
|
||||
sh './gradlew --version'
|
||||
withGradle {
|
||||
sh './gradlew build --stacktrace --no-watch-fs -x test -PCAVIS_CHIP=cuda -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
|
||||
parallel {
|
||||
stage('build-linux-cuda') {
|
||||
agent {
|
||||
dockerfile {
|
||||
filename 'Dockerfile'
|
||||
dir '.docker'
|
||||
label 'linuxdocker'
|
||||
//additionalBuildArgs '--build-arg version=1.0.2'
|
||||
args '--gpus all'
|
||||
}
|
||||
}
|
||||
|
||||
steps {
|
||||
withGradle {
|
||||
sh 'sh ./gradlew publish --stacktrace -x test -PCAVIS_CHIP=cuda -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
|
||||
}
|
||||
//stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
|
||||
}
|
||||
}
|
||||
stage('build-linux-cpu') {
|
||||
agent {
|
||||
label 'linux'
|
||||
}
|
||||
steps {
|
||||
withGradle {
|
||||
sh 'sh ./gradlew publish --stacktrace -x test -PCAVIS_CHIP=cpu -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
|
||||
}
|
||||
//stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cpu-build'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* stage('collect-release') {
|
||||
agent {
|
||||
label 'linux'
|
||||
}
|
||||
steps {
|
||||
//unstash 'cuda-build'
|
||||
//unstash 'cpu-build'
|
||||
}
|
||||
}
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue