Brutex
/
cavis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add jenkinsfile for pipeline build and dockerfile for build

master
Brian Rosenberger 2022-09-24 16:38:24 +02:00
parent a31fbb9942
commit 1ce4a0ba70
4 changed files with 204 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.docker/Dockerfile
View File

@ -1,7 +1,8 @@
FROM nvidia/cuda:11.4.0-devel-ubuntu20.04
FROM nvidia/cuda:11.4.0-cudnn8-devel-ubuntu20.04
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-11-jdk wget build-essential checkinstall zlib1g-dev libssl-dev
DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-11-jdk wget build-essential checkinstall zlib1g-dev libssl-dev git
RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2.tar.gz && \
tar -xvf cmake-3.24.2.tar.gz && cd cmake-3.24.2 && \
./bootstrap && make && make install

129
build_requirements.md Normal file
View File

@ -0,0 +1,129 @@
# BRUTEX CAVIS Build Requirements #
You can build CAVIS on either x86_64 linux or x86_64 windows
(sometimes referred to as amd64). Support for native GPU (CUDA) or CPU (AVX2/ AVX512) support can be enabled.
Linux on x86_64 is the primary target platform.
CAVIS is written in Java, C, C++ and Kotlin, therefore you will need a Java JDK and
C/C++ compiler.
# Table of contents
1. [Introduction](#introduction)
2.
2. [Some paragraph](#paragraph1)
1. [Sub paragraph](#subparagraph1)
3. [Another paragraph](#paragraph2)
## Build Environment Prerequisites <a id="introduction"></a>
CAVIS uses the [Gradle Build Tool](https://gradle.org) and [CMake](https://cmake.org/) to execute the build.
A Gradle Wrapper is provided, that will pull the right version of gradle.
### Linux x86_64 Prerequisites
Linux Kernel 5.10 or higher
**Required build tools:**
| Tools | Version | Remarks |
|----------|------------|--------------------------------------------------|
| Gradle | \> 7.2 | |
| CMake | \> 2.24.10 | |
| GNU Make | 3.82 | |
| Compiler | Version | Remarks |
| -------- |----------------|---------------------------------------------------|
| GCC | 7.3.1 - 10.2.1 | C and C++ compiler |
| nvcc | \> 11.4 | only in case you want to compile for CUDA support |
| javac | JDK 11 | openjdk-11 |
#### Amazon Linux 2 ####
Preparation shell script
```shell
sudo amazon-linux-extras install java-openjdk11 && \
sudo yum install --assumeyes git make gcc gcc-c++ openssl-devel java-11-openjdk-devel && \
wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2.tar.gz && \
tar -xf cmake-3.24.2.tar.gz && cd cmake-3.24.2/ && \
./bootstrap && make && sudo make install && cd ..
```
In case you have less than 8GB RAM, a swap file should be created
```shell
sudo dd if=/dev/zero of=/swapfile count=8k bs=1M && sudo chmod 600 /swapfile && \
sudo mkswap /swapfile && sudo swapon /swapfile
```
You can make the swap change permanent (surviving a reboot) by putting it into /etc/fstab file.
```shell
echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab
```
In order to resolve Brutex Network IP6 endpoints, you need to add an additional nameserver
```shell
echo 'nameserver 8.8.8.8' | sudo tee -a /etc/resolv.conf
```
## Some paragraph <a id="paragraph1"></a>
The first paragraph text
### Sub paragraph <a id="subparagraph1"></a>
This is a sub paragraph, formatted in heading 3 style
## Troubleshooting <a id="paragraph2"></a>
### Gradle process just crashes ###
This is likely an out-of-memory problem, check
```shell
dmesg
```
for
```
oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/,task=java,pid=16477,uid=1000
[ 6291.694315] Out of memory: Killed process 16477 (java) total-vm:11373580kB, anon-rss:1373168kB, file-rss:0kB, shmem-rss:0kB, UID:1000 pgtables:11220kB oom_score_adj:0
```
and increase i.e. swapfile size (by adding another one).
sudo apt-get -q update
ret=$?
while [ $ret -ne 0 ];
do
echo Waiting for instance to be ready ...
sleep 20s
sudo apt-get -q update
ret=$?
done
echo Instance is ready.
#Create swap
echo Adding swap file
sudo fallocate -l 16G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
echo Swap created.
#echo "deb http://deb.debian.org/debian/ buster main" | sudo tee /etc/apt/sources.list
#echo "deb http://deb.debian.org/debian/ buster main contrib non-free" | sudo tee -a /etc/apt/sources.list
sudo apt-get -q --yes install build-essential default-jdk git ninja-build gnupg software-properties-common snapd make g++-multilib g++ g++-8 gcc gcc-8 libopenblas-base libopenblas-dev
ret=$?
while [ $ret -ne 0 ];
do
echo Waiting for instance to be ready ...
sleep 20s
sudo apt-get -q --yes install build-essential default-jdk git ninja-build gnupg software-properties-common snapd make g++-multilib g++ g++-8 gcc gcc-8 libopenblas-base libopenblas-dev
ret=$?
done
#sudo snap install core
#sudo snap install cmake --classic
wget https://github.com/Kitware/CMake/releases/download/v3.20.4/cmake-3.20.4-linux-x86_64.sh
chmod a+x cmake-3.20.4-linux-x86_64.sh
sudo sh cmake-3.20.4-linux-x86_64.sh --skip-license
echo "supersede domain-name-servers 172.31.0.2, 8.8.8.8" | sudo tee -a /etc/dhcp/dhclient.conf
echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf

5
gradle.properties
View File

@ -31,7 +31,10 @@ systemProp.org.gradle.internal.publish.checksums.insecure=true
# Specifies the JVM arguments used for the daemon process.
# The setting is particularly useful for tweaking memory settings.
org.gradle.jvmargs=-Xmx8192m
#for whatever reason we had to add MaxMetaspace and file encoding = utf8, gradle crashed otherwise
org.gradle.jvmargs=-Xmx8192m -XX:MaxMetaspaceSize=768m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 -XX:ErrorFile=/var/log/java/hs_err_pid%p.log
# When configured, Gradle will run in incubating parallel mode.
# This option should only be used with decoupled projects. More details, visit

67
jenkinsfile
View File

@ -1,8 +1,23 @@
pipeline {
agent none
stages {
stage('prep-build-environment-linux') {
agent {
label 'linux'
}
steps {
checkout scm
//sh 'nvidia-smi'
//sh 'nvcc --version'
sh 'gcc --version'
sh 'cmake --version'
sh 'sh ./gradlew --version'
}
}
stage('prep-build-environment-linux-cuda') {
agent {
dockerfile {
filename 'Dockerfile'
dir '.docker'
@ -11,34 +26,60 @@ pipeline {
args '--gpus all'
}
}
stages {
stage('prep-build-environment') {
steps {
//git poll: false, url: 'http://bru3-pc.fritz.box:3000/Brutex/ci-deeplearning4j.git'
checkout scm
sh 'nvidia-smi'
sh 'nvcc --version'
sh 'gcc --version'
sh 'cmake --version'
checkout scm
sh 'sh ./gradlew --version'
}
}
stage('build') {
environment {
MAVEN = credentials('Internal Archiva')
OSSRH = credentials('OSSRH')
}
parallel {
stage('build-linux-cuda') {
agent {
dockerfile {
filename 'Dockerfile'
dir '.docker'
label 'linuxdocker'
//additionalBuildArgs '--build-arg version=1.0.2'
args '--gpus all'
}
}
steps {
sh 'chmod u+x ./gradlew && ls -la'
sh './gradlew --version'
withGradle {
sh './gradlew build --stacktrace --no-watch-fs -x test -PCAVIS_CHIP=cuda -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
sh 'sh ./gradlew publish --stacktrace -x test -PCAVIS_CHIP=cuda -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
}
//stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
}
}
stage('build-linux-cpu') {
agent {
label 'linux'
}
steps {
withGradle {
sh 'sh ./gradlew publish --stacktrace -x test -PCAVIS_CHIP=cpu -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
}
//stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cpu-build'
}
}
}
}
/* stage('collect-release') {
agent {
label 'linux'
}
steps {
//unstash 'cuda-build'
//unstash 'cpu-build'
}
}
*/
}
}