gan example

Signed-off-by: brian <brian@brutex.de>

.docker/Dockerfile

@@ -1,8 +1,15 @@
-FROM nvidia/cuda:11.4.0-cudnn8-devel-ubuntu20.04
+FROM nvidia/cuda:11.4.3-cudnn8-devel-ubuntu20.04
 
 RUN apt-get update &&  \
     DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-11-jdk wget build-essential checkinstall zlib1g-dev libssl-dev git
-RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2.tar.gz && \
+#Build cmake version from source \
-    tar -xvf cmake-3.24.2.tar.gz && cd cmake-3.24.2 && \
+#RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2.tar.gz && \
-    ./bootstrap && make && make install
+#    tar -xvf cmake-3.24.2.tar.gz && cd cmake-3.24.2 && \
+#    ./bootstrap && make && make install
+RUN wget -nv https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.sh && \
+    mkdir /opt/cmake && sh ./cmake-3.24.2-linux-x86_64.sh --skip-license --prefix=/opt/cmake && ln -s /opt/cmake/bin/cmake /usr/bin/cmake && \
+    rm cmake-3.24.2-linux-x86_64.sh
+
+
+RUN echo "nameserver 8.8.8.8" >> /etc/resolv.conf
 

.gitignore

@@ -36,6 +36,8 @@ pom.xml.versionsBackup
 pom.xml.next
 release.properties
 *dependency-reduced-pom.xml
+**/build/*
+.gradle/*
 
 # Specific for Nd4j
 *.md5
@@ -50,12 +52,12 @@ release.properties
 *.dylib
 .vs/
 .vscode/
-nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/bin
+.old/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/bin
-nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/test/resources/writeNumpy.csv
+.old/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/test/resources/writeNumpy.csv
-nd4j/nd4j-backends/nd4j-tests/src/test/resources/tf_graphs/examples/**/data-all*
+.old/nd4j/nd4j-backends/nd4j-tests/src/test/resources/tf_graphs/examples/**/data-all*
-nd4j/nd4j-backends/nd4j-tests/src/test/resources/tf_graphs/examples/**/checkpoint
+.old/nd4j/nd4j-backends/nd4j-tests/src/test/resources/tf_graphs/examples/**/checkpoint
-nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/onnx/
+.old/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/onnx/
-nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/tensorflow/
+.old/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/tensorflow/
 
 doc_sources/
 doc_sources_*
@@ -67,8 +69,8 @@ venv/
 venv2/
 
 # Ignore the nd4j files that are created by javacpp at build to stop merge conflicts
-nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpu.java
+.old/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpu.java
-nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativeblas/Nd4jCuda.java
+.old/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativeblas/Nd4jCuda.java
 
 # Ignore meld temp files
 *.orig
@@ -82,3 +84,15 @@ bruai4j-native-common/cmake*
 *.dll
 /bruai4j-native/bruai4j-native-common/blasbuild/
 /bruai4j-native/bruai4j-native-common/build/
+/cavis-native/cavis-native-lib/blasbuild/
+/cavis-dnn/cavis-dnn-core/build/reports/tests/cudaTest/classes/org.deeplearning4j.gradientcheck.AttentionLayerTest.html
+/cavis-dnn/cavis-dnn-core/build/reports/tests/cudaTest/css/base-style.css
+/cavis-dnn/cavis-dnn-core/build/reports/tests/cudaTest/css/style.css
+/cavis-dnn/cavis-dnn-core/build/reports/tests/cudaTest/js/report.js
+/cavis-dnn/cavis-dnn-core/build/reports/tests/cudaTest/packages/org.deeplearning4j.gradientcheck.html
+/cavis-dnn/cavis-dnn-core/build/reports/tests/cudaTest/index.html
+/cavis-dnn/cavis-dnn-core/build/resources/main/iris.dat
+/cavis-dnn/cavis-dnn-core/build/resources/test/junit-platform.properties
+/cavis-dnn/cavis-dnn-core/build/resources/test/logback-test.xml
+/cavis-dnn/cavis-dnn-core/build/test-results/cudaTest/TEST-org.deeplearning4j.gradientcheck.AttentionLayerTest.xml
+/cavis-dnn/cavis-dnn-core/build/tmp/jar/MANIFEST.MF

.jenkins/linux-x86_64-cpu-build.jenkinsfile

@@ -0,0 +1,82 @@
+/*
+ *
+ *    ******************************************************************************
+ *    *
+ *    * This program and the accompanying materials are made available under the
+ *    * terms of the Apache License, Version 2.0 which is available at
+ *    * https://www.apache.org/licenses/LICENSE-2.0.
+ *    *
+ *    *  See the NOTICE file distributed with this work for additional
+ *    *  information regarding copyright ownership.
+ *    * Unless required by applicable law or agreed to in writing, software
+ *    * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    * License for the specific language governing permissions and limitations
+ *    * under the License.
+ *    *
+ *    * SPDX-License-Identifier: Apache-2.0
+ *    *****************************************************************************
+ *
+ */
+
+pipeline {
+    agent {
+            label 'linux'
+    }
+
+    stages {
+        stage('prep-build-environment-linux-cpu') {
+                    steps {
+                        checkout scm
+                        sh 'gcc --version'
+                        sh 'cmake --version'
+                        sh 'sh ./gradlew --version'
+                    }
+                }
+        stage('build-linux-cpu') {
+            environment {
+                        MAVEN = credentials('Internal_Archiva')
+                        OSSRH = credentials('OSSRH')
+            }
+
+            steps {
+                withGradle {
+                    sh 'sh ./gradlew build --stacktrace -x test -PCAVIS_CHIP=cpu \
+                                -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
+                                -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
+                }
+                //stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
+            }
+        }
+        /*stage('test-linux-cpu') {
+            environment {
+                MAVEN = credentials('Internal Archiva')
+                OSSRH = credentials('OSSRH')
+            }
+
+            steps {
+                withGradle {
+                    sh 'sh ./gradlew test --stacktrace -PexcludeTests=\'long-running,performance\' -PCAVIS_CHIP=cpu \
+                                -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
+                                -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
+                }
+                //stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
+            }
+        }*/
+        stage('publish-linux-cpu') {
+            environment {
+                MAVEN = credentials('Internal Archiva')
+                OSSRH = credentials('OSSRH')
+            }
+
+            steps {
+                withGradle {
+                    sh 'sh ./gradlew publish --stacktrace -PCAVIS_CHIP=cpu \
+                                -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
+                                -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
+                }
+                //stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
+            }
+        }
+    }
+}

.jenkins/linux-x86_64-cuda-build.jenkinsfile

@@ -0,0 +1,62 @@
+/*
+ *
+ *    ******************************************************************************
+ *    *
+ *    * This program and the accompanying materials are made available under the
+ *    * terms of the Apache License, Version 2.0 which is available at
+ *    * https://www.apache.org/licenses/LICENSE-2.0.
+ *    *
+ *    *  See the NOTICE file distributed with this work for additional
+ *    *  information regarding copyright ownership.
+ *    * Unless required by applicable law or agreed to in writing, software
+ *    * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    * License for the specific language governing permissions and limitations
+ *    * under the License.
+ *    *
+ *    * SPDX-License-Identifier: Apache-2.0
+ *    *****************************************************************************
+ *
+ */
+
+pipeline {
+    agent {
+      /*  dockerfile {
+            filename 'Dockerfile'
+            dir '.docker'
+            label 'linux && cuda'
+            //additionalBuildArgs  '--build-arg version=1.0.2'
+            //args '--gpus all' --needed for test only, you can build without GPU
+        }
+        */
+       label 'linux && cuda'
+    }
+
+    stages {
+        stage('prep-build-environment-linux-cuda') {
+                    steps {
+                        checkout scm
+                        //sh 'nvidia-smi'
+                        sh 'nvcc --version'
+                        sh 'gcc --version'
+                        sh 'cmake --version'
+                        sh 'sh ./gradlew --version'
+                    }
+                }
+        stage('build-linux-cuda') {
+            environment {
+                        MAVEN = credentials('Internal_Archiva')
+                        OSSRH = credentials('OSSRH')
+            }
+
+            steps {
+                withGradle {
+                    sh 'sh ./gradlew build --stacktrace  -PCAVIS_CHIP=cuda \
+                                -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
+                                -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
+                }
+                //stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
+            }
+        }
+    }
+}

.jenkins/linux-x86_64-docker-all-publish.jenkinsfile

@@ -0,0 +1,66 @@
+/*
+ *
+ *    ******************************************************************************
+ *    *
+ *    * This program and the accompanying materials are made available under the
+ *    * terms of the Apache License, Version 2.0 which is available at
+ *    * https://www.apache.org/licenses/LICENSE-2.0.
+ *    *
+ *    *  See the NOTICE file distributed with this work for additional
+ *    *  information regarding copyright ownership.
+ *    * Unless required by applicable law or agreed to in writing, software
+ *    * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    * License for the specific language governing permissions and limitations
+ *    * under the License.
+ *    *
+ *    * SPDX-License-Identifier: Apache-2.0
+ *    *****************************************************************************
+ *
+ */
+
+pipeline {
+    agent {
+        dockerfile {
+            filename 'Dockerfile'
+            dir '.docker'
+            label 'linux && docker && cuda'
+            //additionalBuildArgs  '--build-arg version=1.0.2'
+            //args '--gpus all' --needed for test only, you can build without GPU
+        }
+    }
+
+    stages {
+        stage("Build all chip") {
+            parallel {
+
+                stage('prep-build-environment-linux-cuda') {
+
+                    steps {
+                        checkout scm
+                        //sh 'nvidia-smi'
+                        sh 'nvcc --version'
+                        sh 'gcc --version'
+                        sh 'cmake --version'
+                        sh 'sh ./gradlew --version'
+                    }
+                }
+                stage('build-linux-cuda') {
+                    environment {
+                        MAVEN = credentials('Internal_Archiva')
+                        OSSRH = credentials('OSSRH')
+                    }
+
+                    steps {
+                        withGradle {
+                            sh 'sh ./gradlew build --stacktrace -x test -PCAVIS_CHIP=cuda \
+                                    -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
+                                    -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
+                        }
+                        //stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
+                    }
+                }
+            }
+        }
+    }
+}

.jenkins/linux-x86_64-docker-cpu-build.jenkinsfile

@@ -0,0 +1,88 @@
+/*
+ *
+ *    ******************************************************************************
+ *    *
+ *    * This program and the accompanying materials are made available under the
+ *    * terms of the Apache License, Version 2.0 which is available at
+ *    * https://www.apache.org/licenses/LICENSE-2.0.
+ *    *
+ *    *  See the NOTICE file distributed with this work for additional
+ *    *  information regarding copyright ownership.
+ *    * Unless required by applicable law or agreed to in writing, software
+ *    * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    * License for the specific language governing permissions and limitations
+ *    * under the License.
+ *    *
+ *    * SPDX-License-Identifier: Apache-2.0
+ *    *****************************************************************************
+ *
+ */
+
+pipeline {
+    agent {
+        dockerfile {
+            filename 'Dockerfile'
+            dir '.docker'
+            label 'linux && docker'
+            //additionalBuildArgs  '--build-arg version=1.0.2'
+            //args '--gpus all'
+        }
+    }
+
+    stages {
+        stage('prep-build-environment-linux-cpu') {
+                    steps {
+                        checkout scm
+                        sh 'gcc --version'
+                        sh 'cmake --version'
+                        sh 'sh ./gradlew --version'
+                    }
+                }
+        stage('build-linux-cpu') {
+            environment {
+                        MAVEN = credentials('Internal_Archiva')
+                        OSSRH = credentials('OSSRH')
+            }
+
+            steps {
+                withGradle {
+                    sh 'sh ./gradlew build --stacktrace -x test -PCAVIS_CHIP=cpu \
+                                -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
+                                -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
+                }
+                //stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
+            }
+        }
+        stage('test-linux-cpu') {
+            environment {
+                MAVEN = credentials('Internal Archiva')
+                OSSRH = credentials('OSSRH')
+            }
+
+            steps {
+                withGradle {
+                    //sh 'sh ./gradlew test --stacktrace -PCAVIS_CHIP=cpu \
+                    //            -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
+                    //            -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
+                }
+                //stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
+            }
+        }
+        stage('publish-linux-cpu') {
+            environment {
+                MAVEN = credentials('Internal Archiva')
+                OSSRH = credentials('OSSRH')
+            }
+
+            steps {
+                withGradle {
+                    sh 'sh ./gradlew publish --stacktrace -PCAVIS_CHIP=cpu \
+                                -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
+                                -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
+                }
+                //stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
+            }
+        }
+    }
+}

.jenkins/linux-x86_64-docker-cpu-publish.jenkinsfile

@@ -0,0 +1,49 @@
+/*
+ *
+ *    ******************************************************************************
+ *    *
+ *    * This program and the accompanying materials are made available under the
+ *    * terms of the Apache License, Version 2.0 which is available at
+ *    * https://www.apache.org/licenses/LICENSE-2.0.
+ *    *
+ *    *  See the NOTICE file distributed with this work for additional
+ *    *  information regarding copyright ownership.
+ *    * Unless required by applicable law or agreed to in writing, software
+ *    * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    * License for the specific language governing permissions and limitations
+ *    * under the License.
+ *    *
+ *    * SPDX-License-Identifier: Apache-2.0
+ *    *****************************************************************************
+ *
+ */
+
+pipeline {
+    agent {
+        dockerfile {
+            filename 'Dockerfile'
+            dir '.docker'
+            label 'linux && docker'
+            //additionalBuildArgs  '--build-arg version=1.0.2'
+            //args '--gpus all'
+        }
+    }
+
+    stages {
+        stage('publish-linux-cpu') {
+            environment {
+                MAVEN = credentials('Internal_Archiva')
+                OSSRH = credentials('OSSRH')
+            }
+
+            steps {
+                withGradle {
+                    sh 'sh ./gradlew publish -x test -PCAVIS_CHIP=cpu \
+                                -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
+                                -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
+                }
+            }
+        }
+    }
+}

.jenkins/linux-x86_64-docker-cuda-build.jenkinsfile

@@ -20,14 +20,13 @@
  */
 
 pipeline {
-
     agent {
         dockerfile {
             filename 'Dockerfile'
             dir '.docker'
-            label 'linuxdocker'
+            label 'linux && docker && cuda'
             //additionalBuildArgs  '--build-arg version=1.0.2'
-            args '--gpus all'
+            //args '--gpus all' --needed for test only, you can build without GPU
         }
     }
 
@@ -35,7 +34,7 @@ pipeline {
         stage('prep-build-environment-linux-cuda') {
                     steps {
                         checkout scm
-                        sh 'nvidia-smi'
+                        //sh 'nvidia-smi'
                         sh 'nvcc --version'
                         sh 'gcc --version'
                         sh 'cmake --version'
@@ -44,19 +43,33 @@ pipeline {
                 }
         stage('build-linux-cuda') {
             environment {
-                        MAVEN = credentials('Internal Archiva')
+                        MAVEN = credentials('Internal_Archiva')
                         OSSRH = credentials('OSSRH')
             }
 
             steps {
                 withGradle {
-                    sh 'sh ./gradlew publish --stacktrace -x test -PCAVIS_CHIP=cuda \
+                    sh 'sh ./gradlew build --stacktrace -x test -PCAVIS_CHIP=cuda \
                                 -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
                                 -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
                 }
                 //stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
             }
+        }
+        stage('test-linux-cuda') {
+            environment {
+                MAVEN = credentials('Internal_Archiva')
+                OSSRH = credentials('OSSRH')
+            }
 
+            steps {
+                withGradle {
+                    sh 'sh ./gradlew test --stacktrace -PexcludeTests=\'long-running,performance\' -Pskip-native=true -PCAVIS_CHIP=cuda \
+                                -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
+                                -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
+                }
+                //stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
+            }
         }
     }
 }

.jenkins/wsl-x86_64-docker-cpu-build.jenkinsfile

@@ -0,0 +1,58 @@
+/*
+ *
+ *    ******************************************************************************
+ *    *
+ *    * This program and the accompanying materials are made available under the
+ *    * terms of the Apache License, Version 2.0 which is available at
+ *    * https://www.apache.org/licenses/LICENSE-2.0.
+ *    *
+ *    *  See the NOTICE file distributed with this work for additional
+ *    *  information regarding copyright ownership.
+ *    * Unless required by applicable law or agreed to in writing, software
+ *    * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    * License for the specific language governing permissions and limitations
+ *    * under the License.
+ *    *
+ *    * SPDX-License-Identifier: Apache-2.0
+ *    *****************************************************************************
+ *
+ */
+
+pipeline {
+    agent {
+        dockerfile {
+            filename 'Dockerfile'
+            dir '.docker'
+            label 'WSL-docker'
+            //additionalBuildArgs  '--build-arg version=1.0.2'
+            //args '--gpus all'
+        }
+    }
+
+    stages {
+        stage('prep-build-environment-linux-cpu') {
+                    steps {
+                        checkout scm
+                        sh 'gcc --version'
+                        sh 'cmake --version'
+                        sh 'sh ./gradlew --version'
+                    }
+                }
+        stage('build-linux-cpu') {
+            environment {
+                        MAVEN = credentials('Internal_Archiva')
+                        OSSRH = credentials('OSSRH')
+            }
+
+            steps {
+                withGradle {
+                    sh 'sh ./gradlew publish --stacktrace -x test -PCAVIS_CHIP=cpu \
+                                -Pmavenuser=$MAVEN_USR -Pmavenpass=$MAVEN_PSW \
+                                -PossrhUsername=$OSSRH_USR -PossrhPassword=$OSSRH_PSW'
+                }
+                //stash includes: '/cavis-native/cavis-native-lib/build/lib/*.jar', name: 'cuda-build'
+            }
+        }
+    }
+}