import org.gradle.api.publish.maven.internal.publisher.MavenRemotePublisher import org.gradle.language.nativeplatform.internal.Dimensions buildscript { /**************************************************************************** * Establish Visual Studio configuration environment for Windows native builds * NOTE: vsconfig.gradle path is relative to each GPL project module ****************************************************************************/ apply from: "../../vsconfig.gradle" apply from: "../../chooseBackend.gradle" ext { host_cores = Runtime.getRuntime().availableProcessors() buildHelper = "-mingw" javacppPlatform = osdetector.classifier println "Building on ${host_cores} CPU cores." println "JavaCPP target plattform is ${javacppPlatform}" if (project.hasProperty("CAVIS_AVX_EXTENSION")) { avxExtension = project.getProperty("CAVIS_AVX_EXTENSION").toLowerCase() println "Bulding with ${avxExtension}" } else { avxExtension = "avx2" logger.quiet("No AVX CPU extension selected (avx2|avx512). Building with default 'avx2'") } javacppPlatformExtension = "-${avxExtension}".toString() getBuildPlatform = { String chip, Task tsk -> def pf ="" if(chip.equals("cuda")) { pf = osdetector.classifier } else { if(osdetector.os.equals("windows")) { pf = "${osdetector.classifier}-mingw" } else { pf = "${osdetector.classifier}" } } logger.info("Setting properties for task '{}' to '{}'", tsk.getName(), pf) return pf } } dependencies { classpath platform(project(":cavis-common-platform")) classpath group: "org.bytedeco", name: "openblas" classpath group: "org.bytedeco", name: "openblas", classifier: "${javacppPlatform}" classpath group: "org.bytedeco", name:"mkl-dnn" classpath group: "org.bytedeco", name:"mkl-dnn", classifier: "${javacppPlatform}" classpath group: "org.bytedeco", name: "javacpp" classpath group: "org.bytedeco", name: "javacpp", classifier: "${javacppPlatform}" } } plugins { id 'java-library' id 'org.bytedeco.gradle-javacpp-build' version "1.5.7" id 'maven-publish' id 'signing' } chipList.each {thisChip -> sourceSets.register("${thisChip}Support") { java { srcDirs = ['src/main/java', "${buildDir}/generated/sources/javacpp/${thisChip}//${javacppPlatform}${javacppPlatformExtension}/"] include "org/nd4j/nativeblas/${thisChip}/Nd4j${thisChip.capitalize()}Helper.java" include "org/nd4j/nativeblas/${thisChip}/Nd4j${thisChip.capitalize()}Presets.java" include "org/nd4j/nativeblas/Nd4j${thisChip.capitalize()}.java" } it.compiledBy("javacpp${thisChip.capitalize()}SupportBuildCommand", "javacpp${thisChip.capitalize()}SupportBuildCompiler") } } if(osdetector.os.startsWith("windows")) { sourceSets { main { java { srcDirs = ['src/main/java'] include 'org/nd4j/nativeblas/Dummy.java' } } } } java { chipList.each {thisChip -> registerFeature("${thisChip}Support") { usingSourceSet(sourceSets.findByName("${thisChip}Support")) //withJavadocJar() //withSourcesJar() } } } /* configurations.each(s -> { println "Configurations: " + s.name + " " + s.artifacts.each( x -> { println x.getFile().getName()}) }) */ dependencies { api platform(project(':cavis-common-platform')) api "org.bytedeco:javacpp" if(withCuda()) { cudaSupportImplementation platform(project(':cavis-common-platform')) cudaSupportImplementation project(":cavis-dnn:cavis-dnn-api") cudaSupportImplementation project(":cavis-dnn:cavis-dnn-common") cudaSupportImplementation project(":cavis-native:cavis-native-blas") cudaSupportImplementation project(":cavis-native:cavis-native-common") cudaSupportImplementation "commons-io:commons-io" cudaSupportImplementation group: "org.bytedeco", name: "openblas" cudaSupportImplementation group: "org.bytedeco", name: "openblas", classifier: "${javacppPlatform}" cudaSupportImplementation group: "org.bytedeco", name: "cuda" cudaSupportImplementation group: "org.bytedeco", name: "cuda", classifier: "${javacppPlatform}" cudaSupportImplementation "org.apache.logging.log4j:log4j-core:2.17.0" cudaSupportImplementation "com.google.guava:guava:14.0.1" cudaSupportImplementation "org.apache.commons:commons-lang3" cudaSupportImplementation "org.apache.commons:commons-math3" cudaSupportImplementation "com.google.flatbuffers:flatbuffers-java" cudaSupportImplementation 'javax.mail:javax.mail-api:1.6.2' } if(withCpu()) { cpuSupportImplementation platform(project(':cavis-common-platform')) cpuSupportImplementation project(":cavis-dnn:cavis-dnn-api") cpuSupportImplementation project(":cavis-dnn:cavis-dnn-common") cpuSupportImplementation project(":cavis-native:cavis-native-blas") cpuSupportImplementation project(":cavis-native:cavis-native-common") cpuSupportImplementation "commons-io:commons-io" cpuSupportImplementation group: "org.bytedeco", name: "openblas" cpuSupportImplementation group: "org.bytedeco", name: "openblas", classifier: "${javacppPlatform}" cpuSupportImplementation group: "org.bytedeco", name: "opencv" cpuSupportImplementation group: "org.bytedeco", name: "opencv", classifier: "${javacppPlatform}" cpuSupportImplementation "org.apache.logging.log4j:log4j-core:2.17.0" cpuSupportImplementation "com.google.guava:guava:14.0.1" cpuSupportImplementation "org.apache.commons:commons-lang3" cpuSupportImplementation "org.apache.commons:commons-math3" cpuSupportImplementation "com.google.flatbuffers:flatbuffers-java" cpuSupportImplementation 'javax.mail:javax.mail-api:1.6.2' } implementation projects.cavisDnn.cavisDnnApi implementation projects.cavisDnn.cavisDnnCommon implementation project(":cavis-native:cavis-native-blas") implementation project(":cavis-native:cavis-native-common") implementation "commons-io:commons-io" implementation "org.bytedeco:openblas" implementation group: "org.bytedeco", name: "openblas", classifier: "${javacppPlatform}" implementation "org.apache.logging.log4j:log4j-core" implementation "com.google.guava:guava:14.0.1" implementation "org.apache.commons:commons-lang3" implementation "org.apache.commons:commons-math3" implementation "com.google.flatbuffers:flatbuffers-java" //javacppPlatform project(":cavis-native:cavis-native-blas") } clean { doFirst { delete "${projectDir}/build" delete "${projectDir}/src/main/include/config.h" chipList.each { delete "${projectDir}/blasbuild/${it}" } } } task deepClean(type: Delete) { dependsOn clean doFirst { delete "$projectDir}/blasbuild" } } tasks.withType(org.bytedeco.gradle.javacpp.BuildTask) { buildResource = [ "/org/bytedeco/openblas/${javacppPlatform}/", "/org/bytedeco/mkldnn/${javacppPlatform}/"] includeResource = ["/org/bytedeco/openblas/${javacppPlatform}/include/"] linkResource = ["/org/bytedeco/openblas/${javacppPlatform}/", "/org/bytedeco/openblas/${javacppPlatform}/lib/"] //buildPath = [ org.bytedeco.javacpp.Loader.getCacheDir() ] } // Disable the standard javacpp generated tasks and use own // versions below. This allows to build for each variant [javacppBuildParser, javacppBuildCommand, javacppCompileJava, javacppBuildCompiler].each { it.enabled false; } chipList.each { thisChip -> // 1) //Run the C++ compile first tasks.register("javacpp${thisChip.capitalize()}SupportBuildCommand", org.bytedeco.gradle.javacpp.BuildTask) { if (project.hasProperty("skip-native") && project.getProperty("skip-native").equals("true")) { enabled = false } properties = getBuildPlatform( thisChip, it ) includePath = ["${projectDir}/src/main/cpp/blas/", "${projectDir}/blasbuild/${thisChip}/${avxExtension}/src/main/include/", "${projectDir}/blasbuild/${thisChip}/${avxExtension}/flatbuffers-src/include", "${projectDir}/blasbuild/${thisChip}/${avxExtension}/cpu_features-src/include", "${projectDir}/blasbuild/${thisChip}/${avxExtension}/mkldnn-src/include"] linkPath = ["${projectDir}/blasbuild/${thisChip}/${avxExtension}/output"] //No idea why this is here, but it looks like even for the javacppBuildCommand task, //there is a javacpp Loader actively determining platform etc. classOrPackageNames = ["org.nd4j.nativeblas.${thisChip}.Nd4j${thisChip.capitalize()}Presets"] workingDirectory = projectDir //if the classpath is not set here, the javacpp classloader starts to look around //everywhere and causes java.io.IOExceptions: because files is being used by another process classPath = [:] classPath += ["${buildDir}/classes/java/${thisChip}Support/"] //classPath += ["${buildDir}/classes/java/main/"] /* Get VCVARS in case we want to build CUDA * MinGW64 g++ on MSYS is used otherwise */ if (thisChip.equals('cuda') && osdetector.os.startsWith("win") && !VISUAL_STUDIO_INSTALL_DIR.isEmpty()) { def proc = ["cmd.exe", "/c", "${VISUAL_STUDIO_VCVARS_CMD} > nul && set"].execute() it.environmentVariables = it.environmentVariables ?: [:] def lines = proc.text.split("\\r?\\n") for (def line in lines) { if (line.contains("=")) { def parts = line.split("=") it.environmentVariables.put(parts[0], parts[1]) } } } if (thisChip.equals('cuda') && osdetector.os.startsWith("windows")) { //cuDNN requires CUDA it.buildCommand = ['sh', 'buildnativeoperations.sh', '-V', '--build-type', 'release', '--chip', thisChip, '--plattform', 'x86_64', '--chip-extension', avxExtension, '-j', "${host_cores}", // '--helper', 'mkldnn', '--helper', 'cudnn'] } else if (thisChip.equals('cuda') && osdetector.os.startsWith("linux")) { //cuDNN requires CUDA it.buildCommand = ['bash', 'buildnativeoperations.sh', '-V', '--build-type', 'release', '--chip', thisChip, '--plattform', 'x86_64', '--chip-extension', avxExtension, '-j', "${host_cores}", // '--helper', 'mkldnn', '--helper', 'cudnn'] } else { it.buildCommand = ['bash', 'buildnativeoperations.sh', '-V', '--build-type', 'release', '--chip', thisChip, '--plattform', 'x86_64', '--chip-extension', avxExtension, '-j', "${host_cores}", '--helper', 'mkldnn'] } } //Create a task to (pre)compile the java presets (required for javacppBuildParser) tasks.register("compile${thisChip.capitalize()}Support", JavaCompile) { def thisSS = sourceSets.findByName("${thisChip}Support") it.source = thisSS.allSource it.classpath = thisSS.compileClasspath it.destinationDirectory = file("${buildDir}/classes/java/${thisChip}Support/") } //Run the parser on the InfoMap in Nd4j$ChipPresets and listed header files in @Platform //Generates Nd4jCpu.java and/ or Nd4jCuda.java Java JNI code tasks.register("javacpp${thisChip.capitalize()}SupportBuildParser", org.bytedeco.gradle.javacpp.BuildTask) { if (project.hasProperty("skip-native") && project.getProperty("skip-native").equals("true")) { enabled = false } dependsOn "compile${thisChip.capitalize()}Support" includePath = ["${projectDir}/src/main/cpp/blas/", "${projectDir}/blasbuild/${thisChip}/${avxExtension}/src/main/include/", "${projectDir}/blasbuild/${thisChip}/${avxExtension}/flatbuffers-src/include", "${projectDir}/blasbuild/${thisChip}/${avxExtension}/cpu_features-src/include", "${projectDir}/blasbuild/${thisChip}/${avxExtension}/mkldnn-src/include"] classOrPackageNames = ["org.nd4j.nativeblas.${thisChip}.Nd4j${thisChip.capitalize()}Presets"] outputDirectory = file("${buildDir}/generated/sources/javacpp/${thisChip}/${javacppPlatform}${javacppPlatformExtension}/") classPath = sourceSets.getByName("${thisChip}Support").getRuntimeClasspath() classPath += ["${buildDir}/classes/java/${thisChip}Support/"] } // Generates jnijavacpp.cpp and jniNativeLibrary.cpp, compiles and links it tasks.register("javacpp${thisChip.capitalize()}SupportBuildCompiler", org.bytedeco.gradle.javacpp.BuildTask) { if (project.hasProperty("skip-native") && project.getProperty("skip-native").equals("true")) { enabled = false } def thisTask = (org.bytedeco.gradle.javacpp.BuildTask) it thisTask.dependsOn = ["javacpp${thisChip.capitalize()}SupportBuildParser"] thisTask.linkPath = ["${projectDir}/blasbuild/${thisChip}/${avxExtension}/output"] thisTask.includePath = ["${projectDir}/src/main/cpp/blas/", "${projectDir}/blasbuild/${thisChip}/${avxExtension}/src/main/include/", "${projectDir}/blasbuild/${thisChip}/${avxExtension}/flatbuffers-src/include", "${projectDir}/blasbuild/${thisChip}/${avxExtension}/cpu_features-src/include", "${projectDir}/blasbuild/${thisChip}/${avxExtension}/mkldnn-src/include"] thisTask.properties = getBuildPlatform( thisChip, thisTask ) if(thisChip.equals('cuda') && osdetector.os.startsWith("win") && !VISUAL_STUDIO_INSTALL_DIR.isEmpty()) { def proc = ["cmd.exe", "/c", "${VISUAL_STUDIO_VCVARS_CMD} > nul && where.exe cl.exe"].execute() def outp = proc.text def cl = outp.replace("\\", "\\\\").trim() def currentCompiler = "" doFirst{ currentCompiler = System.getProperty("org.bytedeco.javacpp.platform.compiler") System.setProperty("org.bytedeco.javacpp.platform.compiler", cl) logger.quiet("Task ${thisTask.name} overrides compiler '${currentCompiler}' with '${cl}'.") } doLast { //restore compiler System.setProperty("org.bytedeco.javacpp.platform.compiler", currentCompiler ?: "") }//System.setProperty("org.bytedeco.javacpp.platform.compiler", cl) //System.setProperty("org.bytedeco.javacpp.platform.compiler.cpp11", cl) proc = ["cmd.exe", "/c", "${VISUAL_STUDIO_VCVARS_CMD} > nul && set"].execute() thisTask.environmentVariables = thisTask.environmentVariables ?: [:] def lines = proc.text.split("\\r?\\n") for (def line in lines) { if (line.contains("=")) { def parts = line.split("=") thisTask.environmentVariables.put(parts[0], parts[1]) } } } else { //System.setProperty("org.bytedeco.javacpp.platform.compiler", "g++") } thisTask.buildPath = ["$buildDir/generated/sources/javacpp/${thisChip}/${javacppPlatform}${javacppPlatformExtension}/"] thisTask.copyLibs = true thisTask.deleteJniFiles(false) outputName = "jnind4j${thisChip}" thisTask.outputDirectory = file("$buildDir/generated/sources/javacpp/${thisChip}/${javacppPlatform}${javacppPlatformExtension}/") thisTask.classOrPackageNames= ["org.nd4j.nativeblas.Nd4j${thisChip.capitalize()}"] thisTask.configDirectory = file("${buildDir}/classes/java/${thisChip}Support/META-INF/native-image/${javacppPlatform}") //Need to set the classpath, so that external jars from the dependency list are resolved by the ClassLoader as well thisTask.classPath = [:] thisTask.classPath = ["${buildDir}/classes/java/${thisChip}Support"] thisTask.classPath += sourceSets.findByName("${thisChip}Support").runtimeClasspath //sourceSets.findByName("${thisChip}Support").runtimeClasspath.each{ s -> // thisTask.classPath += s //} } // Generates jnijavacpp.cpp and jniNativeLibrary.cpp, compiles and links it tasks.getByName("${thisChip}SupportJar") { Jar thisTask -> dependsOn "javacpp${thisChip.capitalize()}SupportBuildCompiler" dependsOn "javacpp${thisChip.capitalize()}SupportBuildCommand" //it.from sourceSets.getByName("${thisChip}Support").getOutput() def spec = copySpec { from(tasks.getByName("javacpp${thisChip.capitalize()}SupportBuildCompiler")) { exclude { f -> def exclude = f.file.isDirectory() if(exclude) { logger.info("${thisTask.name}: excluding '${f}'") } else { logger.info("${thisTask.name}: including '${f}'") } return exclude } into "${javacppPlatform}/" //we need it in a platform, that javacpp Loader understands } from(sourceSets.getByName("${thisChip}Support").getOutput()) { } duplicatesStrategy DuplicatesStrategy.EXCLUDE } thisTask.with spec thisTask.archiveClassifier = "${javacppPlatform}${javacppPlatformExtension}-${thisChip}" } //tasks.getByName("${thisChip}SupportJar").dependsOn("javacpp${thisChip.capitalize()}SupportJar") } //Before we can compile the whole java part, we //need to generate the Nd4jXXX.java files first chipList.each { thisChip -> tasks.findByName("compile${thisChip.capitalize()}SupportJava").each { t -> t.dependsOn "javacpp${thisChip.capitalize()}SupportBuildParser" } } tasks.withType(JavaCompile) { // options.setCompilerArgs(Arrays.asList("-Xlint:unchecked")) } tasks.withType(Javadoc) { options.addStringOption('Xdoclint:none', '-quiet') } jar { manifest { attributes 'Class-Path': configurations.runtimeClasspath.collect { it.getName() }.join(' '), 'Implementation-Title': 'Brutex AI - Native Components', 'Implementation-Vendor': 'Brutex Network', 'Implementation-Version': archiveVersion, 'Specification-Title': 'Brutex AI - Native Components', 'Specification-Vendor': 'Brutex Network', 'Specification-Version': archiveVersion } //archiveClassifier = "${javacppPlatform}${javacppPlatformExtension}-${chip}" } javadoc { dependsOn "javacppPomProperties" failOnError = false //options.links = ['http://bytedeco.org/javacpp/apidocs'] options.addStringOption('Xdoclint:none', '-quiet') //options.JFlags = ["-Xdoclint:none"] } if(! osdetector.os.startsWith("windows")) { tasks.getByName("publish") { enabled = false } tasks.getByName("generatePomFileForMavenJavaPublication") { enabled = false } tasks.getByName("publishMavenJavaPublicationToLocalRemoteRepository") { enabled = false } chipList.each {thisChip -> artifacts { archives tasks.getByName("${thisChip}SupportJar") } } chipList.each { thisChip -> publishing { publications { mavenJava(MavenPublication) { artifact tasks.getByName("${thisChip}SupportJar") } } } } } if( osdetector.os.startsWith("windows")) { FileCollection collection = layout.files { file("build/libs/").listFiles() } //collection.collect { relativePath(it) }.sort().each { println it } publishing { publications { mavenJava(MavenPublication) { artifact jar collection.collect {File fi -> if( fi.name.contains('linux-x86_64-avx2-cpu')) { logger.quiet("Adding artifact ${fi.name} to publication.") artifact source: fi, classifier: 'linux-x86_64-avx2-cpu', extension: 'jar' } } } } } } /* def pomClosure = { name = 'Brutex AI - Native Components' delegate.description = 'Underlying native components for the Brutex AI deeplearning framework for Java' url = 'https://ai.brutex.net' licenses { license { name = 'Apache License, Version 2.0' url = 'http://www.apache.org/licenses/LICENSE-2.0' distribution = 'repo' } } developers { developer { id = 'irnbrux' name = 'Brian Rosenberger' email = 'bru@brutex.de' } } scm { url = 'https://brutex.net/svn/' connection = 'scm:svn:https://brutex.net/svn/bruai4j/' } } */ //tasks.getByName("publishMavenJavaPublicationToOSSRHRepository") { MavenRemotePublisher pub -> // logger.quiet(pub.dump()); //} signing { useGpgCmd() if (!version.endsWith('SNAPSHOT')) { sign publishing.publications.mavenJava //sign publishing.publications.mavenJavacppPlatform } }