cavis/cavis-native/cavis-native-lib/build.gradle

555 lines
22 KiB
Groovy

import org.gradle.api.publish.maven.internal.publisher.MavenRemotePublisher
import org.gradle.language.nativeplatform.internal.Dimensions
buildscript {
/****************************************************************************
* Establish Visual Studio configuration environment for Windows native builds
* NOTE: vsconfig.gradle path is relative to each GPL project module
****************************************************************************/
apply from: "../../vsconfig.gradle"
apply from: "../../chooseBackend.gradle"
ext {
host_cores = Runtime.getRuntime().availableProcessors()
buildHelper = "-mingw"
javacppPlatform = osdetector.classifier
println "Building on ${host_cores} CPU cores."
println "JavaCPP target plattform is ${javacppPlatform}"
if (project.hasProperty("CAVIS_AVX_EXTENSION")) {
avxExtension = project.getProperty("CAVIS_AVX_EXTENSION").toLowerCase()
println "Bulding with ${avxExtension}"
} else {
avxExtension = "avx2"
logger.quiet("No AVX CPU extension selected (avx2|avx512). Building with default 'avx2'")
}
javacppPlatformExtension = "-${avxExtension}".toString()
getBuildPlatform = { String chip, Task tsk ->
def pf =""
if(chip.equals("cuda")) {
pf = osdetector.classifier
} else {
if(osdetector.os.equals("windows")) {
pf = "${osdetector.classifier}-mingw"
} else {
pf = "${osdetector.classifier}"
}
}
logger.info("Setting properties for task '{}' to '{}'", tsk.getName(), pf)
return pf
}
}
dependencies {
classpath platform(project(":cavis-common-platform"))
classpath group: "org.bytedeco", name: "openblas"
classpath group: "org.bytedeco", name: "openblas", classifier: "${javacppPlatform}"
classpath group: "org.bytedeco", name:"mkl-dnn"
classpath group: "org.bytedeco", name:"mkl-dnn", classifier: "${javacppPlatform}"
classpath group: "org.bytedeco", name: "javacpp"
classpath group: "org.bytedeco", name: "javacpp", classifier: "${javacppPlatform}"
}
}
plugins {
id 'java-library'
id 'org.bytedeco.gradle-javacpp-build' version "1.5.7"
id 'maven-publish'
id 'signing'
}
chipList.each {thisChip ->
sourceSets.register("${thisChip}Support") {
java {
srcDirs = ['src/main/java', "${buildDir}/generated/sources/javacpp/${thisChip}//${javacppPlatform}${javacppPlatformExtension}/"]
include "org/nd4j/nativeblas/${thisChip}/Nd4j${thisChip.capitalize()}Helper.java"
include "org/nd4j/nativeblas/${thisChip}/Nd4j${thisChip.capitalize()}Presets.java"
include "org/nd4j/nativeblas/Nd4j${thisChip.capitalize()}.java"
}
it.compiledBy("javacpp${thisChip.capitalize()}SupportBuildCommand",
"javacpp${thisChip.capitalize()}SupportBuildCompiler")
}
}
if(osdetector.os.startsWith("windows")) {
sourceSets {
main {
java {
srcDirs = ['src/main/java']
include 'org/nd4j/nativeblas/Dummy.java'
}
}
}
}
java {
chipList.each {thisChip ->
registerFeature("${thisChip}Support") {
usingSourceSet(sourceSets.findByName("${thisChip}Support"))
//withJavadocJar()
//withSourcesJar()
}
}
}
/*
configurations.each(s -> {
println "Configurations: " + s.name + " " + s.artifacts.each( x ->
{ println x.getFile().getName()})
})
*/
dependencies {
api platform(project(':cavis-common-platform'))
api "org.bytedeco:javacpp"
if(withCuda()) {
cudaSupportImplementation platform(project(':cavis-common-platform'))
cudaSupportImplementation project(":cavis-dnn:cavis-dnn-api")
cudaSupportImplementation project(":cavis-dnn:cavis-dnn-common")
cudaSupportImplementation project(":cavis-native:cavis-native-blas")
cudaSupportImplementation project(":cavis-native:cavis-native-common")
cudaSupportImplementation "commons-io:commons-io"
cudaSupportImplementation group: "org.bytedeco", name: "openblas"
cudaSupportImplementation group: "org.bytedeco", name: "openblas", classifier: "${javacppPlatform}"
cudaSupportImplementation group: "org.bytedeco", name: "cuda"
cudaSupportImplementation group: "org.bytedeco", name: "cuda", classifier: "${javacppPlatform}"
cudaSupportImplementation "org.apache.logging.log4j:log4j-core:2.17.0"
cudaSupportImplementation "com.google.guava:guava:14.0.1"
cudaSupportImplementation "org.apache.commons:commons-lang3"
cudaSupportImplementation "org.apache.commons:commons-math3"
cudaSupportImplementation "com.google.flatbuffers:flatbuffers-java"
cudaSupportImplementation 'javax.mail:javax.mail-api:1.6.2'
}
if(withCpu()) {
cpuSupportImplementation platform(project(':cavis-common-platform'))
cpuSupportImplementation project(":cavis-dnn:cavis-dnn-api")
cpuSupportImplementation project(":cavis-dnn:cavis-dnn-common")
cpuSupportImplementation project(":cavis-native:cavis-native-blas")
cpuSupportImplementation project(":cavis-native:cavis-native-common")
cpuSupportImplementation "commons-io:commons-io"
cpuSupportImplementation group: "org.bytedeco", name: "openblas"
cpuSupportImplementation group: "org.bytedeco", name: "openblas", classifier: "${javacppPlatform}"
cpuSupportImplementation group: "org.bytedeco", name: "opencv"
cpuSupportImplementation group: "org.bytedeco", name: "opencv", classifier: "${javacppPlatform}"
cpuSupportImplementation "org.apache.logging.log4j:log4j-core:2.17.0"
cpuSupportImplementation "com.google.guava:guava:14.0.1"
cpuSupportImplementation "org.apache.commons:commons-lang3"
cpuSupportImplementation "org.apache.commons:commons-math3"
cpuSupportImplementation "com.google.flatbuffers:flatbuffers-java"
cpuSupportImplementation 'javax.mail:javax.mail-api:1.6.2'
}
implementation projects.cavisDnn.cavisDnnApi
implementation projects.cavisDnn.cavisDnnCommon
implementation project(":cavis-native:cavis-native-blas")
implementation project(":cavis-native:cavis-native-common")
implementation "commons-io:commons-io"
implementation "org.bytedeco:openblas"
implementation group: "org.bytedeco", name: "openblas", classifier: "${javacppPlatform}"
implementation "org.apache.logging.log4j:log4j-core"
implementation "com.google.guava:guava:14.0.1"
implementation "org.apache.commons:commons-lang3"
implementation "org.apache.commons:commons-math3"
implementation "com.google.flatbuffers:flatbuffers-java"
//javacppPlatform project(":cavis-native:cavis-native-blas")
}
clean {
doFirst {
delete "${projectDir}/build"
delete "${projectDir}/src/main/include/config.h"
chipList.each {
delete "${projectDir}/blasbuild/${it}"
}
}
}
task deepClean(type: Delete) {
dependsOn clean
doFirst {
delete "$projectDir}/blasbuild"
}
}
tasks.withType(org.bytedeco.gradle.javacpp.BuildTask) {
buildResource = [ "/org/bytedeco/openblas/${javacppPlatform}/",
"/org/bytedeco/mkldnn/${javacppPlatform}/"]
includeResource = ["/org/bytedeco/openblas/${javacppPlatform}/include/"]
linkResource = ["/org/bytedeco/openblas/${javacppPlatform}/",
"/org/bytedeco/openblas/${javacppPlatform}/lib/"]
//buildPath = [ org.bytedeco.javacpp.Loader.getCacheDir() ]
}
// Disable the standard javacpp generated tasks and use own
// versions below. This allows to build for each variant
[javacppBuildParser, javacppBuildCommand, javacppCompileJava, javacppBuildCompiler].each {
it.enabled false;
}
chipList.each { thisChip ->
// 1)
//Run the C++ compile first
tasks.register("javacpp${thisChip.capitalize()}SupportBuildCommand", org.bytedeco.gradle.javacpp.BuildTask) {
if (project.hasProperty("skip-native") && project.getProperty("skip-native").equals("true")) {
enabled = false
}
dependsOn "processResources"
properties = getBuildPlatform( thisChip, it )
includePath = ["${projectDir}/src/main/cpp/blas/",
"${projectDir}/blasbuild/${thisChip}/${avxExtension}/src/main/include/",
"${projectDir}/blasbuild/${thisChip}/${avxExtension}/flatbuffers-src/include",
"${projectDir}/blasbuild/${thisChip}/${avxExtension}/cpu_features-src/include",
"${projectDir}/blasbuild/${thisChip}/${avxExtension}/mkldnn-src/include"]
linkPath = ["${projectDir}/blasbuild/${thisChip}/${avxExtension}/output"]
//No idea why this is here, but it looks like even for the javacppBuildCommand task,
//there is a javacpp Loader actively determining platform etc.
classOrPackageNames = ["org.nd4j.nativeblas.${thisChip}.Nd4j${thisChip.capitalize()}Presets"]
workingDirectory = projectDir
//if the classpath is not set here, the javacpp classloader starts to look around
//everywhere and causes java.io.IOExceptions: because files is being used by another process
classPath = [:]
classPath += ["${buildDir}/classes/java/${thisChip}Support/"]
//classPath += ["${buildDir}/classes/java/main/"]
/* Get VCVARS in case we want to build CUDA
* MinGW64 g++ on MSYS is used otherwise */
if (thisChip.equals('cuda') && osdetector.os.startsWith("win") && !VISUAL_STUDIO_INSTALL_DIR.isEmpty()) {
def proc = ["cmd.exe", "/c", "${VISUAL_STUDIO_VCVARS_CMD} > nul && set"].execute()
it.environmentVariables = it.environmentVariables ?: [:]
def lines = proc.text.split("\\r?\\n")
for (def line in lines) {
if (line.contains("=")) {
def parts = line.split("=")
it.environmentVariables.put(parts[0], parts[1])
}
}
}
if (thisChip.equals('cuda') && osdetector.os.startsWith("windows")) { //cuDNN requires CUDA
it.buildCommand = ['sh', 'buildnativeoperations.sh',
'-V',
'--build-type', 'release',
'--chip', thisChip,
'--plattform', 'x86_64',
'--chip-extension', avxExtension,
'-j', "${host_cores}",
// '--helper', 'mkldnn',
'--helper', 'cudnn']
} else if (thisChip.equals('cuda') && osdetector.os.startsWith("linux")) { //cuDNN requires CUDA
it.buildCommand = ['bash', 'buildnativeoperations.sh',
'-V',
'--build-type', 'release',
'--chip', thisChip,
'--plattform', 'x86_64',
'--chip-extension', avxExtension,
'-j', "${host_cores}",
// '--helper', 'mkldnn',
'--helper', 'cudnn']
} else {
it.buildCommand = ['bash', 'buildnativeoperations.sh',
'-V',
'--build-type', 'release',
'--chip', thisChip,
'--plattform', 'x86_64',
'--chip-extension', avxExtension,
'-j', "${host_cores}",
'--helper', 'mkldnn']
}
}
//Create a task to (pre)compile the java presets (required for javacppBuildParser)
tasks.register("compile${thisChip.capitalize()}Support", JavaCompile) {
def thisSS = sourceSets.findByName("${thisChip}Support")
it.source = thisSS.allSource
it.classpath = thisSS.compileClasspath
it.destinationDirectory = file("${buildDir}/classes/java/${thisChip}Support/")
}
//Run the parser on the InfoMap in Nd4j$ChipPresets and listed header files in @Platform
//Generates Nd4jCpu.java and/ or Nd4jCuda.java Java JNI code
tasks.register("javacpp${thisChip.capitalize()}SupportBuildParser", org.bytedeco.gradle.javacpp.BuildTask) {
if (project.hasProperty("skip-native") && project.getProperty("skip-native").equals("true")) {
enabled = false
}
dependsOn "compile${thisChip.capitalize()}Support"
includePath = ["${projectDir}/src/main/cpp/blas/",
"${projectDir}/blasbuild/${thisChip}/${avxExtension}/src/main/include/",
"${projectDir}/blasbuild/${thisChip}/${avxExtension}/flatbuffers-src/include",
"${projectDir}/blasbuild/${thisChip}/${avxExtension}/cpu_features-src/include",
"${projectDir}/blasbuild/${thisChip}/${avxExtension}/mkldnn-src/include"]
classOrPackageNames = ["org.nd4j.nativeblas.${thisChip}.Nd4j${thisChip.capitalize()}Presets"]
outputDirectory = file("${buildDir}/generated/sources/javacpp/${thisChip}/${javacppPlatform}${javacppPlatformExtension}/")
classPath = sourceSets.getByName("${thisChip}Support").getRuntimeClasspath()
classPath += ["${buildDir}/classes/java/${thisChip}Support/"]
}
// Generates jnijavacpp.cpp and jniNativeLibrary.cpp, compiles and links it
tasks.register("javacpp${thisChip.capitalize()}SupportBuildCompiler", org.bytedeco.gradle.javacpp.BuildTask) {
if (project.hasProperty("skip-native") && project.getProperty("skip-native").equals("true")) {
enabled = false
}
def thisTask = (org.bytedeco.gradle.javacpp.BuildTask) it
thisTask.dependsOn = ["javacpp${thisChip.capitalize()}SupportBuildParser"]
thisTask.linkPath = ["${projectDir}/blasbuild/${thisChip}/${avxExtension}/output"]
thisTask.includePath = ["${projectDir}/src/main/cpp/blas/",
"${projectDir}/blasbuild/${thisChip}/${avxExtension}/src/main/include/",
"${projectDir}/blasbuild/${thisChip}/${avxExtension}/flatbuffers-src/include",
"${projectDir}/blasbuild/${thisChip}/${avxExtension}/cpu_features-src/include",
"${projectDir}/blasbuild/${thisChip}/${avxExtension}/mkldnn-src/include"]
thisTask.properties = getBuildPlatform( thisChip, thisTask )
if(thisChip.equals('cuda') && osdetector.os.startsWith("win") && !VISUAL_STUDIO_INSTALL_DIR.isEmpty()) {
def proc = ["cmd.exe", "/c", "${VISUAL_STUDIO_VCVARS_CMD} > nul && where.exe cl.exe"].execute()
def outp = proc.text
def cl = outp.replace("\\", "\\\\").trim()
def currentCompiler = ""
doFirst{
currentCompiler = System.getProperty("org.bytedeco.javacpp.platform.compiler")
System.setProperty("org.bytedeco.javacpp.platform.compiler", cl)
logger.quiet("Task ${thisTask.name} overrides compiler '${currentCompiler}' with '${cl}'.")
}
doLast {
//restore compiler
System.setProperty("org.bytedeco.javacpp.platform.compiler", currentCompiler ?: "")
}//System.setProperty("org.bytedeco.javacpp.platform.compiler", cl)
//System.setProperty("org.bytedeco.javacpp.platform.compiler.cpp11", cl)
proc = ["cmd.exe", "/c", "${VISUAL_STUDIO_VCVARS_CMD} > nul && set"].execute()
thisTask.environmentVariables = thisTask.environmentVariables ?: [:]
def lines = proc.text.split("\\r?\\n")
for (def line in lines) {
if (line.contains("=")) {
def parts = line.split("=")
thisTask.environmentVariables.put(parts[0], parts[1])
}
}
} else {
//System.setProperty("org.bytedeco.javacpp.platform.compiler", "g++")
}
thisTask.buildPath = ["$buildDir/generated/sources/javacpp/${thisChip}/${javacppPlatform}${javacppPlatformExtension}/"]
thisTask.copyLibs = true
thisTask.deleteJniFiles(false)
outputName = "jnind4j${thisChip}"
thisTask.outputDirectory = file("$buildDir/generated/sources/javacpp/${thisChip}/${javacppPlatform}${javacppPlatformExtension}/")
thisTask.classOrPackageNames= ["org.nd4j.nativeblas.Nd4j${thisChip.capitalize()}"]
thisTask.configDirectory = file("${buildDir}/classes/java/${thisChip}Support/META-INF/native-image/${javacppPlatform}")
//Need to set the classpath, so that external jars from the dependency list are resolved by the ClassLoader as well
thisTask.classPath = [:]
thisTask.classPath = ["${buildDir}/classes/java/${thisChip}Support"]
thisTask.classPath += sourceSets.findByName("${thisChip}Support").runtimeClasspath
//sourceSets.findByName("${thisChip}Support").runtimeClasspath.each{ s ->
// thisTask.classPath += s
//}
}
// Generates jnijavacpp.cpp and jniNativeLibrary.cpp, compiles and links it
tasks.getByName("${thisChip}SupportJar") { Jar thisTask ->
dependsOn "javacpp${thisChip.capitalize()}SupportBuildCompiler"
dependsOn "javacpp${thisChip.capitalize()}SupportBuildCommand"
//it.from sourceSets.getByName("${thisChip}Support").getOutput()
def spec = copySpec {
from(tasks.getByName("javacpp${thisChip.capitalize()}SupportBuildCompiler")) {
exclude { f ->
def exclude = f.file.isDirectory()
if(exclude) {
logger.info("${thisTask.name}: excluding '${f}'")
} else {
logger.info("${thisTask.name}: including '${f}'")
}
return exclude
}
into "${javacppPlatform}/" //we need it in a platform, that javacpp Loader understands
}
from(sourceSets.getByName("${thisChip}Support").getOutput()) {
}
duplicatesStrategy DuplicatesStrategy.EXCLUDE
}
thisTask.with spec
thisTask.archiveClassifier = "${javacppPlatform}${javacppPlatformExtension}-${thisChip}"
}
//tasks.getByName("${thisChip}SupportJar").dependsOn("javacpp${thisChip.capitalize()}SupportJar")
}
//Before we can compile the whole java part, we
//need to generate the Nd4jXXX.java files first
chipList.each { thisChip ->
tasks.findByName("compile${thisChip.capitalize()}SupportJava").each { t ->
t.dependsOn "javacpp${thisChip.capitalize()}SupportBuildParser"
}
}
tasks.withType(JavaCompile) {
// options.setCompilerArgs(Arrays.asList("-Xlint:unchecked"))
}
tasks.withType(Javadoc) {
options.addStringOption('Xdoclint:none', '-quiet')
}
jar {
manifest {
attributes 'Class-Path': configurations.runtimeClasspath.collect { it.getName() }.join(' '),
'Implementation-Title': 'Brutex AI - Native Components',
'Implementation-Vendor': 'Brutex Network',
'Implementation-Version': archiveVersion,
'Specification-Title': 'Brutex AI - Native Components',
'Specification-Vendor': 'Brutex Network',
'Specification-Version': archiveVersion
}
//archiveClassifier = "${javacppPlatform}${javacppPlatformExtension}-${chip}"
}
javadoc {
dependsOn "javacppPomProperties"
failOnError = false
//options.links = ['http://bytedeco.org/javacpp/apidocs']
options.addStringOption('Xdoclint:none', '-quiet')
//options.JFlags = ["-Xdoclint:none"]
}
if(! osdetector.os.startsWith("windows")) {
tasks.getByName("publish") {
enabled = true
}
tasks.getByName("generatePomFileForMavenJavaPublication") {
enabled = true
}
tasks.getByName("publishMavenJavaPublicationToLocalRemoteRepository") {
enabled = true
}
chipList.each {thisChip ->
artifacts {
archives tasks.getByName("${thisChip}SupportJar")
}
}
chipList.each { thisChip ->
publishing {
publications {
mavenJava(MavenPublication) {
logger.quiet("Adding artifacts from task {} to the publication.", "${thisChip}SupportJar" )
artifact tasks.getByName("${thisChip}SupportJar")
}
}
}
}
}
if( osdetector.os.startsWith("windows")) {
FileCollection collection = layout.files { file("build/libs/").listFiles() }
//collection.collect { relativePath(it) }.sort().each { println it }
publishing {
publications {
mavenJava(MavenPublication) {
artifact jar
collection.collect {File fi ->
if( fi.name.contains('linux-x86_64-avx2-cpu')) {
logger.quiet("Adding artifact ${fi.name} to publication.")
artifact source: fi, classifier: 'linux-x86_64-avx2-cpu', extension: 'jar'
}
}
}
}
}
}
/*
def pomClosure = {
name = 'Brutex AI - Native Components'
delegate.description = 'Underlying native components for the Brutex AI deeplearning framework for Java'
url = 'https://ai.brutex.net'
licenses {
license {
name = 'Apache License, Version 2.0'
url = 'http://www.apache.org/licenses/LICENSE-2.0'
distribution = 'repo'
}
}
developers {
developer {
id = 'irnbrux'
name = 'Brian Rosenberger'
email = 'bru@brutex.de'
}
}
scm {
url = 'https://brutex.net/svn/'
connection = 'scm:svn:https://brutex.net/svn/bruai4j/'
}
}
*/
//tasks.getByName("publishMavenJavaPublicationToOSSRHRepository") { MavenRemotePublisher pub ->
// logger.quiet(pub.dump());
//}
signing {
useGpgCmd()
if (!version.endsWith('SNAPSHOT')) {
sign publishing.publications.mavenJava
//sign publishing.publications.mavenJavacppPlatform
}
}