From d854e28b34b080ac9714185e4ecfa89b8f78be57 Mon Sep 17 00:00:00 2001 From: Ryan Nett Date: Tue, 2 Jul 2019 12:19:07 -0700 Subject: [PATCH] System info export for debugging and bug reporting (#34) * System info export for debugging and bug reporting Signed-off-by: Ryan Nett * class name fix Signed-off-by: Ryan Nett * add version information, pointer memory info Signed-off-by: Ryan Nett * add nvidia-smi and nvcc info Signed-off-by: Ryan Nett * line cleanup Signed-off-by: Ryan Nett * nvidia-smi run works Signed-off-by: Ryan Nett * add oshi dependency Signed-off-by: Ryan Nett * use OS info, add workspaces info Signed-off-by: Ryan Nett * use ServiceLoader to load GPU information Signed-off-by: Ryan Nett * register service Signed-off-by: Ryan Nett * moved service out of NativeOpsHolder (private constructor) Signed-off-by: Ryan Nett * added newline Signed-off-by: Ryan Nett * added license Signed-off-by: Ryan Nett * and one more Signed-off-by: Ryan Nett * copyright update Signed-off-by: Ryan Nett * fixes Signed-off-by: Ryan Nett * removed unused imports Signed-off-by: Ryan Nett * removed more unused imports Signed-off-by: Ryan Nett * close streams Signed-off-by: Ryan Nett * and another one Signed-off-by: Ryan Nett * use method Signed-off-by: Ryan Nett * one more copyright Signed-off-by: Ryan Nett * remove double license Signed-off-by: Ryan Nett * moved test to correct package Signed-off-by: Ryan Nett * classpath update Signed-off-by: Ryan Nett * classpath for java >8 fix Signed-off-by: Ryan Nett --- .../util/CrashReportingUtil.java | 51 +- .../nd4j-api-parent/nd4j-api/pom.xml | 7 + .../java/org/nd4j/systeminfo/GPUInfo.java | 39 ++ .../org/nd4j/systeminfo/GPUInfoProvider.java | 23 + .../java/org/nd4j/systeminfo/SystemInfo.java | 442 ++++++++++++++++++ .../nativeblas/NativeOpsGPUInfoProvider.java | 55 +++ .../org/nd4j/nativeblas/NativeOpsHolder.java | 5 +- .../org.nd4j.systeminfo.GPUInfoProvider | 17 + .../org/nd4j/systeminfo/TestSystemInfo.java | 27 ++ 9 files changed, 628 insertions(+), 38 deletions(-) create mode 100644 nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/GPUInfo.java create mode 100644 nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/GPUInfoProvider.java create mode 100644 nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/SystemInfo.java create mode 100644 nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeOpsGPUInfoProvider.java create mode 100644 nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/resources/META-INF/services/org.nd4j.systeminfo.GPUInfoProvider create mode 100644 nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/systeminfo/TestSystemInfo.java diff --git a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/CrashReportingUtil.java b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/CrashReportingUtil.java index 8aae34eb0..d716204c5 100644 --- a/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/CrashReportingUtil.java +++ b/deeplearning4j/deeplearning4j-nn/src/main/java/org/deeplearning4j/util/CrashReportingUtil.java @@ -16,7 +16,24 @@ package org.deeplearning4j.util; +import static org.deeplearning4j.nn.conf.inputs.InputType.inferInputType; +import static org.deeplearning4j.nn.conf.inputs.InputType.inferInputTypes; +import static org.nd4j.systeminfo.SystemInfo.inferVersion; + import com.jakewharton.byteunits.BinaryByteUnit; +import java.io.File; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; import lombok.Getter; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -45,20 +62,9 @@ import org.nd4j.linalg.primitives.Pair; import org.nd4j.linalg.util.ArrayUtil; import org.nd4j.nativeblas.NativeOps; import org.nd4j.nativeblas.NativeOpsHolder; -import org.nd4j.versioncheck.VersionCheck; -import org.nd4j.versioncheck.VersionInfo; import oshi.SystemInfo; import oshi.software.os.OperatingSystem; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.text.SimpleDateFormat; -import java.util.*; - -import static org.deeplearning4j.nn.conf.inputs.InputType.inferInputType; -import static org.deeplearning4j.nn.conf.inputs.InputType.inferInputTypes; - /** * A utility for generating crash reports when an out of memory error occurs. * @@ -396,8 +402,6 @@ public class CrashReportingUtil { sb.append(String.format(fGpu, "Name", "CC", "Total Memory", "Used Memory", "Free Memory")).append("\n"); for (int i = 0; i < nDevices; i++) { try { - Class c = Class.forName("org.nd4j.jita.allocator.pointers.CudaPointer"); - Constructor constructor = c.getConstructor(long.class); String name = nativeOps.getDeviceName(i); long total = nativeOps.getDeviceTotalMemory(i); long free = nativeOps.getDeviceFreeMemory(i); @@ -638,25 +642,4 @@ public class CrashReportingUtil { sb.append(fBytes("Total Activation Gradient Memory", totalExOutput)); } - public static Pair inferVersion(){ - List vi = VersionCheck.getVersionInfos(); - - String dl4jVersion = null; - String dl4jCudaArtifact = null; - for(VersionInfo v : vi){ - if("org.deeplearning4j".equals(v.getGroupId()) && "deeplearning4j-core".equals(v.getArtifactId())){ - String version = v.getBuildVersion(); - if(version.contains("SNAPSHOT")){ - dl4jVersion = version + " (" + v.getCommitIdAbbrev() + ")"; - } - dl4jVersion = version; - } else if("org.deeplearning4j".equals(v.getGroupId()) && v.getArtifactId() != null && v.getArtifactId().contains("deeplearning4j-cuda")){ - dl4jCudaArtifact = v.getArtifactId(); - } - - } - - return new Pair<>(dl4jVersion, dl4jCudaArtifact); - } - } diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/pom.xml b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/pom.xml index 030edb286..7168b57e0 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/pom.xml +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/pom.xml @@ -174,6 +174,13 @@ + + + com.github.oshi + oshi-core + ${oshi.version} + + junit junit diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/GPUInfo.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/GPUInfo.java new file mode 100644 index 000000000..a77414c25 --- /dev/null +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/GPUInfo.java @@ -0,0 +1,39 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.nd4j.systeminfo; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class GPUInfo { + + public static final String fGpu = " %-30s %-5s %24s %24s %24s"; + + private String name; + private long totalMemory; + private long freeMemory; + int major; + int minor; + + @Override + public String toString(){ + return String.format(fGpu, name, major + "." + minor, SystemInfo.fBytes(totalMemory), + SystemInfo.fBytes(totalMemory - freeMemory), SystemInfo.fBytes(freeMemory)); + } +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/GPUInfoProvider.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/GPUInfoProvider.java new file mode 100644 index 000000000..78ee866a5 --- /dev/null +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/GPUInfoProvider.java @@ -0,0 +1,23 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.nd4j.systeminfo; + +import java.util.List; + +public interface GPUInfoProvider { + List getGPUs(); +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/SystemInfo.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/SystemInfo.java new file mode 100644 index 000000000..6a19c19ab --- /dev/null +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/systeminfo/SystemInfo.java @@ -0,0 +1,442 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.nd4j.systeminfo; + +import static org.nd4j.systeminfo.GPUInfo.fGpu; + +import com.jakewharton.byteunits.BinaryByteUnit; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.lang.management.ManagementFactory; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.ServiceLoader; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.SystemUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.bytedeco.javacpp.Pointer; +import org.nd4j.linalg.api.environment.Nd4jEnvironment; +import org.nd4j.linalg.api.memory.MemoryWorkspace; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.primitives.Pair; +import org.nd4j.versioncheck.VersionCheck; +import org.nd4j.versioncheck.VersionInfo; +import oshi.software.os.OperatingSystem; + +/** + * Utility class to get system info for debugging and error reporting + */ +public class SystemInfo { + + private static void appendField(StringBuilder sb, String name, Object value){ + sb.append(name).append(": ").append(value.toString()).append("\n"); + } + + private static void appendProperty(StringBuilder sb, String name, String property){ + appendField(sb, name, System.getProperty(property)); + } + + private static void appendHeader(StringBuilder sb, String name){ + sb.append("\n\n---------------").append(name).append("---------------\n\n"); + } + + private static final String FORMAT = "%-40s%s"; + + public static String f(String s1, Object o){ + return String.format(FORMAT, s1, (o == null ? "null" : o.toString())) + "\n"; + } + + public static String fBytes(long bytes){ + String s = BinaryByteUnit.format(bytes, "#.00"); + String format = "%10s"; + s = String.format(format, s); + if(bytes >= 1024){ + s += " (" + bytes + ")"; + } + return s; + } + + public static String fBytes(String s1, long bytes){ + String s = fBytes(bytes); + return f(s1, s); + } + + + private static void appendCUDAInfo(StringBuilder sb, boolean isWindows){ + + + sb.append("Nvidia-smi:\n"); + + try { + ProcessBuilder pb = new ProcessBuilder("nvidia-smi"); + appendOutput(sb, pb); + } catch (IOException e) { + sb.append("nvidia-smi run failed."); + + if(isWindows) { + sb.append(" Trying in C:\\Program Files\\NVIDIA Corporation\\NVSMI\n"); + + try { + ProcessBuilder pb = new ProcessBuilder( + "C:\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe"); + appendOutput(sb, pb); + } catch (IOException e1) { + sb.append("C:\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi run failed\n"); + sb.append(e1.getMessage()); + } + } else { + sb.append("\n"); + } + + } + + sb.append("\nnvcc --version:\n"); + try { + + ProcessBuilder pb = new ProcessBuilder("nvcc", "--version"); + appendOutput(sb, pb); + } catch (IOException e) { + sb.append("nvcc --version run failed."); + } + } + + private static void appendOutput(StringBuilder sb, ProcessBuilder pb) throws IOException { + pb.redirectErrorStream(true); + pb.redirectOutput(); + + Process p = pb.start(); + try(InputStreamReader isr = new InputStreamReader(p.getInputStream())) { + try(BufferedReader reader = new BufferedReader(isr)) { + String line = null; + while ((line = reader.readLine()) != null) { + sb.append(line); + sb.append(System.getProperty("line.separator")); + } + } + } + sb.append("\n"); + } + + + /** + * Gets system info in a string + */ + public static String getSystemInfo(){ + StringBuilder sb = new StringBuilder(); + + //nd4j info + appendHeader(sb, "ND4J Info"); + + Pair pair = inferVersion(); + sb.append(f("Deeplearning4j Version", (pair.getFirst() == null ? "" : pair.getFirst()))); + sb.append(f("Deeplearning4j CUDA", (pair.getSecond() == null ? "" : pair.getSecond()))); + + sb.append("\n"); + + boolean isCUDA = false; + + try { + appendField(sb, "Nd4j Backend", Nd4j.getBackend().getClass().getSimpleName()); + + Properties props = Nd4j.getExecutioner().getEnvironmentInformation(); + + double memory = ((Long) props.get("memory.available")) / (double) 1024 / 1024 / 1024; + String fm = String.format("%.1f", memory); + sb.append("Backend used: [").append(props.get("backend")).append("]; OS: [").append(props.get("os")) + .append("]\n"); + sb.append("Cores: [").append(props.get("cores")).append("]; Memory: [").append(fm).append("GB];\n"); + sb.append("Blas vendor: [").append(props.get("blas.vendor")).append("]\n"); + + if (Nd4j.getExecutioner().getClass().getSimpleName().equals("CudaExecutioner")) { + isCUDA = true; + + List> devicesList = (List>) props.get(Nd4jEnvironment.CUDA_DEVICE_INFORMATION_KEY); + for (Map dev : devicesList) { + sb.append("Device Name: [").append(dev.get(Nd4jEnvironment.CUDA_DEVICE_NAME_KEY)).append("]; ") + .append("CC: [").append(dev.get(Nd4jEnvironment.CUDA_DEVICE_MAJOR_VERSION_KEY)).append(".") + .append(dev.get(Nd4jEnvironment.CUDA_DEVICE_MINOR_VERSION_KEY)) + .append("]; Total/free memory: [").append(dev.get(Nd4jEnvironment.CUDA_TOTAL_MEMORY_KEY)) + .append("]").append("\n"); + } + } + + sb.append("\nExecutor Properties:\n"); + + for (Map.Entry prop : props.entrySet()) { + sb.append(prop.getKey().toString()).append("=").append(prop.getValue()).append("\n"); + } + } catch (Exception e){ + sb.append("Could not get ND4J info\n"); + sb.append("Exception: ").append(e.getMessage()).append("\n"); + sb.append(ExceptionUtils.getStackTrace(e)).append("\n\n"); + } + + //hardware info + appendHeader(sb, "Hardware Info"); + + appendField(sb, "Available processors (cores)", + Runtime.getRuntime().availableProcessors()); + + oshi.SystemInfo sys = new oshi.SystemInfo(); + OperatingSystem os = sys.getOperatingSystem(); + String procName = sys.getHardware().getProcessor().getName(); + long totalMem = sys.getHardware().getMemory().getTotal(); + + sb.append(f("Operating System", os.getManufacturer() + " " + os.getFamily() + " " + os.getVersion().getVersion())); + sb.append(f("CPU", procName)); + sb.append(f("CPU Cores - Physical", sys.getHardware().getProcessor().getPhysicalProcessorCount())); + sb.append(f("CPU Cores - Logical", sys.getHardware().getProcessor().getLogicalProcessorCount())); + sb.append(fBytes("Total System Memory", totalMem)); + + sb.append("\n"); + + boolean hasGPUs = false; + + ServiceLoader loader = ServiceLoader.load(GPUInfoProvider.class); + Iterator iter = loader.iterator(); + if(iter.hasNext()){ + List gpus = iter.next().getGPUs(); + + sb.append(f("Number of GPUs Detected", gpus.size())); + + if(!gpus.isEmpty()) + hasGPUs = true; + + sb.append(String.format(fGpu, "Name", "CC", "Total Memory", "Used Memory", "Free Memory")).append("\n"); + + for(GPUInfo gpuInfo : gpus){ + sb.append(gpuInfo).append("\n"); + } + + } else { + sb.append("GPU Provider not found (are you missing nd4j-native?)"); + } + + appendHeader(sb, "CUDA Info"); + + if(!isCUDA){ + sb.append("NOT USING CUDA Nd4j\n"); + + if(hasGPUs) + sb.append("GPUs detected, trying to list CUDA info anyways\n"); + } + + if(isCUDA || hasGPUs) + appendCUDAInfo(sb, SystemUtils.IS_OS_WINDOWS); + + //OS info + appendHeader(sb, "OS Info"); + + appendProperty(sb, "OS" , "os.name"); + appendProperty(sb, "Version","os.version"); + appendProperty(sb, "Arch","os.arch"); + + //memory settings + appendHeader(sb, "Memory Settings"); + + appendField(sb, "Free memory (bytes)", + Runtime.getRuntime().freeMemory()); + + long maxMemory = Runtime.getRuntime().maxMemory(); + appendField(sb, "Maximum memory (bytes)", + (maxMemory == Long.MAX_VALUE ? "No Limit" : maxMemory)); + + appendField(sb, "Total memory available to JVM (bytes)", + Runtime.getRuntime().totalMemory()); + + sb.append("\n"); + + long xmx = Runtime.getRuntime().maxMemory(); + long jvmTotal = Runtime.getRuntime().totalMemory(); + long javacppMaxPhys = Pointer.maxPhysicalBytes(); + long javacppMaxBytes = Pointer.maxBytes(); + long javacppCurrPhys = Pointer.physicalBytes(); + long javacppCurrBytes = Pointer.totalBytes(); + sb.append(fBytes("JVM Memory: XMX", xmx)) + .append(fBytes("JVM Memory: current", jvmTotal)) + .append(fBytes("JavaCPP Memory: Max Bytes", javacppMaxBytes)) + .append(fBytes("JavaCPP Memory: Max Physical", javacppMaxPhys)) + .append(fBytes("JavaCPP Memory: Current Bytes", javacppCurrBytes)) + .append(fBytes("JavaCPP Memory: Current Physical", javacppCurrPhys)); + boolean periodicGcEnabled = Nd4j.getMemoryManager().isPeriodicGcActive(); + long autoGcWindow = Nd4j.getMemoryManager().getAutoGcWindow(); + sb.append(f("Periodic GC Enabled", periodicGcEnabled)); + if(periodicGcEnabled){ + sb.append(f("Periodic GC Frequency", autoGcWindow + " ms")); + } + + // Workspaces info + + appendHeader(sb, "Workspace Information"); + List allWs = Nd4j.getWorkspaceManager().getAllWorkspacesForCurrentThread(); + sb.append(f("Workspaces: # for current thread", (allWs == null ? 0 : allWs.size()))); + //sb.append(f("Workspaces: # for all threads", allWs.size())); //TODO + long totalWsSize = 0; + if(allWs != null && allWs.size() > 0) { + sb.append("Current thread workspaces:\n"); + //Name, open, size, currently allocated + String wsFormat = " %-26s%-12s%-30s%-20s"; + sb.append(String.format(wsFormat, "Name", "State", "Size", "# Cycles")).append("\n"); + for (MemoryWorkspace ws : allWs) { + totalWsSize += ws.getCurrentSize(); + long numCycles = ws.getGenerationId(); + sb.append(String.format(wsFormat, ws.getId(), + (ws.isScopeActive() ? "OPEN" : "CLOSED"), + fBytes(ws.getCurrentSize()), + String.valueOf(numCycles))).append("\n"); + } + } + sb.append(fBytes("Workspaces total size", totalWsSize)); + + //JVM info + appendHeader(sb, "JVM Info"); + + appendProperty(sb, "Runtime Name", "java.runtime.name"); + appendProperty(sb, "Java Version", "java.version"); + appendProperty(sb, "Runtime Version", "java.runtime.version"); + appendProperty(sb, "Vendor", "java.vm.vendor"); + appendProperty(sb, "Vendor Url", "java.vendor.url"); + + sb.append("\n"); + + appendProperty(sb, "VM Name", "java.vm.name"); + appendProperty(sb, "VM Version", "java.vm.version"); + appendProperty(sb, "VM Specification Name", "java.vm.specification.name"); + + sb.append("\n"); + + appendProperty(sb, "Library Path", "java.library.path"); + + + //classpath + appendHeader(sb, "Classpath"); + ClassLoader cl = ClassLoader.getSystemClassLoader(); + + URL[] urls = null; + try{ + urls = ((URLClassLoader)cl).getURLs(); + } catch (ClassCastException e){ + try { + urls = ((URLClassLoader) SystemInfo.class.getClassLoader()).getURLs(); + } catch (ClassCastException e1){ + try{ + urls = ((URLClassLoader) (Thread.currentThread().getContextClassLoader())).getURLs(); + } catch (ClassCastException e2) { + sb.append("Can't cast class loader to URLClassLoader\n"); + } + } + } + + if(urls != null) { + for (URL url : urls) { + sb.append(url.getFile()).append("\n"); + } + } else { + sb.append("Using System property java.class.path\n"); + String[] cps = System.getProperty("java.class.path").split(";"); + for(String c : cps){ + sb.append(c).append("\n"); + } + } + + + //launch command + appendHeader(sb, "Launch Command"); + + try{ + // only works on Oracle JVMs + appendProperty(sb, "Launch Command", "sun.java.command"); + } catch (Exception e){ + appendField(sb, "Launch Command", "Not available on this JVM"); + } + + List inputArguments = ManagementFactory.getRuntimeMXBean().getInputArguments(); + appendField(sb, "JVM Arguments", inputArguments); + + + //system properties + appendHeader(sb, "System Properties"); + + Properties props = System.getProperties(); + for(Map.Entry prop : props.entrySet()){ + if(prop.getKey().toString().equals("line.separator")) { + sb.append(prop.getKey().toString()).append("=") + .append(prop.getValue().toString().replace("\\", "\\\\")).append("\n"); + } else { + sb.append(prop.getKey().toString()).append("=").append(prop.getValue()).append("\n"); + } + } + + + //enviroment variables + appendHeader(sb, "Environment Variables"); + + Map env = System.getenv(); + + for(String key : env.keySet()){ + sb.append(key).append("=").append(env.get(key)).append("\n"); + } + + return sb.toString(); + } + + /** + * Writes system info to the given file + */ + public static void writeSystemInfo(File file){ + try { + file.createNewFile(); + FileUtils.writeStringToFile(file, getSystemInfo()); + } catch (IOException e) { + throw new RuntimeException("IOException:" + e.getMessage(), e); + } + } + + /** + * Prints system info + */ + public static void printSystemInfo(){ + System.out.println(getSystemInfo()); + } + + public static Pair inferVersion(){ + List vi = VersionCheck.getVersionInfos(); + + String dl4jVersion = null; + String dl4jCudaArtifact = null; + for(VersionInfo v : vi){ + if("org.deeplearning4j".equals(v.getGroupId()) && "deeplearning4j-core".equals(v.getArtifactId())){ + String version = v.getBuildVersion(); + if(version.contains("SNAPSHOT")){ + dl4jVersion = version + " (" + v.getCommitIdAbbrev() + ")"; + } + dl4jVersion = version; + } else if("org.deeplearning4j".equals(v.getGroupId()) && v.getArtifactId() != null && v.getArtifactId().contains("deeplearning4j-cuda")){ + dl4jCudaArtifact = v.getArtifactId(); + } + + } + + return new Pair<>(dl4jVersion, dl4jCudaArtifact); + } +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeOpsGPUInfoProvider.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeOpsGPUInfoProvider.java new file mode 100644 index 000000000..777f5dff0 --- /dev/null +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeOpsGPUInfoProvider.java @@ -0,0 +1,55 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.nd4j.nativeblas; + +import java.util.ArrayList; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.nd4j.systeminfo.GPUInfo; +import org.nd4j.systeminfo.GPUInfoProvider; + +@Slf4j +public class NativeOpsGPUInfoProvider implements GPUInfoProvider { + + @Override + public List getGPUs() { + NativeOps nativeOps = NativeOpsHolder.getInstance().getDeviceNativeOps(); + + List gpus = new ArrayList<>(); + + + int nDevices = nativeOps.getAvailableDevices(); + if (nDevices > 0) { + for (int i = 0; i < nDevices; i++) { + try { + String name = nativeOps.getDeviceName(i); + long total = nativeOps.getDeviceTotalMemory(i); + long free = nativeOps.getDeviceFreeMemory(i); + int major = nativeOps.getDeviceMajor(i); + int minor = nativeOps.getDeviceMinor(i); + + gpus.add(new GPUInfo(name, total, free, major, minor)); + } catch (Exception e) { + log.warn("Can't add GPU", e); + } + } + } + + return gpus; + } + +} diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeOpsHolder.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeOpsHolder.java index b9cfe0cc2..4dc07045d 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeOpsHolder.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/java/org/nd4j/nativeblas/NativeOpsHolder.java @@ -16,17 +16,15 @@ package org.nd4j.nativeblas; +import java.util.Properties; import lombok.Getter; import org.bytedeco.javacpp.Loader; -import org.bytedeco.javacpp.Pointer; import org.nd4j.config.ND4JEnvironmentVars; import org.nd4j.context.Nd4jContext; import org.nd4j.linalg.factory.Nd4j; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Properties; - /** * @author raver119@gmail.com * @author saudet @@ -74,5 +72,4 @@ public class NativeOpsHolder { public static NativeOpsHolder getInstance() { return INSTANCE; } - } diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/resources/META-INF/services/org.nd4j.systeminfo.GPUInfoProvider b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/resources/META-INF/services/org.nd4j.systeminfo.GPUInfoProvider new file mode 100644 index 000000000..cd932c4f0 --- /dev/null +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-native-api/src/main/resources/META-INF/services/org.nd4j.systeminfo.GPUInfoProvider @@ -0,0 +1,17 @@ +################################################################################ +# Copyright (c) 2015-2019 Skymind, Inc. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 +################################################################################ + +org.nd4j.nativeblas.NativeOpsGPUInfoProvider \ No newline at end of file diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/systeminfo/TestSystemInfo.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/systeminfo/TestSystemInfo.java new file mode 100644 index 000000000..764fcbd23 --- /dev/null +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/systeminfo/TestSystemInfo.java @@ -0,0 +1,27 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.nd4j.systeminfo; + +import org.junit.Test; +import org.nd4j.systeminfo.SystemInfo; + +public class TestSystemInfo { + @Test + public void testSystemInfo(){ + SystemInfo.printSystemInfo(); + } +}