From 3c28caa52aae9f9903770a21c7f352caca7ca465 Mon Sep 17 00:00:00 2001 From: Alex Black Date: Sun, 26 Apr 2020 18:32:49 +1000 Subject: [PATCH] Add ResourceUtils.listClassPathFiles (#416) Signed-off-by: Alex Black --- nd4j/nd4j-common-tests/pom.xml | 9 +- .../org/nd4j/common/tests/ResourceUtils.java | 122 ++++++++++++++++++ .../java/org/nd4j/resources/Resolver.java | 7 + .../java/org/nd4j/resources/Resources.java | 16 +++ .../resources/strumpf/StrumpfResolver.java | 8 ++ 5 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/ResourceUtils.java diff --git a/nd4j/nd4j-common-tests/pom.xml b/nd4j/nd4j-common-tests/pom.xml index 9f1765e21..ab280ae0a 100644 --- a/nd4j/nd4j-common-tests/pom.xml +++ b/nd4j/nd4j-common-tests/pom.xml @@ -35,7 +35,6 @@ org.reflections reflections ${reflections.version} - com.google.code.findbugs @@ -43,6 +42,14 @@ + + + + org.springframework + spring-core + 5.0.2.RELEASE + + diff --git a/nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/ResourceUtils.java b/nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/ResourceUtils.java new file mode 100644 index 000000000..95c09c154 --- /dev/null +++ b/nd4j/nd4j-common-tests/src/main/java/org/nd4j/common/tests/ResourceUtils.java @@ -0,0 +1,122 @@ +/* ****************************************************************************** + * Copyright (c) 2019 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ +package org.nd4j.common.tests; + +import org.nd4j.linalg.io.ClassPathResource; +import org.nd4j.resources.Resources; +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; +import org.springframework.core.io.support.ResourcePatternResolver; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Utilities for dealing with class path resources such as test files in JARs + * + * @author Alex Black + */ +public class ResourceUtils { + + private ResourceUtils() { + } + + /** + * List all classpath resource files, optionally recursively, inside the specified path/directory + * The path argument should be a directory. + * Returns the path of the resources, normalized by {@link Resources#normalize(String)} + * + * @param path Path in which to list all files + * @param recursive If true: list all files in subdirectories also. If false: only include files in the specified + * directory, but not any files in subdirectories + * @param includeDirectories If true: include any subdirectories in the returned list of files. False: Only return + * files, not directories + * @param extensions Optional - may be null (or length 0). If null/length 0: files with any extension are returned + * If non-null: only files matching one of the specified extensions are included. + * Extensions can we specified with or without "." - i.e., "csv" and ".csv" are the same + * @return List of files (and optionally directories) in the specified path + */ + public static List listClassPathFiles(String path, boolean recursive, boolean includeDirectories, String... extensions) { + try { + return listClassPathFilesHelper(path, recursive, includeDirectories, extensions); + } catch (IOException e) { + throw new RuntimeException("Error listing class path files", e); + } + } + + private static List listClassPathFilesHelper(String path, boolean recursive, boolean includeDirectories, String... extensions) throws IOException { + ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(new ClassPathResource(path).getClassLoader()); + + StringBuilder sbPattern = new StringBuilder("classpath*:" + path); + if (recursive) { + sbPattern.append("/**/*"); + } else { + sbPattern.append("/*"); + } + + //Normalize extensions so they are all like ".csv" etc - with leading "." + String[] normExt = null; + if (extensions != null && extensions.length > 0) { + normExt = new String[extensions.length]; + for (int i = 0; i < extensions.length; i++) { + if (!extensions[i].startsWith(".")) { + normExt[i] = "." + extensions[i]; + } else { + normExt[i] = extensions[i]; + } + } + } + + String pattern = sbPattern.toString(); + Resource[] resources = resolver.getResources(pattern); + List out = new ArrayList<>(resources.length); + for (Resource r : resources) { + String origPath = r.getURL().toString(); + int idx = origPath.indexOf(path); + String relativePath = origPath.substring(idx); + String resourcePath = Resources.normalizePath(relativePath); + + + if (resourcePath.endsWith("/")) { + if (includeDirectories) { + out.add(resourcePath); + } else { + continue; //Skip directory + } + } + + if (normExt != null) { + //Check if it matches any of the specified extensions + boolean matches = false; + for (String ext : normExt) { + if (resourcePath.endsWith(ext)) { + matches = true; + break; + } + } + if (matches) { + out.add(resourcePath); + } + } else { + //Include all files + out.add(resourcePath); + } + + } + return out; + } +} diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/resources/Resolver.java b/nd4j/nd4j-common/src/main/java/org/nd4j/resources/Resolver.java index 51e75dc5d..c25b7d123 100644 --- a/nd4j/nd4j-common/src/main/java/org/nd4j/resources/Resolver.java +++ b/nd4j/nd4j-common/src/main/java/org/nd4j/resources/Resolver.java @@ -69,4 +69,11 @@ public interface Resolver { */ File localCacheRoot(); + /** + * Normalize the path that may be a resource reference. + * For example: "someDir/myFile.zip.resource_reference" --> "someDir/myFile.zip" + * Returns null if the file cannot be resolved. + * If the file is not a reference, the original path is returned + */ + String normalizePath(String path); } diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/resources/Resources.java b/nd4j/nd4j-common/src/main/java/org/nd4j/resources/Resources.java index b9e8253e3..14088f7c3 100644 --- a/nd4j/nd4j-common/src/main/java/org/nd4j/resources/Resources.java +++ b/nd4j/nd4j-common/src/main/java/org/nd4j/resources/Resources.java @@ -87,6 +87,15 @@ public class Resources { INSTANCE.copyDir(directoryPath, destinationDir); } + /** + * Normalize the path that may be a resource reference. + * For example: "someDir/myFile.zip.resource_reference" --> "someDir/myFile.zip" + * Returns null if the file cannot be resolved. + * If the file is not a reference, the original path is returned + */ + public static String normalizePath(String path){ + return INSTANCE.normalize(path); + } protected boolean resourceExists(String resourcePath) { for (Resolver r : resolvers) { @@ -128,4 +137,11 @@ public class Resources { } } + public String normalize(String path){ + for(Resolver r : resolvers){ + path = r.normalizePath(path); + } + return path; + } + } diff --git a/nd4j/nd4j-common/src/main/java/org/nd4j/resources/strumpf/StrumpfResolver.java b/nd4j/nd4j-common/src/main/java/org/nd4j/resources/strumpf/StrumpfResolver.java index a9d2904ed..07ab95676 100644 --- a/nd4j/nd4j-common/src/main/java/org/nd4j/resources/strumpf/StrumpfResolver.java +++ b/nd4j/nd4j-common/src/main/java/org/nd4j/resources/strumpf/StrumpfResolver.java @@ -258,6 +258,14 @@ public class StrumpfResolver implements Resolver { return cacheDir; } + @Override + public String normalizePath(@NonNull String path) { + if(path.endsWith(REF)){ + return path.substring(0, path.length()-REF.length()); + } + return path; + } + protected void assertExists(String resourcePath) { if (!exists(resourcePath)) {