From bde0a4ec98d9a20cda19b25bb214ae49356647ec Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Thu, 21 May 2020 05:47:12 +0400 Subject: [PATCH] Python4j (#422) * types * pom fix * basic exec + tests * safe exec * exec fixes + tests * prim tests * lists and dicts * collections tests * list test * api * exec and return all vars * context manager + fixes * leak fixes * jobs tests * gc basic working * more gc fixed * copyright headers * try-catch-finally * gc fixes * validate var name (startswith _collapsed..) * try block refac * pythonexecutioner nits * hashset->set * call() gc fix * gc fixes * type check fix * types fixes * refacs * rem numpyarray * threadsafety check * private->public * threadsafe checks * pythonGC test * threading fixes + tests * threading tests+ * threading test fixes * make PythonException unchecked * nits * docstrings * path fixes --- pom.xml | 1 + python4j/pom.xml | 66 ++ python4j/python4j-core/pom.xml | 44 ++ .../java/org/eclipse/python4j/Python.java | 611 ++++++++++++++++++ .../python4j/PythonContextManager.java | 241 +++++++ .../org/eclipse/python4j/PythonException.java | 52 ++ .../eclipse/python4j/PythonExecutioner.java | 342 ++++++++++ .../java/org/eclipse/python4j/PythonGC.java | 137 ++++ .../java/org/eclipse/python4j/PythonGIL.java | 93 +++ .../java/org/eclipse/python4j/PythonJob.java | 175 +++++ .../org/eclipse/python4j/PythonObject.java | 244 +++++++ .../java/org/eclipse/python4j/PythonType.java | 47 ++ .../org/eclipse/python4j/PythonTypes.java | 344 ++++++++++ .../org/eclipse/python4j/PythonVariable.java | 64 ++ .../eclipse/python4j/pythonexec/pythonexec.py | 36 ++ .../test/java/PythonBasicExecutionTest.java | 108 ++++ .../src/test/java/PythonCollectionsTest.java | 62 ++ .../test/java/PythonContextManagerTest.java | 51 ++ .../src/test/java/PythonGCTest.java | 54 ++ .../src/test/java/PythonJobTest.java | 287 ++++++++ .../src/test/java/PythonMultiThreadTest.java | 169 +++++ .../test/java/PythonPrimitiveTypesTest.java | 82 +++ python4j/python4j-numpy/pom.xml | 42 ++ 23 files changed, 3352 insertions(+) create mode 100644 python4j/pom.xml create mode 100644 python4j/python4j-core/pom.xml create mode 100644 python4j/python4j-core/src/main/java/org/eclipse/python4j/Python.java create mode 100644 python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonContextManager.java create mode 100644 python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonException.java create mode 100644 python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonExecutioner.java create mode 100644 python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonGC.java create mode 100644 python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonGIL.java create mode 100644 python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonJob.java create mode 100644 python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonObject.java create mode 100644 python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonType.java create mode 100644 python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonTypes.java create mode 100644 python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonVariable.java create mode 100644 python4j/python4j-core/src/main/resources/org/eclipse/python4j/pythonexec/pythonexec.py create mode 100644 python4j/python4j-core/src/test/java/PythonBasicExecutionTest.java create mode 100644 python4j/python4j-core/src/test/java/PythonCollectionsTest.java create mode 100644 python4j/python4j-core/src/test/java/PythonContextManagerTest.java create mode 100644 python4j/python4j-core/src/test/java/PythonGCTest.java create mode 100644 python4j/python4j-core/src/test/java/PythonJobTest.java create mode 100644 python4j/python4j-core/src/test/java/PythonMultiThreadTest.java create mode 100644 python4j/python4j-core/src/test/java/PythonPrimitiveTypesTest.java create mode 100644 python4j/python4j-numpy/pom.xml diff --git a/pom.xml b/pom.xml index ab9f80b92..184eeb11f 100644 --- a/pom.xml +++ b/pom.xml @@ -137,6 +137,7 @@ jumpy pydatavec pydl4j + python4j diff --git a/python4j/pom.xml b/python4j/pom.xml new file mode 100644 index 000000000..57af8f1bb --- /dev/null +++ b/python4j/pom.xml @@ -0,0 +1,66 @@ + + + + + + deeplearning4j + org.deeplearning4j + 1.0.0-SNAPSHOT + + 4.0.0 + + org.eclipse + python4j-parent + pom + + python4j-core + python4j-numpy + + + + + org.projectlombok + lombok + ${lombok.version} + provided + + + ch.qos.logback + logback-classic + ${logback.version} + test + + + junit + junit + ${junit.version} + test + + + commons-io + commons-io + ${commons-io.version} + + + com.google.code.findbugs + jsr305 + 3.0.2 + + + \ No newline at end of file diff --git a/python4j/python4j-core/pom.xml b/python4j/python4j-core/pom.xml new file mode 100644 index 000000000..b429d8272 --- /dev/null +++ b/python4j/python4j-core/pom.xml @@ -0,0 +1,44 @@ + + + + + + + python4j-parent + org.eclipse + 1.0.0-SNAPSHOT + + jar + 4.0.0 + + python4j-core + + + org.json + json + 20190722 + + + org.bytedeco + cpython-platform + ${cpython-platform.version} + + + + \ No newline at end of file diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/Python.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/Python.java new file mode 100644 index 000000000..fd6fff112 --- /dev/null +++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/Python.java @@ -0,0 +1,611 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + + +package org.eclipse.python4j; + +import org.bytedeco.cpython.PyObject; + +import java.util.Collections; +import java.util.List; + +import static org.bytedeco.cpython.global.python.*; + + +public class Python { + + static { + new PythonExecutioner(); + } + + /** + * Imports a python module, similar to python import statement. + * + * @param moduleName name of the module to be imported + * @return reference to the module object + */ + public static PythonObject importModule(String moduleName) { + PythonGIL.assertThreadSafe(); + PythonObject module = new PythonObject(PyImport_ImportModule(moduleName)); + if (module.isNone()) { + throw new PythonException("Error importing module: " + moduleName); + } + return module; + } + + /** + * Gets a builtins attribute + * + * @param attrName Attribute name + * @return + */ + public static PythonObject attr(String attrName) { + PythonGIL.assertThreadSafe(); + PyObject builtins = PyImport_ImportModule("builtins"); + try { + return new PythonObject(PyObject_GetAttrString(builtins, attrName)); + } finally { + Py_DecRef(builtins); + } + } + + + /** + * Gets the size of a PythonObject. similar to len() in python. + * + * @param pythonObject + * @return + */ + public static PythonObject len(PythonObject pythonObject) { + PythonGIL.assertThreadSafe(); + long n = PyObject_Size(pythonObject.getNativePythonObject()); + if (n < 0) { + throw new PythonException("Object has no length: " + pythonObject); + } + return PythonTypes.INT.toPython(n); + } + + /** + * Gets the string representation of an object. + * + * @param pythonObject + * @return + */ + public static PythonObject str(PythonObject pythonObject) { + PythonGIL.assertThreadSafe(); + try { + return PythonTypes.STR.toPython(pythonObject.toString()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + + } + + /** + * Returns an empty string + * + * @return + */ + public static PythonObject str() { + PythonGIL.assertThreadSafe(); + try { + return PythonTypes.STR.toPython(""); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Returns the str type object + * @return + */ + public static PythonObject strType() { + return attr("str"); + } + + /** + * Returns a floating point number from a number or a string. + * @param pythonObject + * @return + */ + public static PythonObject float_(PythonObject pythonObject) { + return PythonTypes.FLOAT.toPython(PythonTypes.FLOAT.toJava(pythonObject)); + } + + /** + * Reutrns 0. + * @return + */ + public static PythonObject float_() { + try { + return PythonTypes.FLOAT.toPython(0d); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + /** + * Returns the float type object + * @return + */ + public static PythonObject floatType() { + return attr("float"); + } + + + /** + * Converts a value to a Boolean value i.e., True or False, using the standard truth testing procedure. + * @param pythonObject + * @return + */ + public static PythonObject bool(PythonObject pythonObject) { + return PythonTypes.BOOL.toPython(PythonTypes.BOOL.toJava(pythonObject)); + + } + + /** + * Returns False. + * @return + */ + public static PythonObject bool() { + return PythonTypes.BOOL.toPython(false); + + } + + /** + * Returns the bool type object + * @return + */ + public static PythonObject boolType() { + return attr("bool"); + } + + /** + * Returns an integer from a number or a string. + * @param pythonObject + * @return + */ + public static PythonObject int_(PythonObject pythonObject) { + return PythonTypes.INT.toPython(PythonTypes.INT.toJava(pythonObject)); + } + + /** + * Returns 0 + * @return + */ + public static PythonObject int_() { + return PythonTypes.INT.toPython(0L); + + } + + /** + * Returns the int type object + * @return + */ + public static PythonObject intType() { + return attr("int"); + } + + /** + * Takes sequence types and converts them to lists. + * @param pythonObject + * @return + */ + public static PythonObject list(PythonObject pythonObject) { + PythonGIL.assertThreadSafe(); + try (PythonGC _ = PythonGC.watch()) { + PythonObject listF = attr("list"); + PythonObject ret = listF.call(pythonObject); + if (ret.isNone()) { + throw new PythonException("Object is not iterable: " + pythonObject.toString()); + } + return ret; + } + } + + /** + * Returns empty list. + * @return + */ + public static PythonObject list() { + return PythonTypes.LIST.toPython(Collections.emptyList()); + } + + /** + * Returns list type object. + * @return + */ + public static PythonObject listType() { + return attr("list"); + } + + /** + * Creates a dictionary. + * @param pythonObject + * @return + */ + public static PythonObject dict(PythonObject pythonObject) { + PythonObject dictF = attr("dict"); + PythonObject ret = dictF.call(pythonObject); + if (ret.isNone()) { + throw new PythonException("Cannot build dict from object: " + pythonObject.toString()); + } + dictF.del(); + return ret; + } + + /** + * Returns empty dict + * @return + */ + public static PythonObject dict() { + return PythonTypes.DICT.toPython(Collections.emptyMap()); + } + + /** + * Returns dict type object. + * @return + */ + public static PythonObject dictType() { + return attr("dict"); + } + + /** + * Creates a set. + * @param pythonObject + * @return + */ + public static PythonObject set(PythonObject pythonObject) { + PythonObject setF = attr("set"); + PythonObject ret = setF.call(pythonObject); + if (ret.isNone()) { + throw new PythonException("Cannot build set from object: " + pythonObject.toString()); + } + setF.del(); + return ret; + } + + /** + * Returns empty set. + * @return + */ + public static PythonObject set() { + PythonObject setF = attr("set"); + PythonObject ret; + ret = setF.call(); + setF.del(); + return ret; + } + + /** + * Returns empty set. + * @return + */ + public static PythonObject setType() { + return attr("set"); + } + + /** + * Creates a bytearray. + * @param pythonObject + * @return + */ + public static PythonObject bytearray(PythonObject pythonObject) { + PythonObject baF = attr("bytearray"); + PythonObject ret = baF.call(pythonObject); + if (ret.isNone()) { + throw new PythonException("Cannot build bytearray from object: " + pythonObject.toString()); + } + baF.del(); + return ret; + } + + /** + * Returns empty bytearray. + * @return + */ + public static PythonObject bytearray() { + PythonObject baF = attr("bytearray"); + PythonObject ret; + ret = baF.call(); + baF.del(); + return ret; + } + + /** + * Returns bytearray type object + * @return + */ + public static PythonObject bytearrayType() { + return attr("bytearray"); + } + + /** + * Creates a memoryview. + * @param pythonObject + * @return + */ + public static PythonObject memoryview(PythonObject pythonObject) { + PythonObject mvF = attr("memoryview"); + PythonObject ret = mvF.call(pythonObject); + if (ret.isNone()) { + throw new PythonException("Cannot build memoryview from object: " + pythonObject.toString()); + } + mvF.del(); + return ret; + } + + /** + * Returns memoryview type object. + * @return + */ + public static PythonObject memoryviewType() { + return attr("memoryview"); + } + + /** + * Creates a byte string. + * @param pythonObject + * @return + */ + public static PythonObject bytes(PythonObject pythonObject) { + PythonObject bytesF = attr("bytes"); + PythonObject ret = bytesF.call(pythonObject); + if (ret.isNone()) { + throw new PythonException("Cannot build bytes from object: " + pythonObject.toString()); + } + bytesF.del(); + return ret; + } + + /** + * Returns empty byte string. + * @return + */ + public static PythonObject bytes() { + PythonObject bytesF = attr("bytes"); + PythonObject ret; + ret = bytesF.call(); + bytesF.del(); + return ret; + } + + /** + * Returns bytes type object + * @return + */ + public static PythonObject bytesType() { + return attr("bytes"); + } + + /** + * Creates a tuple. + * @param pythonObject + * @return + */ + public static PythonObject tuple(PythonObject pythonObject) { + PythonObject tupleF = attr("tupleF"); + PythonObject ret = tupleF.call(pythonObject); + if (ret.isNone()) { + throw new PythonException("Cannot build tuple from object: " + pythonObject.toString()); + } + tupleF.del(); + return ret; + } + + /** + * Returns empty tuple. + * @return + */ + public static PythonObject tuple() { + PythonObject tupleF = attr("tuple"); + PythonObject ret; + ret = tupleF.call(); + tupleF.del(); + return ret; + } + + /** + * Returns tuple type object + * @return + */ + public static PythonObject tupleType() { + return attr("tuple"); + } + + /** + * Creates an Exception + * @param pythonObject + * @return + */ + public static PythonObject Exception(PythonObject pythonObject) { + PythonObject excF = attr("Exception"); + PythonObject ret = excF.call(pythonObject); + excF.del(); + return ret; + } + + /** + * Creates an Exception + * @return + */ + public static PythonObject Exception() { + PythonObject excF = attr("Exception"); + PythonObject ret; + ret = excF.call(); + excF.del(); + return ret; + } + + /** + * Returns Exception type object + * @return + */ + public static PythonObject ExceptionType() { + return attr("Exception"); + } + + + /** + * Returns the globals dictionary. + * @return + */ + public static PythonObject globals() { + PythonGIL.assertThreadSafe(); + PyObject main = PyImport_ImportModule("__main__"); + PyObject globals = PyModule_GetDict(main); + Py_DecRef(main); + return new PythonObject(globals, false); + } + + /** + * Returns the type of an object. + * @param pythonObject + * @return + */ + public static PythonObject type(PythonObject pythonObject) { + PythonObject typeF = attr("type"); + PythonObject ret = typeF.call(pythonObject); + typeF.del(); + return ret; + } + + /** + * Returns True if the specified object is of the specified type, otherwise False. + * @param obj + * @param type + * @return + */ + public static boolean isinstance(PythonObject obj, PythonObject... type) { + PythonGIL.assertThreadSafe(); + PyObject argsTuple = PyTuple_New(type.length); + try { + for (int i = 0; i < type.length; i++) { + PythonObject x = type[i]; + Py_IncRef(x.getNativePythonObject()); + PyTuple_SetItem(argsTuple, i, x.getNativePythonObject()); + } + return PyObject_IsInstance(obj.getNativePythonObject(), argsTuple) != 0; + } finally { + Py_DecRef(argsTuple); + } + + } + + /** + * Evaluates the specified expression. + * @param expression + * @return + */ + public static PythonObject eval(String expression) { + + PythonGIL.assertThreadSafe(); + PyObject compiledCode = Py_CompileString(expression, "", Py_eval_input); + PyObject main = PyImport_ImportModule("__main__"); + PyObject globals = PyModule_GetDict(main); + PyObject locals = PyDict_New(); + try { + return new PythonObject(PyEval_EvalCode(compiledCode, globals, locals)); + } finally { + Py_DecRef(main); + Py_DecRef(locals); + Py_DecRef(compiledCode); + } + + } + + /** + * Returns the builtins module + * @return + */ + public static PythonObject builtins() { + return importModule("builtins"); + + } + + /** + * Returns None. + * @return + */ + public static PythonObject None() { + return eval("None"); + } + + /** + * Returns True. + * @return + */ + public static PythonObject True() { + return eval("True"); + } + + /** + * Returns False. + * @return + */ + public static PythonObject False() { + return eval("False"); + } + + /** + * Returns True if the object passed is callable callable, otherwise False. + * @param pythonObject + * @return + */ + public static boolean callable(PythonObject pythonObject) { + PythonGIL.assertThreadSafe(); + return PyCallable_Check(pythonObject.getNativePythonObject()) == 1; + } + + + public static void setContext(String context){ + PythonContextManager.setContext(context); + } + + public static String getCurrentContext() { + return PythonContextManager.getCurrentContext(); + } + + public static void deleteContext(String context){ + PythonContextManager.deleteContext(context); + } + public static void resetContext() { + PythonContextManager.reset(); + } + + /** + * Executes a string of code. + * @param code + * @throws PythonException + */ + public static void exec(String code) throws PythonException { + PythonExecutioner.exec(code); + } + + /** + * Executes a string of code. + * @param code + * @param inputs + * @param outputs + */ + public static void exec(String code, List inputs, List outputs){ + PythonExecutioner.exec(code, inputs, outputs); + } + + +} diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonContextManager.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonContextManager.java new file mode 100644 index 000000000..a34d8a239 --- /dev/null +++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonContextManager.java @@ -0,0 +1,241 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.eclipse.python4j; + +import javax.lang.model.SourceVersion; + + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Emulates multiples interpreters in a single interpreter. + * This works by simply obfuscating/de-obfuscating variable names + * such that only the required subset of the global namespace is "visible" + * at any given time. + * By default, there exists a "main" context emulating the default interpreter + * + * @author Fariz Rahman + */ + + +public class PythonContextManager { + + private static Set contexts = new HashSet<>(); + private static AtomicBoolean init = new AtomicBoolean(false); + private static String currentContext; + private static final String MAIN_CONTEXT = "main"; + private static final String COLLAPSED_KEY = "__collapsed__"; + + static { + init(); + } + + private static void init() { + if (init.get()) return; + new PythonExecutioner(); + init.set(true); + currentContext = MAIN_CONTEXT; + contexts.add(currentContext); + } + + + /** + * Adds a new context. + * @param contextName + */ + public static void addContext(String contextName) { + if (!validateContextName(contextName)) { + throw new PythonException("Invalid context name: " + contextName); + } + contexts.add(contextName); + } + + /** + * Returns true if context exists, else false. + * @param contextName + * @return + */ + public static boolean hasContext(String contextName) { + return contexts.contains(contextName); + } + + private static boolean validateContextName(String s) { + return SourceVersion.isIdentifier(s) && !s.startsWith(COLLAPSED_KEY); + } + + private static String getContextPrefix(String contextName) { + return COLLAPSED_KEY + contextName + "__"; + } + + private static String getCollapsedVarNameForContext(String varName, String contextName) { + return getContextPrefix(contextName) + varName; + } + + private static String expandCollapsedVarName(String varName, String contextName) { + String prefix = COLLAPSED_KEY + contextName + "__"; + return varName.substring(prefix.length()); + + } + + private static void collapseContext(String contextName) { + try (PythonGC _ = PythonGC.watch()) { + PythonObject globals = Python.globals(); + PythonObject pop = globals.attr("pop"); + PythonObject keysF = globals.attr("keys"); + PythonObject keys = keysF.call(); + PythonObject keysList = Python.list(keys); + int numKeys = Python.len(keysList).toInt(); + for (int i = 0; i < numKeys; i++) { + PythonObject key = keysList.get(i); + String keyStr = key.toString(); + if (!((keyStr.startsWith("__") && keyStr.endsWith("__")) || keyStr.startsWith("__collapsed_"))) { + String collapsedKey = getCollapsedVarNameForContext(keyStr, contextName); + PythonObject val = pop.call(key); + + PythonObject pyNewKey = new PythonObject(collapsedKey); + globals.set(pyNewKey, val); + } + } + } catch (Exception pe) { + throw new RuntimeException(pe); + } + } + + private static void expandContext(String contextName) { + try (PythonGC _ = PythonGC.watch()) { + String prefix = getContextPrefix(contextName); + PythonObject globals = Python.globals(); + PythonObject pop = globals.attr("pop"); + PythonObject keysF = globals.attr("keys"); + + PythonObject keys = keysF.call(); + + PythonObject keysList = Python.list(keys); + try (PythonGC __ = PythonGC.pause()) { + int numKeys = Python.len(keysList).toInt(); + + for (int i = 0; i < numKeys; i++) { + PythonObject key = keysList.get(i); + String keyStr = key.toString(); + if (keyStr.startsWith(prefix)) { + String expandedKey = expandCollapsedVarName(keyStr, contextName); + PythonObject val = pop.call(key); + PythonObject newKey = new PythonObject(expandedKey); + globals.set(newKey, val); + } + } + } + } + } + + + /** + * Activates the specified context + * @param contextName + */ + public static void setContext(String contextName) { + if (contextName.equals(currentContext)) { + return; + } + if (!hasContext(contextName)) { + addContext(contextName); + } + + + collapseContext(currentContext); + + expandContext(contextName); + currentContext = contextName; + + } + + /** + * Activates the main context + */ + public static void setMainContext() { + setContext(MAIN_CONTEXT); + + } + + /** + * Returns the current context's name. + * @return + */ + public static String getCurrentContext() { + return currentContext; + } + + /** + * Resets the current context. + */ + public static void reset() { + String tempContext = "___temp__context___"; + String currContext = currentContext; + setContext(tempContext); + deleteContext(currContext); + setContext(currContext); + } + + /** + * Deletes the specified context. + * @param contextName + */ + public static void deleteContext(String contextName) { + if (contextName.equals(currentContext)) { + throw new PythonException("Cannot delete current context!"); + } + if (!contexts.contains(contextName)) { + return; + } + String prefix = getContextPrefix(contextName); + PythonObject globals = Python.globals(); + PythonObject keysList = Python.list(globals.attr("keys").call()); + int numKeys = Python.len(keysList).toInt(); + for (int i = 0; i < numKeys; i++) { + PythonObject key = keysList.get(i); + String keyStr = key.toString(); + if (keyStr.startsWith(prefix)) { + globals.attr("__delitem__").call(key); + } + } + contexts.remove(contextName); + } + + /** + * Deletes all contexts except the main context. + */ + public static void deleteNonMainContexts() { + setContext(MAIN_CONTEXT); // will never fail + for (String c : contexts.toArray(new String[0])) { + if (!c.equals(MAIN_CONTEXT)) { + deleteContext(c); // will never fail + } + } + + } + + /** + * Returns the names of all contexts. + * @return + */ + public String[] getContexts() { + return contexts.toArray(new String[0]); + } + +} diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonException.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonException.java new file mode 100644 index 000000000..a9bbf596c --- /dev/null +++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonException.java @@ -0,0 +1,52 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.eclipse.python4j; + + +/** + * Thrown when an exception occurs in python land + */ +public class PythonException extends RuntimeException { + public PythonException(String message) { + super(message); + } + + private static String getExceptionString(PythonObject exception) { + try (PythonGC gc = PythonGC.watch()) { + if (Python.isinstance(exception, Python.ExceptionType())) { + String exceptionClass = Python.type(exception).attr("__name__").toString(); + String message = exception.toString(); + return exceptionClass + ": " + message; + } + return exception.toString(); + } catch (Exception e) { + throw new RuntimeException("An error occurred while trying to create a PythonException.", e); + } + } + + public PythonException(PythonObject exception) { + this(getExceptionString(exception)); + } + + public PythonException(String message, Throwable cause) { + super(message, cause); + } + + public PythonException(Throwable cause) { + super(cause); + } +} diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonExecutioner.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonExecutioner.java new file mode 100644 index 000000000..57e1a22ae --- /dev/null +++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonExecutioner.java @@ -0,0 +1,342 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + + +package org.eclipse.python4j; + +import org.bytedeco.cpython.PyObject; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.io.IOUtils; +import org.bytedeco.cpython.global.python; + +import static org.bytedeco.cpython.global.python.*; +import static org.bytedeco.cpython.global.python.PyImport_ImportModule; +import static org.bytedeco.cpython.helper.python.Py_SetPath; + + +public class PythonExecutioner { + private final static String PYTHON_EXCEPTION_KEY = "__python_exception__"; + private static AtomicBoolean init = new AtomicBoolean(false); + private final static String DEFAULT_PYTHON_PATH_PROPERTY = "org.eclipse.python4j.path"; + private final static String JAVACPP_PYTHON_APPEND_TYPE = "org.eclipse.python4j.path.append"; + private final static String DEFAULT_APPEND_TYPE = "before"; + + static { + init(); + } + + private static synchronized void init() { + if (init.get()) { + return; + } + init.set(true); + initPythonPath(); + PyEval_InitThreads(); + Py_InitializeEx(0); + } + + /** + * Sets a variable. + * + * @param name + * @param value + */ + public static void setVariable(String name, PythonObject value) { + PythonGIL.assertThreadSafe(); + PyObject main = PyImport_ImportModule("__main__"); + PyObject globals = PyModule_GetDict(main); + PyDict_SetItemString(globals, name, value.getNativePythonObject()); + Py_DecRef(main); + + } + + /** + * Sets given list of PythonVariables in the interpreter. + * + * @param pyVars + */ + public static void setVariables(List pyVars) { + for (PythonVariable pyVar : pyVars) + setVariable(pyVar.getName(), pyVar.getPythonObject()); + } + + /** + * Sets given list of PythonVariables in the interpreter. + * + * @param pyVars + */ + public static void setVariables(PythonVariable... pyVars) { + setVariables(Arrays.asList(pyVars)); + } + + /** + * Gets the given list of PythonVariables from the interpreter. + * + * @param pyVars + */ + public static void getVariables(List pyVars) { + for (PythonVariable pyVar : pyVars) + pyVar.setValue(getVariable(pyVar.getName(), pyVar.getType()).getValue()); + } + + /** + * Gets the given list of PythonVariables from the interpreter. + * + * @param pyVars + */ + public static void getVariables(PythonVariable... pyVars) { + getVariables(Arrays.asList(pyVars)); + } + + /** + * Gets the variable with the given name from the interpreter. + * + * @param name + * @return + */ + public static PythonObject getVariable(String name) { + PythonGIL.assertThreadSafe(); + PyObject main = PyImport_ImportModule("__main__"); + PyObject globals = PyModule_GetDict(main); + PyObject pyName = PyUnicode_FromString(name); + try { + if (PyDict_Contains(globals, pyName) == 1) { + return new PythonObject(PyObject_GetItem(globals, pyName), false); + } + } finally { + Py_DecRef(main); + //Py_DecRef(globals); + Py_DecRef(pyName); + } + return new PythonObject(null); + } + + /** + * Gets the variable with the given name from the interpreter. + * + * @param name + * @return + */ + public static PythonVariable getVariable(String name, PythonType type) { + PythonObject val = getVariable(name); + return new PythonVariable<>(name, type, type.toJava(val)); + } + + /** + * Executes a string of code + * + * @param code + */ + public static synchronized void simpleExec(String code) { + PythonGIL.assertThreadSafe(); + int result = PyRun_SimpleStringFlags(code, null); + if (result != 0) { + throw new PythonException("Execution failed, unable to retrieve python exception."); + } + } + + private static void throwIfExecutionFailed() { + PythonObject ex = getVariable(PYTHON_EXCEPTION_KEY); + if (ex != null && !ex.isNone() && !ex.toString().isEmpty()) { + setVariable(PYTHON_EXCEPTION_KEY, PythonTypes.STR.toPython("")); + throw new PythonException(ex); + } + } + + + private static String getWrappedCode(String code) { + + try (InputStream is = PythonExecutioner.class + .getResourceAsStream("pythonexec/pythonexec.py")) { + String base = IOUtils.toString(is, StandardCharsets.UTF_8); + String indentedCode = " " + code.replace("\n", "\n "); + String out = base.replace(" pass", indentedCode); + return out; + } catch (IOException e) { + throw new IllegalStateException("Unable to read python code!", e); + } + + } + + /** + * Executes a string of code. Throws PythonException if execution fails. + * + * @param code + */ + public static void exec(String code) { + simpleExec(getWrappedCode(code)); + throwIfExecutionFailed(); + } + + public static void exec(String code, List inputs, List outputs) { + if (inputs != null) { + setVariables(inputs.toArray(new PythonVariable[0])); + } + exec(code); + if (outputs != null) { + getVariables(outputs.toArray(new PythonVariable[0])); + } + } + + /** + * Return list of all supported variables in the interpreter. + * + * @return + */ + public static List getAllVariables() { + PythonGIL.assertThreadSafe(); + List ret = new ArrayList<>(); + PyObject main = PyImport_ImportModule("__main__"); + PyObject globals = PyModule_GetDict(main); + PyObject keys = PyDict_Keys(globals); + PyObject keysIter = PyObject_GetIter(keys); + try { + + long n = PyObject_Size(globals); + for (int i = 0; i < n; i++) { + PyObject pyKey = PyIter_Next(keysIter); + try { + if (!new PythonObject(pyKey, false).toString().startsWith("_")) { + + PyObject pyVal = PyObject_GetItem(globals, pyKey); // TODO check ref count + PythonType pt; + try { + pt = PythonTypes.getPythonTypeForPythonObject(new PythonObject(pyVal, false)); + + } catch (PythonException pe) { + pt = null; + } + if (pt != null) { + ret.add( + new PythonVariable<>( + new PythonObject(pyKey, false).toString(), + pt, + pt.toJava(new PythonObject(pyVal, false)) + ) + ); + } + } + } finally { + Py_DecRef(pyKey); + } + } + } finally { + Py_DecRef(keysIter); + Py_DecRef(keys); + Py_DecRef(main); + return ret; + } + + } + + + /** + * Executes a string of code and returns a list of all supported variables. + * + * @param code + * @param inputs + * @return + */ + public static List execAndReturnAllVariables(String code, List inputs) { + setVariables(inputs); + simpleExec(getWrappedCode(code)); + return getAllVariables(); + } + + /** + * Executes a string of code and returns a list of all supported variables. + * + * @param code + * @return + */ + public static List execAndReturnAllVariables(String code) { + simpleExec(getWrappedCode(code)); + return getAllVariables(); + } + + private static synchronized void initPythonPath() { + try { + String path = System.getProperty(DEFAULT_PYTHON_PATH_PROPERTY); + if (path == null) { + File[] packages = cachePackages(); + + //// TODO: fix in javacpp + File sitePackagesWindows = new File(python.cachePackage(), "site-packages"); + File[] packages2 = new File[packages.length + 1]; + for (int i = 0; i < packages.length; i++) { + //System.out.println(packages[i].getAbsolutePath()); + packages2[i] = packages[i]; + } + packages2[packages.length] = sitePackagesWindows; + //System.out.println(sitePackagesWindows.getAbsolutePath()); + packages = packages2; + ////////// + + Py_SetPath(packages); + } else { + StringBuffer sb = new StringBuffer(); + File[] packages = cachePackages(); + JavaCppPathType pathAppendValue = JavaCppPathType.valueOf(System.getProperty(JAVACPP_PYTHON_APPEND_TYPE, DEFAULT_APPEND_TYPE).toUpperCase()); + switch (pathAppendValue) { + case BEFORE: + for (File cacheDir : packages) { + sb.append(cacheDir); + sb.append(java.io.File.pathSeparator); + } + + sb.append(path); + break; + case AFTER: + sb.append(path); + + for (File cacheDir : packages) { + sb.append(cacheDir); + sb.append(java.io.File.pathSeparator); + } + break; + case NONE: + sb.append(path); + break; + } + + Py_SetPath(sb.toString()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private enum JavaCppPathType { + BEFORE, AFTER, NONE + } + + private static File[] cachePackages() throws IOException { + File[] path = org.bytedeco.cpython.global.python.cachePackages(); + path = Arrays.copyOf(path, path.length + 1); + path[path.length - 1] = cachePackage(); + return path; + } + +} diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonGC.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonGC.java new file mode 100644 index 000000000..5531b67d3 --- /dev/null +++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonGC.java @@ -0,0 +1,137 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + + +package org.eclipse.python4j; + +import org.bytedeco.cpython.PyObject; +import org.bytedeco.javacpp.Pointer; + +import java.io.Closeable; +import java.util.HashSet; +import java.util.Set; + +import static org.bytedeco.cpython.global.python.*; + +/** + * Wrap your code in a try-with-PythonGC block for automatic GC: + * ``` + * try(PythonGC gc = PythonGC.lock()){ + * // your code here + * } + * + * If a PythonObject created inside such a block has to be used outside + * the block, use PythonGC.keep() to exclude that object from GC. + * + * ``` + * PythonObject pyObj; + * + * try(PythonGC gc = PythonG.lock()){ + * // do stuff + * pyObj = someFunction(); + * PythonGC.keep(pyObj); + * } + * + */ +public class PythonGC implements Closeable { + + private PythonGC previousFrame = null; + private boolean active = true; + private static PythonGC currentFrame = new PythonGC(); + + private Set objects = new HashSet<>(); + + private boolean alreadyRegistered(PyObject pyObject) { + if (objects.contains(pyObject)) { + return true; + } + if (previousFrame == null) { + return false; + } + return previousFrame.alreadyRegistered(pyObject); + + } + + private void addObject(PythonObject pythonObject) { + if (!active) return; + if (Pointer.isNull(pythonObject.getNativePythonObject()))return; + if (alreadyRegistered(pythonObject.getNativePythonObject())) { + return; + } + objects.add(pythonObject.getNativePythonObject()); + } + + public static void register(PythonObject pythonObject) { + currentFrame.addObject(pythonObject); + } + + public static void keep(PythonObject pythonObject) { + currentFrame.objects.remove(pythonObject.getNativePythonObject()); + if (currentFrame.previousFrame != null) { + currentFrame.previousFrame.addObject(pythonObject); + } + } + + private PythonGC() { + } + + public static PythonGC watch() { + PythonGC ret = new PythonGC(); + ret.previousFrame = currentFrame; + ret.active = currentFrame.active; + currentFrame = ret; + return ret; + } + + private void collect() { + for (PyObject pyObject : objects) { + // TODO find out how globals gets collected here + if (pyObject.equals(Python.globals().getNativePythonObject())) continue; +// try{ +// System.out.println(PythonTypes.STR.toJava(new PythonObject(pyObject, false))); +// }catch (Exception e){} + Py_DecRef(pyObject); + + } + this.objects = new HashSet<>(); + } + + @Override + public void close() { + if (active) collect(); + currentFrame = previousFrame; + } + + public static boolean isWatching() { + if (!currentFrame.active) return false; + return currentFrame.previousFrame != null; + } + + public static PythonGC pause() { + PythonGC pausedFrame = new PythonGC(); + pausedFrame.active = false; + pausedFrame.previousFrame = currentFrame; + currentFrame = pausedFrame; + return pausedFrame; + } + + public static void resume() { + if (currentFrame.active) { + throw new RuntimeException("GC not paused!"); + } + currentFrame = currentFrame.previousFrame; + } +} diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonGIL.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonGIL.java new file mode 100644 index 000000000..46b3db431 --- /dev/null +++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonGIL.java @@ -0,0 +1,93 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.eclipse.python4j; + + +import org.bytedeco.cpython.PyThreadState; +import org.omg.SendingContext.RunTime; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.bytedeco.cpython.global.python.*; + + +public class PythonGIL implements AutoCloseable { + private static PyThreadState mainThreadState; + private static final AtomicBoolean acquired = new AtomicBoolean(); + private boolean acquiredByMe = false; + private static long defaultThreadId = -1; + + public static void assertThreadSafe() { + if (acquired.get()) { + return; + } + if (defaultThreadId == -1) { + defaultThreadId = Thread.currentThread().getId(); + } else if (defaultThreadId != Thread.currentThread().getId()) { + throw new RuntimeException("Attempt to use Python4j from multiple threads without " + + "acquiring GIL. Enclose your code in a try(PythonGIL gil = PythonGIL.lock()){...}" + + " block to ensure that GIL is acquired in multi-threaded environments."); + } + + + } + + static { + new PythonExecutioner(); + } + + private PythonGIL() { + while (acquired.get()) { + try { + Thread.sleep(10); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + acquire(); + acquired.set(true); + acquiredByMe = true; + + } + + @Override + public void close() { + if (acquiredByMe) { + release(); + acquired.set(false); + acquiredByMe = false; + } + + } + + public static synchronized PythonGIL lock() { + return new PythonGIL(); + } + + private static synchronized void acquire() { + mainThreadState = PyEval_SaveThread(); + PyThreadState ts = PyThreadState_New(mainThreadState.interp()); + PyEval_RestoreThread(ts); + PyThreadState_Swap(ts); + } + + private static void release() { // do not synchronize! + PyEval_SaveThread(); + PyEval_RestoreThread(mainThreadState); + } +} diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonJob.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonJob.java new file mode 100644 index 000000000..cdbb1b81d --- /dev/null +++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonJob.java @@ -0,0 +1,175 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.eclipse.python4j; + + +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.annotation.Nonnull; +import java.util.List; + + +@Data +@NoArgsConstructor +/** + * PythonJob is the right abstraction for executing multiple python scripts + * in a multi thread stateful environment. The setup-and-run mode allows your + * "setup" code (imports, model loading etc) to be executed only once. + */ +public class PythonJob { + + private String code; + private String name; + private String context; + private boolean setupRunMode; + private PythonObject runF; + + static { + new PythonExecutioner(); + } + + @Builder + /** + * @param name Name for the python job. + * @param code Python code. + * @param setupRunMode If true, the python code is expected to have two methods: setup(), which takes no arguments, + * and run() which takes some or no arguments. setup() method is executed once, + * and the run() method is called with the inputs(if any) per transaction, and is expected to return a dictionary + * mapping from output variable names (str) to output values. + * If false, the full script is run on each transaction and the output variables are obtained from the global namespace + * after execution. + */ + public PythonJob(@Nonnull String name, @Nonnull String code, boolean setupRunMode){ + this.name = name; + this.code = code; + this.setupRunMode = setupRunMode; + context = "__job_" + name; + if (PythonContextManager.hasContext(context)) { + throw new PythonException("Unable to create python job " + name + ". Context " + context + " already exists!"); + } + if (setupRunMode) setup(); + } + + + /** + * Clears all variables in current context and calls setup() + */ + public void clearState(){ + String context = this.context; + PythonContextManager.setContext("main"); + PythonContextManager.deleteContext(context); + this.context = context; + setup(); + } + + public void setup(){ + try (PythonGIL gil = PythonGIL.lock()) { + PythonContextManager.setContext(context); + PythonObject runF = PythonExecutioner.getVariable("run"); + if (runF == null || runF.isNone() || !Python.callable(runF)) { + PythonExecutioner.exec(code); + runF = PythonExecutioner.getVariable("run"); + } + if (runF.isNone() || !Python.callable(runF)) { + throw new PythonException("run() method not found! " + + "If a PythonJob is created with 'setup and run' " + + "mode enabled, the associated python code is " + + "expected to contain a run() method " + + "(with or without arguments)."); + } + this.runF = runF; + PythonObject setupF = PythonExecutioner.getVariable("setup"); + if (!setupF.isNone()) { + setupF.call(); + } + } + } + + public void exec(List inputs, List outputs) { + try (PythonGIL gil = PythonGIL.lock()) { + try (PythonGC _ = PythonGC.watch()) { + PythonContextManager.setContext(context); + + if (!setupRunMode) { + + PythonExecutioner.exec(code, inputs, outputs); + + return; + } + PythonExecutioner.setVariables(inputs); + + PythonObject inspect = Python.importModule("inspect"); + PythonObject getfullargspec = inspect.attr("getfullargspec"); + PythonObject argspec = getfullargspec.call(runF); + PythonObject argsList = argspec.attr("args"); + PythonObject runargs = Python.dict(); + int argsCount = Python.len(argsList).toInt(); + for (int i = 0; i < argsCount; i++) { + PythonObject arg = argsList.get(i); + PythonObject val = Python.globals().get(arg); + if (val.isNone()) { + throw new PythonException("Input value not received for run() argument: " + arg.toString()); + } + runargs.set(arg, val); + } + PythonObject outDict = runF.callWithKwargs(runargs); + PythonObject globals = Python.globals(); + PythonObject updateF = globals.attr("update"); + updateF.call(outDict); + PythonExecutioner.getVariables(outputs); + } + } + + } + + public List execAndReturnAllVariables(List inputs){ + try (PythonGIL gil = PythonGIL.lock()) { + try (PythonGC _ = PythonGC.watch()) { + PythonContextManager.setContext(context); + if (!setupRunMode) { + return PythonExecutioner.execAndReturnAllVariables(code, inputs); + } + PythonExecutioner.setVariables(inputs); + PythonObject inspect = Python.importModule("inspect"); + PythonObject getfullargspec = inspect.attr("getfullargspec"); + PythonObject argspec = getfullargspec.call(runF); + PythonObject argsList = argspec.attr("args"); + PythonObject runargs = Python.dict(); + int argsCount = Python.len(argsList).toInt(); + for (int i = 0; i < argsCount; i++) { + PythonObject arg = argsList.get(i); + PythonObject val = Python.globals().get(arg); + if (val.isNone()) { + throw new PythonException("Input value not received for run() argument: " + arg.toString()); + } + runargs.set(arg, val); + } + + PythonObject outDict = runF.callWithKwargs(runargs); + PythonObject globals = Python.globals(); + PythonObject updateF = globals.attr("update"); + updateF.call(outDict); + return PythonExecutioner.getAllVariables(); + } + + } + } + + +} diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonObject.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonObject.java new file mode 100644 index 000000000..f8ec17ed9 --- /dev/null +++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonObject.java @@ -0,0 +1,244 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.eclipse.python4j; + + +import org.bytedeco.cpython.PyObject; +import org.bytedeco.javacpp.Pointer; + +import java.util.*; + +import static org.bytedeco.cpython.global.python.*; + +public class PythonObject { + + static { + new PythonExecutioner(); + } + + private boolean owned = true; + private PyObject nativePythonObject; + + + public PythonObject(PyObject nativePythonObject, boolean owned) { + PythonGIL.assertThreadSafe(); + this.nativePythonObject = nativePythonObject; + this.owned = owned; + if (owned && nativePythonObject != null) { + PythonGC.register(this); + } + } + + public PythonObject(PyObject nativePythonObject) { + PythonGIL.assertThreadSafe(); + this.nativePythonObject = nativePythonObject; + if (nativePythonObject != null) { + PythonGC.register(this); + } + + } + + public PyObject getNativePythonObject() { + return nativePythonObject; + } + + public String toString() { + return PythonTypes.STR.toJava(this); + + } + + public boolean isNone() { + if (nativePythonObject == null || Pointer.isNull(nativePythonObject)) { + return true; + } + try (PythonGC _ = PythonGC.pause()) { + PythonObject type = Python.type(this); + boolean ret = Python.type(this).toString().equals("") && toString().equals("None"); + Py_DecRef(type.nativePythonObject); + return ret; + } + } + + public void del() { + PythonGIL.assertThreadSafe(); + if (owned && nativePythonObject != null && !PythonGC.isWatching()) { + Py_DecRef(nativePythonObject); + nativePythonObject = null; + } + } + + public PythonObject callWithArgs(PythonObject args) { + return callWithArgsAndKwargs(args, null); + } + + public PythonObject callWithKwargs(PythonObject kwargs) { + if (!Python.callable(this)) { + throw new PythonException("Object is not callable: " + toString()); + } + PyObject tuple = PyTuple_New(0); + PyObject dict = kwargs.nativePythonObject; + if (PyObject_IsInstance(dict, new PyObject(PyDict_Type())) != 1) { + throw new PythonException("Expected kwargs to be dict. Received: " + kwargs.toString()); + } + PythonObject ret = new PythonObject(PyObject_Call(nativePythonObject, tuple, dict)); + Py_DecRef(tuple); + return ret; + } + + public PythonObject callWithArgsAndKwargs(PythonObject args, PythonObject kwargs) { + PythonGIL.assertThreadSafe(); + PyObject tuple = null; + boolean ownsTuple = false; + try { + if (!Python.callable(this)) { + throw new PythonException("Object is not callable: " + toString()); + } + + if (PyObject_IsInstance(args.nativePythonObject, new PyObject(PyTuple_Type())) == 1) { + tuple = args.nativePythonObject; + } else if (PyObject_IsInstance(args.nativePythonObject, new PyObject(PyList_Type())) == 1) { + tuple = PyList_AsTuple(args.nativePythonObject); + ownsTuple = true; + } else { + throw new PythonException("Expected args to be tuple or list. Received: " + args.toString()); + } + if (kwargs != null && PyObject_IsInstance(kwargs.nativePythonObject, new PyObject(PyDict_Type())) != 1) { + throw new PythonException("Expected kwargs to be dict. Received: " + kwargs.toString()); + } + return new PythonObject(PyObject_Call(nativePythonObject, tuple, kwargs == null ? null : kwargs.nativePythonObject)); + } finally { + if (ownsTuple) Py_DecRef(tuple); + } + + } + + + public PythonObject call(Object... args) { + return callWithArgsAndKwargs(Arrays.asList(args), null); + } + + public PythonObject callWithArgs(List args) { + return call(args, null); + } + + public PythonObject callWithKwargs(Map kwargs) { + return call(null, kwargs); + } + + public PythonObject callWithArgsAndKwargs(List args, Map kwargs) { + PythonGIL.assertThreadSafe(); + try (PythonGC _ = PythonGC.watch()) { + if (!Python.callable(this)) { + throw new PythonException("Object is not callable: " + toString()); + } + PythonObject pyArgs; + PythonObject pyKwargs; + if (args == null) { + pyArgs = new PythonObject(PyTuple_New(0)); + } else { + PythonObject argsList = PythonTypes.convert(args); + pyArgs = new PythonObject(PyList_AsTuple(argsList.getNativePythonObject())); + } + if (kwargs == null) { + pyKwargs = null; + } else { + pyKwargs = PythonTypes.convert(kwargs); + } + PythonObject ret = new PythonObject( + PyObject_Call( + nativePythonObject, + pyArgs.nativePythonObject, + pyKwargs == null ? null : pyKwargs.nativePythonObject + ) + ); + PythonGC.keep(ret); + return ret; + } + + } + + + public PythonObject attr(String attrName) { + PythonGIL.assertThreadSafe(); + return new PythonObject(PyObject_GetAttrString(nativePythonObject, attrName)); + } + + + public PythonObject(Object javaObject) { + PythonGIL.assertThreadSafe(); + if (javaObject instanceof PythonObject) { + owned = false; + nativePythonObject = ((PythonObject) javaObject).nativePythonObject; + } else { + try (PythonGC _ = PythonGC.pause()) { + nativePythonObject = PythonTypes.convert(javaObject).getNativePythonObject(); + } + PythonGC.register(this); + } + + } + + public int toInt() { + return PythonTypes.INT.toJava(this).intValue(); + } + + public long toLong() { + return PythonTypes.INT.toJava(this); + } + + public float toFloat() { + return PythonTypes.FLOAT.toJava(this).floatValue(); + } + + public double toDouble() { + return PythonTypes.FLOAT.toJava(this); + } + + public boolean toBoolean() { + return PythonTypes.BOOL.toJava(this); + + } + + public List toList() { + return PythonTypes.LIST.toJava(this); + } + + public Map toMap() { + return PythonTypes.DICT.toJava(this); + } + + public PythonObject get(int key) { + PythonGIL.assertThreadSafe(); + return new PythonObject(PyObject_GetItem(nativePythonObject, PyLong_FromLong(key))); + } + + public PythonObject get(String key) { + PythonGIL.assertThreadSafe(); + return new PythonObject(PyObject_GetItem(nativePythonObject, PyUnicode_FromString(key))); + } + + public PythonObject get(PythonObject key) { + PythonGIL.assertThreadSafe(); + return new PythonObject(PyObject_GetItem(nativePythonObject, key.nativePythonObject)); + } + + public void set(PythonObject key, PythonObject value) { + PythonGIL.assertThreadSafe(); + PyObject_SetItem(nativePythonObject, key.nativePythonObject, value.nativePythonObject); + } + +} diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonType.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonType.java new file mode 100644 index 000000000..b4806aa37 --- /dev/null +++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonType.java @@ -0,0 +1,47 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.eclipse.python4j; + + +public abstract class PythonType { + + private final String name; + private final Class javaType; + + public PythonType(String name, Class javaType) { + this.name = name; + this.javaType = javaType; + } + + public T adapt(Object javaObject) throws PythonException { + return (T) javaObject; + } + + public abstract T toJava(PythonObject pythonObject); + + public abstract PythonObject toPython(T javaObject); + + public boolean accepts(Object javaObject) { + return javaType.isAssignableFrom(javaObject.getClass()); + } + + public String getName() { + return name; + } + + +} diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonTypes.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonTypes.java new file mode 100644 index 000000000..0dc20f712 --- /dev/null +++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonTypes.java @@ -0,0 +1,344 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.eclipse.python4j; + + +import org.bytedeco.cpython.PyObject; + +import java.util.*; + +import static org.bytedeco.cpython.global.python.*; +import static org.bytedeco.cpython.global.python.Py_DecRef; + +public class PythonTypes { + + + private static List getPrimitiveTypes() { + return Arrays.asList(STR, INT, FLOAT, BOOL); + } + + private static List getCollectionTypes() { + return Arrays.asList(LIST, DICT); + } + + private static List getExternalTypes() { + //TODO service loader + return new ArrayList<>(); + } + + public static List get() { + List ret = new ArrayList<>(); + ret.addAll(getPrimitiveTypes()); + ret.addAll(getCollectionTypes()); + ret.addAll(getExternalTypes()); + return ret; + } + + public static PythonType get(String name) { + for (PythonType pt : get()) { + if (pt.getName().equals(name)) { // TODO use map instead? + return pt; + } + } + throw new PythonException("Unknown python type: " + name); + } + + public static PythonType getPythonTypeForJavaObject(Object javaObject) { + for (PythonType pt : get()) { + if (pt.accepts(javaObject)) { + return pt; + } + } + throw new PythonException("Unable to find python type for java type: " + javaObject.getClass()); + } + + public static PythonType getPythonTypeForPythonObject(PythonObject pythonObject) { + PyObject pyType = PyObject_Type(pythonObject.getNativePythonObject()); + try { + String pyTypeStr = PythonTypes.STR.toJava(new PythonObject(pyType, false)); + + for (PythonType pt : get()) { + String pyTypeStr2 = ""; + if (pyTypeStr.equals(pyTypeStr2)) { + return pt; + } + } + throw new PythonException("Unable to find converter for python object of type " + pyTypeStr); + } finally { + Py_DecRef(pyType); + } + + + } + + public static PythonObject convert(Object javaObject) { + PythonType pt = getPythonTypeForJavaObject(javaObject); + return pt.toPython(pt.adapt(javaObject)); + } + + public static final PythonType STR = new PythonType("str", String.class) { + + @Override + public String adapt(Object javaObject) { + if (javaObject instanceof String) { + return (String) javaObject; + } + throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to String"); + } + + @Override + public String toJava(PythonObject pythonObject) { + PythonGIL.assertThreadSafe(); + PyObject repr = PyObject_Str(pythonObject.getNativePythonObject()); + PyObject str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); + String jstr = PyBytes_AsString(str).getString(); + Py_DecRef(repr); + Py_DecRef(str); + return jstr; + } + + @Override + public PythonObject toPython(String javaObject) { + return new PythonObject(PyUnicode_FromString(javaObject)); + } + }; + + public static final PythonType INT = new PythonType("int", Long.class) { + @Override + public Long adapt(Object javaObject) { + if (javaObject instanceof Number) { + return ((Number) javaObject).longValue(); + } + throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to Long"); + } + + @Override + public Long toJava(PythonObject pythonObject) { + PythonGIL.assertThreadSafe(); + long val = PyLong_AsLong(pythonObject.getNativePythonObject()); + if (val == -1 && PyErr_Occurred() != null) { + throw new PythonException("Could not convert value to int: " + pythonObject.toString()); + } + return val; + } + + @Override + public boolean accepts(Object javaObject) { + return (javaObject instanceof Integer) || (javaObject instanceof Long); + } + + @Override + public PythonObject toPython(Long javaObject) { + return new PythonObject(PyLong_FromLong(javaObject)); + } + }; + + public static final PythonType FLOAT = new PythonType("float", Double.class) { + + @Override + public Double adapt(Object javaObject) { + if (javaObject instanceof Number) { + return ((Number) javaObject).doubleValue(); + } + throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to Long"); + } + + @Override + public Double toJava(PythonObject pythonObject) { + PythonGIL.assertThreadSafe(); + double val = PyFloat_AsDouble(pythonObject.getNativePythonObject()); + if (val == -1 && PyErr_Occurred() != null) { + throw new PythonException("Could not convert value to float: " + pythonObject.toString()); + } + return val; + } + + @Override + public boolean accepts(Object javaObject) { + return (javaObject instanceof Float) || (javaObject instanceof Double); + } + + @Override + public PythonObject toPython(Double javaObject) { + return new PythonObject(PyFloat_FromDouble(javaObject)); + } + }; + + + public static final PythonType BOOL = new PythonType("bool", Boolean.class) { + + @Override + public Boolean adapt(Object javaObject) { + if (javaObject instanceof Boolean) { + return (Boolean) javaObject; + } + throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to Boolean"); + } + + @Override + public Boolean toJava(PythonObject pythonObject) { + PythonGIL.assertThreadSafe(); + PyObject builtins = PyImport_ImportModule("builtins"); + PyObject boolF = PyObject_GetAttrString(builtins, "bool"); + + PythonObject bool = new PythonObject(boolF, false).call(pythonObject); + boolean ret = PyLong_AsLong(bool.getNativePythonObject()) > 0; + bool.del(); + Py_DecRef(boolF); + Py_DecRef(builtins); + return ret; + } + + @Override + public PythonObject toPython(Boolean javaObject) { + return new PythonObject(PyBool_FromLong(javaObject ? 1 : 0)); + } + }; + + + public static final PythonType LIST = new PythonType("list", List.class) { + + @Override + public List adapt(Object javaObject) { + if (javaObject instanceof List) { + return (List) javaObject; + } else if (javaObject instanceof Object[]) { + return Arrays.asList((Object[]) javaObject); + } else { + throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to List"); + } + } + + @Override + public List toJava(PythonObject pythonObject) { + PythonGIL.assertThreadSafe(); + List ret = new ArrayList(); + long n = PyObject_Size(pythonObject.getNativePythonObject()); + if (n < 0) { + throw new PythonException("Object cannot be interpreted as a List"); + } + for (long i = 0; i < n; i++) { + PyObject pyIndex = PyLong_FromLong(i); + PyObject pyItem = PyObject_GetItem(pythonObject.getNativePythonObject(), + pyIndex); + Py_DecRef(pyIndex); + PythonType pyItemType = getPythonTypeForPythonObject(new PythonObject(pyItem, false)); + ret.add(pyItemType.toJava(new PythonObject(pyItem, false))); + Py_DecRef(pyItem); + } + return ret; + } + + @Override + public PythonObject toPython(List javaObject) { + PythonGIL.assertThreadSafe(); + PyObject pyList = PyList_New(javaObject.size()); + for (int i = 0; i < javaObject.size(); i++) { + Object item = javaObject.get(i); + PythonObject pyItem; + boolean owned; + if (item instanceof PythonObject) { + pyItem = (PythonObject) item; + owned = false; + } else if (item instanceof PyObject) { + pyItem = new PythonObject((PyObject) item, false); + owned = false; + } else { + pyItem = PythonTypes.convert(item); + owned = true; + } + Py_IncRef(pyItem.getNativePythonObject()); // reference will be stolen by PyList_SetItem() + PyList_SetItem(pyList, i, pyItem.getNativePythonObject()); + if (owned) pyItem.del(); + } + return new PythonObject(pyList); + } + }; + + public static final PythonType DICT = new PythonType("dict", Map.class) { + + @Override + public Map adapt(Object javaObject) { + if (javaObject instanceof Map) { + return (Map) javaObject; + } + throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to Map"); + } + + @Override + public Map toJava(PythonObject pythonObject) { + PythonGIL.assertThreadSafe(); + HashMap ret = new HashMap(); + PyObject dictType = new PyObject(PyDict_Type()); + if (PyObject_IsInstance(pythonObject.getNativePythonObject(), dictType) != 1) { + throw new PythonException("Expected dict, received: " + pythonObject.toString()); + } + + PyObject keys = PyDict_Keys(pythonObject.getNativePythonObject()); + PyObject keysIter = PyObject_GetIter(keys); + PyObject vals = PyDict_Values(pythonObject.getNativePythonObject()); + PyObject valsIter = PyObject_GetIter(vals); + try { + long n = PyObject_Size(pythonObject.getNativePythonObject()); + for (long i = 0; i < n; i++) { + PythonObject pyKey = new PythonObject(PyIter_Next(keysIter), false); + PythonObject pyVal = new PythonObject(PyIter_Next(valsIter), false); + PythonType pyKeyType = getPythonTypeForPythonObject(pyKey); + PythonType pyValType = getPythonTypeForPythonObject(pyVal); + ret.put(pyKeyType.toJava(pyKey), pyValType.toJava(pyVal)); + Py_DecRef(pyKey.getNativePythonObject()); + Py_DecRef(pyVal.getNativePythonObject()); + } + } finally { + Py_DecRef(keysIter); + Py_DecRef(valsIter); + Py_DecRef(keys); + Py_DecRef(vals); + } + return ret; + } + + @Override + public PythonObject toPython(Map javaObject) { + PythonGIL.assertThreadSafe(); + PyObject pyDict = PyDict_New(); + for (Object k : javaObject.keySet()) { + PythonObject pyKey; + if (k instanceof PythonObject) { + pyKey = (PythonObject) k; + } else if (k instanceof PyObject) { + pyKey = new PythonObject((PyObject) k); + } else { + pyKey = PythonTypes.convert(k); + } + Object v = javaObject.get(k); + PythonObject pyVal; + pyVal = PythonTypes.convert(v); + int errCode = PyDict_SetItem(pyDict, pyKey.getNativePythonObject(), pyVal.getNativePythonObject()); + if (errCode != 0) { + String keyStr = pyKey.toString(); + pyKey.del(); + pyVal.del(); + throw new PythonException("Unable to create python dictionary. Unhashable key: " + keyStr); + } + pyKey.del(); + pyVal.del(); + } + return new PythonObject(pyDict); + } + }; +} diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonVariable.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonVariable.java new file mode 100644 index 000000000..3deb4d2e7 --- /dev/null +++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonVariable.java @@ -0,0 +1,64 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.eclipse.python4j; + +@lombok.Data +public class PythonVariable { + + private String name; + private String type; + private T value; + + private static boolean validateVariableName(String s) { + if (s.isEmpty()) return false; + if (!Character.isJavaIdentifierStart(s.charAt(0))) return false; + for (int i = 1; i < s.length(); i++) + if (!Character.isJavaIdentifierPart(s.charAt(i))) + return false; + return true; + } + + public PythonVariable(String name, PythonType type, Object value) { + if (!validateVariableName(name)) { + throw new PythonException("Invalid identifier: " + name); + } + this.name = name; + this.type = type.getName(); + setValue(value); + } + + public PythonVariable(String name, PythonType type) { + this(name, type, null); + } + + public PythonType getType() { + return PythonTypes.get(this.type); + } + + public T getValue() { + return this.value; + } + + public void setValue(Object value) { + this.value = value == null ? null : getType().adapt(value); + } + + public PythonObject getPythonObject() { + return getType().toPython(value); + } + +} diff --git a/python4j/python4j-core/src/main/resources/org/eclipse/python4j/pythonexec/pythonexec.py b/python4j/python4j-core/src/main/resources/org/eclipse/python4j/pythonexec/pythonexec.py new file mode 100644 index 000000000..7ae8f6734 --- /dev/null +++ b/python4j/python4j-core/src/main/resources/org/eclipse/python4j/pythonexec/pythonexec.py @@ -0,0 +1,36 @@ +# /******************************************************************************* +# * Copyright (c) 2019 Konduit K.K. +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License, Version 2.0 which is available at +# * https://www.apache.org/licenses/LICENSE-2.0. +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# * License for the specific language governing permissions and limitations +# * under the License. +# * +# * SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************/ + +import sys +import traceback +import json +import inspect + +__python_exception__ = "" +try: + pass + sys.stdout.flush() + sys.stderr.flush() +except Exception as ex: + __python_exception__ = ex + try: + exc_info = sys.exc_info() + finally: + print(ex) + traceback.print_exception(*exc_info) + sys.stdout.flush() + sys.stderr.flush() + diff --git a/python4j/python4j-core/src/test/java/PythonBasicExecutionTest.java b/python4j/python4j-core/src/test/java/PythonBasicExecutionTest.java new file mode 100644 index 000000000..9f5b43dba --- /dev/null +++ b/python4j/python4j-core/src/test/java/PythonBasicExecutionTest.java @@ -0,0 +1,108 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + + +import org.eclipse.python4j.*; +import org.junit.Assert; +import org.junit.Test; + +import javax.annotation.concurrent.NotThreadSafe; +import java.util.*; + +@NotThreadSafe +public class PythonBasicExecutionTest { + + @Test + public void testSimpleExec() { + String code = "print('Hello World')"; + PythonExecutioner.exec(code); + } + + @Test + public void testBadCode() throws Exception { + try { + String code = "printx('Hello world')"; + PythonExecutioner.exec(code); + } catch (Exception e) { + Assert.assertEquals("NameError: name 'printx' is not defined", e.getMessage()); + return; + } + throw new Exception("Bad code did not throw!"); + } + + @Test + public void testExecWithInputs() { + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("x", PythonTypes.STR, "Hello ")); + inputs.add(new PythonVariable<>("y", PythonTypes.STR, "World")); + String code = "print(x + y)"; + PythonExecutioner.exec(code, inputs, null); + + } + + @Test + public void testExecWithInputsAndOutputs() { + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("x", PythonTypes.STR, "Hello ")); + inputs.add(new PythonVariable<>("y", PythonTypes.STR, "World")); + PythonVariable out = new PythonVariable<>("z", PythonTypes.STR); + String code = "z = x + y"; + PythonExecutioner.exec(code, inputs, Collections.singletonList(out)); + Assert.assertEquals("Hello World", out.getValue()); + + } + + @Test + public void testExecAndReturnAllVariables() { + PythonContextManager.reset(); + String code = "a = 5\nb = '10'\nc = 20.0"; + List vars = PythonExecutioner.execAndReturnAllVariables(code); + + Assert.assertEquals("a", vars.get(0).getName()); + Assert.assertEquals(PythonTypes.INT, vars.get(0).getType()); + Assert.assertEquals(5L, (long) vars.get(0).getValue()); + + Assert.assertEquals("b", vars.get(1).getName()); + Assert.assertEquals(PythonTypes.STR, vars.get(1).getType()); + Assert.assertEquals("10", vars.get(1).getValue().toString()); + + Assert.assertEquals("c", vars.get(2).getName()); + Assert.assertEquals(PythonTypes.FLOAT, vars.get(2).getType()); + Assert.assertEquals(20.0, (double) vars.get(2).getValue(), 1e-5); + } + + @Test + public void testExecWithInputsAndReturnAllVariables() { + PythonContextManager.reset(); + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.INT, 5)); + String code = "b = '10'\nc = 20.0 + a"; + List vars = PythonExecutioner.execAndReturnAllVariables(code, inputs); + + Assert.assertEquals("a", vars.get(0).getName()); + Assert.assertEquals(PythonTypes.INT, vars.get(0).getType()); + Assert.assertEquals(5L, (long) vars.get(0).getValue()); + + Assert.assertEquals("b", vars.get(1).getName()); + Assert.assertEquals(PythonTypes.STR, vars.get(1).getType()); + Assert.assertEquals("10", vars.get(1).getValue().toString()); + + Assert.assertEquals("c", vars.get(2).getName()); + Assert.assertEquals(PythonTypes.FLOAT, vars.get(2).getType()); + Assert.assertEquals(25.0, (double) vars.get(2).getValue(), 1e-5); + } + +} diff --git a/python4j/python4j-core/src/test/java/PythonCollectionsTest.java b/python4j/python4j-core/src/test/java/PythonCollectionsTest.java new file mode 100644 index 000000000..7e63d9d28 --- /dev/null +++ b/python4j/python4j-core/src/test/java/PythonCollectionsTest.java @@ -0,0 +1,62 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + + +import org.eclipse.python4j.PythonException; +import org.eclipse.python4j.PythonObject; +import org.eclipse.python4j.PythonTypes; +import org.junit.Assert; +import org.junit.Test; + +import java.util.*; + + +@javax.annotation.concurrent.NotThreadSafe +public class PythonCollectionsTest { + + + @Test + public void testPythonDictFromMap() throws PythonException { + Map map = new HashMap(); + map.put("a", 1); + map.put(1, "a"); + map.put("list1", Arrays.asList(1, 2.0, 3, 4f)); + Map innerMap = new HashMap(); + innerMap.put("b", 2); + innerMap.put(2, "b"); + map.put("innermap", innerMap); + map.put("list2", Arrays.asList(4, "5", innerMap, false, true)); + PythonObject dict = PythonTypes.convert(map); + Map map2 = PythonTypes.DICT.toJava(dict); + Assert.assertEquals(map.toString(), map2.toString()); + } + + @Test + public void testPythonListFromList() throws PythonException{ + List list = new ArrayList<>(); + list.add(1); + list.add("2"); + list.add(Arrays.asList("a", 1.0, 2f, 10, true, false)); + Map map = new HashMap(); + map.put("a", 1); + map.put(1, "a"); + map.put("list1", Arrays.asList(1, 2.0, 3, 4f)); + list.add(map); + PythonObject dict = PythonTypes.convert(list); + List list2 = PythonTypes.LIST.toJava(dict); + Assert.assertEquals(list.toString(), list2.toString()); + } +} diff --git a/python4j/python4j-core/src/test/java/PythonContextManagerTest.java b/python4j/python4j-core/src/test/java/PythonContextManagerTest.java new file mode 100644 index 000000000..a4451764c --- /dev/null +++ b/python4j/python4j-core/src/test/java/PythonContextManagerTest.java @@ -0,0 +1,51 @@ + +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + + +import org.eclipse.python4j.Python; +import org.eclipse.python4j.PythonContextManager; +import org.eclipse.python4j.PythonExecutioner; +import org.junit.Assert; +import org.junit.Test; +import javax.annotation.concurrent.NotThreadSafe; + +@NotThreadSafe +public class PythonContextManagerTest { + + @Test + public void testInt() throws Exception{ + Python.setContext("context1"); + Python.exec("a = 1"); + Python.setContext("context2"); + Python.exec("a = 2"); + Python.setContext("context3"); + Python.exec("a = 3"); + + + Python.setContext("context1"); + Assert.assertEquals(1, PythonExecutioner.getVariable("a").toInt()); + + Python.setContext("context2"); + Assert.assertEquals(2, PythonExecutioner.getVariable("a").toInt()); + + Python.setContext("context3"); + Assert.assertEquals(3, PythonExecutioner.getVariable("a").toInt()); + + PythonContextManager.deleteNonMainContexts(); + } + +} diff --git a/python4j/python4j-core/src/test/java/PythonGCTest.java b/python4j/python4j-core/src/test/java/PythonGCTest.java new file mode 100644 index 000000000..f8c6ecba5 --- /dev/null +++ b/python4j/python4j-core/src/test/java/PythonGCTest.java @@ -0,0 +1,54 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +import org.eclipse.python4j.Python; +import org.eclipse.python4j.PythonGC; +import org.eclipse.python4j.PythonObject; +import org.junit.Assert; +import org.junit.Test; + +import javax.annotation.concurrent.NotThreadSafe; + + +@NotThreadSafe +public class PythonGCTest { + + @Test + public void testGC() throws Exception{ + PythonObject gcModule = Python.importModule("gc"); + PythonObject getObjects = gcModule.attr("get_objects"); + PythonObject pyObjCount1 = Python.len(getObjects.call()); + long objCount1 = pyObjCount1.toLong(); + PythonObject pyList = Python.list(); + pyList.attr("append").call("a"); + pyList.attr("append").call(1.0); + pyList.attr("append").call(true); + PythonObject pyObjCount2 = Python.len(getObjects.call()); + long objCount2 = pyObjCount2.toLong(); + long diff = objCount2 - objCount1; + Assert.assertTrue(diff > 2); + try(PythonGC gc = PythonGC.watch()){ + PythonObject pyList2 = Python.list(); + pyList2.attr("append").call("a"); + pyList2.attr("append").call(1.0); + pyList2.attr("append").call(true); + } + PythonObject pyObjCount3 = Python.len(getObjects.call()); + long objCount3 = pyObjCount3.toLong(); + diff = objCount3 - objCount2; + Assert.assertEquals(2, diff);// 2 objects created during function call + } +} diff --git a/python4j/python4j-core/src/test/java/PythonJobTest.java b/python4j/python4j-core/src/test/java/PythonJobTest.java new file mode 100644 index 000000000..016045a25 --- /dev/null +++ b/python4j/python4j-core/src/test/java/PythonJobTest.java @@ -0,0 +1,287 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +import org.eclipse.python4j.PythonContextManager; +import org.eclipse.python4j.PythonJob; +import org.eclipse.python4j.PythonTypes; +import org.eclipse.python4j.PythonVariable; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + + +@javax.annotation.concurrent.NotThreadSafe +public class PythonJobTest { + + @Test + public void testPythonJobBasic() throws Exception{ + PythonContextManager.deleteNonMainContexts(); + + String code = "c = a + b"; + PythonJob job = new PythonJob("job1", code, false); + + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.INT, 2)); + inputs.add(new PythonVariable<>("b", PythonTypes.INT, 3)); + + List outputs = new ArrayList<>(); + outputs.add(new PythonVariable<>("c", PythonTypes.INT)); + + + job.exec(inputs, outputs); + assertEquals("c", outputs.get(0).getName()); + assertEquals(5L, (long)outputs.get(0).getValue()); + + inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.FLOAT, 3.0)); + inputs.add(new PythonVariable<>("b", PythonTypes.FLOAT, 4.0)); + + outputs = new ArrayList<>(); + outputs.add(new PythonVariable<>("c", PythonTypes.FLOAT)); + + + job.exec(inputs, outputs); + + assertEquals("c", outputs.get(0).getName()); + assertEquals(7.0, (double)outputs.get(0).getValue(), 1e-5); + + + } + + @Test + public void testPythonJobReturnAllVariables()throws Exception{ + PythonContextManager.deleteNonMainContexts(); + + String code = "c = a + b"; + PythonJob job = new PythonJob("job1", code, false); + + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.INT, 2)); + inputs.add(new PythonVariable<>("b", PythonTypes.INT, 3)); + + + List outputs = job.execAndReturnAllVariables(inputs); + + + assertEquals("a", outputs.get(0).getName()); + assertEquals(2L, (long)outputs.get(0).getValue()); + assertEquals("b", outputs.get(1).getName()); + assertEquals(3L, (long)outputs.get(1).getValue()); + assertEquals("c", outputs.get(2).getName()); + assertEquals(5L, (long)outputs.get(2).getValue()); + + inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.FLOAT, 3.0)); + inputs.add(new PythonVariable<>("b", PythonTypes.FLOAT, 4.0)); + outputs = job.execAndReturnAllVariables(inputs); + assertEquals("a", outputs.get(0).getName()); + assertEquals(3.0, (double)outputs.get(0).getValue(), 1e-5); + assertEquals("b", outputs.get(1).getName()); + assertEquals(4.0, (double)outputs.get(1).getValue(), 1e-5); + assertEquals("c", outputs.get(2).getName()); + assertEquals(7.0, (double)outputs.get(2).getValue(), 1e-5); + + } + + + @Test + public void testMultiplePythonJobsParallel()throws Exception{ + PythonContextManager.deleteNonMainContexts(); + String code1 = "c = a + b"; + PythonJob job1 = new PythonJob("job1", code1, false); + + String code2 = "c = a - b"; + PythonJob job2 = new PythonJob("job2", code2, false); + + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.INT, 2)); + inputs.add(new PythonVariable<>("b", PythonTypes.INT, 3)); + + + List outputs = new ArrayList<>(); + outputs.add(new PythonVariable<>("c", PythonTypes.INT)); + + job1.exec(inputs, outputs); + + assertEquals("c", outputs.get(0).getName()); + assertEquals(5L, (long)outputs.get(0).getValue()); + + + job2.exec(inputs, outputs); + + assertEquals("c", outputs.get(0).getName()); + assertEquals(-1L, (long)outputs.get(0).getValue()); + + inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.FLOAT, 3.0)); + inputs.add(new PythonVariable<>("b", PythonTypes.FLOAT, 4.0)); + + outputs = new ArrayList<>(); + outputs.add(new PythonVariable<>("c", PythonTypes.FLOAT)); + + + job1.exec(inputs, outputs); + + assertEquals("c", outputs.get(0).getName()); + assertEquals(7.0, (double)outputs.get(0).getValue(), 1e-5); + + job2.exec(inputs, outputs); + + assertEquals("c", outputs.get(0).getName()); + assertEquals(-1., (double)outputs.get(0).getValue(), 1e-5); + + } + + + @Test + public void testPythonJobSetupRun()throws Exception{ + + PythonContextManager.deleteNonMainContexts(); + String code = "five=None\n" + + "def setup():\n" + + " global five\n"+ + " five = 5\n\n" + + "def run(a, b):\n" + + " c = a + b + five\n"+ + " return {'c':c}\n\n"; + PythonJob job = new PythonJob("job1", code, true); + + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.INT, 2)); + inputs.add(new PythonVariable<>("b", PythonTypes.INT, 3)); + + List outputs = new ArrayList<>(); + outputs.add(new PythonVariable<>("c", PythonTypes.INT)); + job.exec(inputs, outputs); + + assertEquals("c", outputs.get(0).getName()); + assertEquals(10L, (long)outputs.get(0).getValue()); + + + inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.FLOAT, 3.0)); + inputs.add(new PythonVariable<>("b", PythonTypes.FLOAT, 4.0)); + + + outputs = new ArrayList<>(); + outputs.add(new PythonVariable<>("c", PythonTypes.FLOAT)); + + job.exec(inputs, outputs); + + assertEquals("c", outputs.get(0).getName()); + assertEquals(12.0, (double)outputs.get(0).getValue(), 1e-5); + + } + @Test + public void testPythonJobSetupRunAndReturnAllVariables()throws Exception{ + PythonContextManager.deleteNonMainContexts(); + String code = "five=None\n" + + "c=None\n"+ + "def setup():\n" + + " global five\n"+ + " five = 5\n\n" + + "def run(a, b):\n" + + " global c\n" + + " c = a + b + five\n"; + PythonJob job = new PythonJob("job1", code, true); + + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.INT, 2)); + inputs.add(new PythonVariable<>("b", PythonTypes.INT, 3)); + + List outputs = job.execAndReturnAllVariables(inputs); + + assertEquals("c", outputs.get(1).getName()); + assertEquals(10L, (long)outputs.get(1).getValue()); + + inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.FLOAT, 3.0)); + inputs.add(new PythonVariable<>("b", PythonTypes.FLOAT, 4.0)); + + outputs = job.execAndReturnAllVariables(inputs); + + + assertEquals("c", outputs.get(1).getName()); + assertEquals(12.0, (double)outputs.get(1).getValue(), 1e-5); + + + + } + + @Test + public void testMultiplePythonJobsSetupRunParallel()throws Exception{ + PythonContextManager.deleteNonMainContexts(); + + String code1 = "five=None\n" + + "def setup():\n" + + " global five\n"+ + " five = 5\n\n" + + "def run(a, b):\n" + + " c = a + b + five\n"+ + " return {'c':c}\n\n"; + PythonJob job1 = new PythonJob("job1", code1, true); + + String code2 = "five=None\n" + + "def setup():\n" + + " global five\n"+ + " five = 5\n\n" + + "def run(a, b):\n" + + " c = a + b - five\n"+ + " return {'c':c}\n\n"; + PythonJob job2 = new PythonJob("job2", code2, true); + + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.INT, 2)); + inputs.add(new PythonVariable<>("b", PythonTypes.INT, 3)); + + + List outputs = new ArrayList<>(); + outputs.add(new PythonVariable<>("c", PythonTypes.INT)); + + job1.exec(inputs, outputs); + + assertEquals("c", outputs.get(0).getName()); + assertEquals(10L, (long)outputs.get(0).getValue()); + + job2.exec(inputs, outputs); + + assertEquals("c", outputs.get(0).getName()); + assertEquals(0L, (long)outputs.get(0).getValue()); + + inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.FLOAT, 3.0)); + inputs.add(new PythonVariable<>("b", PythonTypes.FLOAT, 4.0)); + + outputs = new ArrayList<>(); + outputs.add(new PythonVariable<>("c", PythonTypes.FLOAT)); + + + job1.exec(inputs, outputs); + + assertEquals("c", outputs.get(0).getName()); + assertEquals(12.0, (double)outputs.get(0).getValue(), 1e-5); + + job2.exec(inputs, outputs); + + assertEquals("c", outputs.get(0).getName()); + assertEquals(2.0, (double)outputs.get(0).getValue(), 1e-5); + + } + +} diff --git a/python4j/python4j-core/src/test/java/PythonMultiThreadTest.java b/python4j/python4j-core/src/test/java/PythonMultiThreadTest.java new file mode 100644 index 000000000..ec544b65f --- /dev/null +++ b/python4j/python4j-core/src/test/java/PythonMultiThreadTest.java @@ -0,0 +1,169 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +import org.eclipse.python4j.*; +import org.junit.Assert; +import org.junit.Test; + +import javax.annotation.concurrent.NotThreadSafe; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + + +@NotThreadSafe +public class PythonMultiThreadTest { + + @Test + public void testMultiThreading1()throws Throwable{ + final List exceptions = Collections.synchronizedList(new ArrayList()); + Runnable runnable = new Runnable() { + @Override + public void run() { + try(PythonGIL gil = PythonGIL.lock()){ + try(PythonGC gc = PythonGC.watch()){ + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("x", PythonTypes.STR, "Hello ")); + inputs.add(new PythonVariable<>("y", PythonTypes.STR, "World")); + PythonVariable out = new PythonVariable<>("z", PythonTypes.STR); + String code = "z = x + y"; + PythonExecutioner.exec(code, inputs, Collections.singletonList(out)); + Assert.assertEquals("Hello World", out.getValue()); + System.out.println(out.getValue() + " From thread " + Thread.currentThread().getId()); + } + }catch (Throwable e){ + exceptions.add(e); + } + } + }; + + int numThreads = 10; + Thread[] threads = new Thread[numThreads]; + for (int i = 0; i < threads.length; i++){ + threads[i] = new Thread(runnable); + } + for (int i = 0; i < threads.length; i++){ + threads[i].start(); + } + Thread.sleep(100); + for (int i = 0; i < threads.length; i++){ + threads[i].join(); + } + if (!exceptions.isEmpty()){ + throw(exceptions.get(0)); + } + + } + @Test + public void testMultiThreading2()throws Throwable{ + final List exceptions = Collections.synchronizedList(new ArrayList()); + Runnable runnable = new Runnable() { + @Override + public void run() { + try(PythonGIL gil = PythonGIL.lock()){ + try(PythonGC gc = PythonGC.watch()){ + PythonContextManager.reset(); + PythonContextManager.reset(); + List inputs = new ArrayList<>(); + inputs.add(new PythonVariable<>("a", PythonTypes.INT, 5)); + String code = "b = '10'\nc = 20.0 + a"; + List vars = PythonExecutioner.execAndReturnAllVariables(code, inputs); + + Assert.assertEquals("a", vars.get(0).getName()); + Assert.assertEquals(PythonTypes.INT, vars.get(0).getType()); + Assert.assertEquals(5L, (long)vars.get(0).getValue()); + + Assert.assertEquals("b", vars.get(1).getName()); + Assert.assertEquals(PythonTypes.STR, vars.get(1).getType()); + Assert.assertEquals("10", vars.get(1).getValue().toString()); + + Assert.assertEquals("c", vars.get(2).getName()); + Assert.assertEquals(PythonTypes.FLOAT, vars.get(2).getType()); + Assert.assertEquals(25.0, (double)vars.get(2).getValue(), 1e-5); + } + }catch (Throwable e){ + exceptions.add(e); + } + } + }; + + int numThreads = 10; + Thread[] threads = new Thread[numThreads]; + for (int i = 0; i < threads.length; i++){ + threads[i] = new Thread(runnable); + } + for (int i = 0; i < threads.length; i++){ + threads[i].start(); + } + Thread.sleep(100); + for (int i = 0; i < threads.length; i++){ + threads[i].join(); + } + if (!exceptions.isEmpty()){ + throw(exceptions.get(0)); + } + } + + @Test + public void testMultiThreading3() throws Throwable{ + PythonContextManager.deleteNonMainContexts(); + + String code = "c = a + b"; + final PythonJob job = new PythonJob("job1", code, false); + + final List exceptions = Collections.synchronizedList(new ArrayList()); + + class JobThread extends Thread{ + private int a, b, c; + public JobThread(int a, int b, int c){ + this.a = a; + this.b = b; + this.c = c; + } + @Override + public void run(){ + try{ + PythonVariable out = new PythonVariable<>("c", PythonTypes.INT); + job.exec(Arrays.asList(new PythonVariable<>("a", PythonTypes.INT, a), + new PythonVariable<>("b", PythonTypes.INT, b)), + Collections.singletonList(out)); + Assert.assertEquals(c, out.getValue().intValue()); + }catch (Exception e){ + exceptions.add(e); + } + + } + } + int numThreads = 10; + JobThread[] threads = new JobThread[numThreads]; + for (int i=0; i < threads.length; i++){ + threads[i] = new JobThread(i, i + 3, 2 * i +3); + } + + for (int i = 0; i < threads.length; i++){ + threads[i].start(); + } + Thread.sleep(100); + for (int i = 0; i < threads.length; i++){ + threads[i].join(); + } + + if (!exceptions.isEmpty()){ + throw(exceptions.get(0)); + } + } +} diff --git a/python4j/python4j-core/src/test/java/PythonPrimitiveTypesTest.java b/python4j/python4j-core/src/test/java/PythonPrimitiveTypesTest.java new file mode 100644 index 000000000..ae10ed8dc --- /dev/null +++ b/python4j/python4j-core/src/test/java/PythonPrimitiveTypesTest.java @@ -0,0 +1,82 @@ +/******************************************************************************* + * Copyright (c) 2020 Konduit K.K. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + + +import org.eclipse.python4j.PythonException; +import org.eclipse.python4j.PythonObject; +import org.eclipse.python4j.PythonTypes; +import org.junit.Assert; +import org.junit.Test; + +public class PythonPrimitiveTypesTest { + + @Test + public void testInt() throws PythonException { + long j = 3; + PythonObject p = PythonTypes.INT.toPython(j); + long j2 = PythonTypes.INT.toJava(p); + + Assert.assertEquals(j, j2); + + PythonObject p2 = PythonTypes.convert(j); + long j3 = PythonTypes.INT.toJava(p2); + + Assert.assertEquals(j, j3); + } + + @Test + public void testStr() throws PythonException{ + String s = "abcd"; + PythonObject p = PythonTypes.STR.toPython(s); + String s2 = PythonTypes.STR.toJava(p); + + Assert.assertEquals(s, s2); + + PythonObject p2 = PythonTypes.convert(s); + String s3 = PythonTypes.STR.toJava(p2); + + Assert.assertEquals(s, s3); + } + + @Test + public void testFloat() throws PythonException{ + double f = 7; + PythonObject p = PythonTypes.FLOAT.toPython(f); + double f2 = PythonTypes.FLOAT.toJava(p); + + Assert.assertEquals(f, f2, 1e-5); + + PythonObject p2 = PythonTypes.convert(f); + double f3 = PythonTypes.FLOAT.toJava(p2); + + Assert.assertEquals(f, f3, 1e-5); + } + + @Test + public void testBool() throws PythonException{ + boolean b = true; + PythonObject p = PythonTypes.BOOL.toPython(b); + boolean b2 = PythonTypes.BOOL.toJava(p); + + Assert.assertEquals(b, b2); + + PythonObject p2 = PythonTypes.convert(b); + boolean b3 = PythonTypes.BOOL.toJava(p2); + + Assert.assertEquals(b, b3); + } + +} diff --git a/python4j/python4j-numpy/pom.xml b/python4j/python4j-numpy/pom.xml new file mode 100644 index 000000000..527a9343f --- /dev/null +++ b/python4j/python4j-numpy/pom.xml @@ -0,0 +1,42 @@ + + + + python4j-parent + org.eclipse + 1.0.0-SNAPSHOT + + 4.0.0 + + python4j-numpy + + + + org.bytedeco + numpy-platform + ${numpy.javacpp.version} + + + org.nd4j + nd4j-native-api + ${project.version} + + + org.nd4j + nd4j-common-tests + ${nd4j.version} + test + + + + + + test-nd4j-native + + + test-nd4j-cuda-10.2 + + + + \ No newline at end of file