diff --git a/pom.xml b/pom.xml
index ab9f80b92..184eeb11f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -137,6 +137,7 @@
jumpy
pydatavec
pydl4j
+ python4j
diff --git a/python4j/pom.xml b/python4j/pom.xml
new file mode 100644
index 000000000..57af8f1bb
--- /dev/null
+++ b/python4j/pom.xml
@@ -0,0 +1,66 @@
+
+
+
+
+
+ deeplearning4j
+ org.deeplearning4j
+ 1.0.0-SNAPSHOT
+
+ 4.0.0
+
+ org.eclipse
+ python4j-parent
+ pom
+
+ python4j-core
+ python4j-numpy
+
+
+
+
+ org.projectlombok
+ lombok
+ ${lombok.version}
+ provided
+
+
+ ch.qos.logback
+ logback-classic
+ ${logback.version}
+ test
+
+
+ junit
+ junit
+ ${junit.version}
+ test
+
+
+ commons-io
+ commons-io
+ ${commons-io.version}
+
+
+ com.google.code.findbugs
+ jsr305
+ 3.0.2
+
+
+
\ No newline at end of file
diff --git a/python4j/python4j-core/pom.xml b/python4j/python4j-core/pom.xml
new file mode 100644
index 000000000..b429d8272
--- /dev/null
+++ b/python4j/python4j-core/pom.xml
@@ -0,0 +1,44 @@
+
+
+
+
+
+
+ python4j-parent
+ org.eclipse
+ 1.0.0-SNAPSHOT
+
+ jar
+ 4.0.0
+
+ python4j-core
+
+
+ org.json
+ json
+ 20190722
+
+
+ org.bytedeco
+ cpython-platform
+ ${cpython-platform.version}
+
+
+
+
\ No newline at end of file
diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/Python.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/Python.java
new file mode 100644
index 000000000..fd6fff112
--- /dev/null
+++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/Python.java
@@ -0,0 +1,611 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Konduit K.K.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+
+package org.eclipse.python4j;
+
+import org.bytedeco.cpython.PyObject;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.bytedeco.cpython.global.python.*;
+
+
+public class Python {
+
+ static {
+ new PythonExecutioner();
+ }
+
+ /**
+ * Imports a python module, similar to python import statement.
+ *
+ * @param moduleName name of the module to be imported
+ * @return reference to the module object
+ */
+ public static PythonObject importModule(String moduleName) {
+ PythonGIL.assertThreadSafe();
+ PythonObject module = new PythonObject(PyImport_ImportModule(moduleName));
+ if (module.isNone()) {
+ throw new PythonException("Error importing module: " + moduleName);
+ }
+ return module;
+ }
+
+ /**
+ * Gets a builtins attribute
+ *
+ * @param attrName Attribute name
+ * @return
+ */
+ public static PythonObject attr(String attrName) {
+ PythonGIL.assertThreadSafe();
+ PyObject builtins = PyImport_ImportModule("builtins");
+ try {
+ return new PythonObject(PyObject_GetAttrString(builtins, attrName));
+ } finally {
+ Py_DecRef(builtins);
+ }
+ }
+
+
+ /**
+ * Gets the size of a PythonObject. similar to len() in python.
+ *
+ * @param pythonObject
+ * @return
+ */
+ public static PythonObject len(PythonObject pythonObject) {
+ PythonGIL.assertThreadSafe();
+ long n = PyObject_Size(pythonObject.getNativePythonObject());
+ if (n < 0) {
+ throw new PythonException("Object has no length: " + pythonObject);
+ }
+ return PythonTypes.INT.toPython(n);
+ }
+
+ /**
+ * Gets the string representation of an object.
+ *
+ * @param pythonObject
+ * @return
+ */
+ public static PythonObject str(PythonObject pythonObject) {
+ PythonGIL.assertThreadSafe();
+ try {
+ return PythonTypes.STR.toPython(pythonObject.toString());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+
+ }
+
+ /**
+ * Returns an empty string
+ *
+ * @return
+ */
+ public static PythonObject str() {
+ PythonGIL.assertThreadSafe();
+ try {
+ return PythonTypes.STR.toPython("");
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Returns the str type object
+ * @return
+ */
+ public static PythonObject strType() {
+ return attr("str");
+ }
+
+ /**
+ * Returns a floating point number from a number or a string.
+ * @param pythonObject
+ * @return
+ */
+ public static PythonObject float_(PythonObject pythonObject) {
+ return PythonTypes.FLOAT.toPython(PythonTypes.FLOAT.toJava(pythonObject));
+ }
+
+ /**
+ * Reutrns 0.
+ * @return
+ */
+ public static PythonObject float_() {
+ try {
+ return PythonTypes.FLOAT.toPython(0d);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ /**
+ * Returns the float type object
+ * @return
+ */
+ public static PythonObject floatType() {
+ return attr("float");
+ }
+
+
+ /**
+ * Converts a value to a Boolean value i.e., True or False, using the standard truth testing procedure.
+ * @param pythonObject
+ * @return
+ */
+ public static PythonObject bool(PythonObject pythonObject) {
+ return PythonTypes.BOOL.toPython(PythonTypes.BOOL.toJava(pythonObject));
+
+ }
+
+ /**
+ * Returns False.
+ * @return
+ */
+ public static PythonObject bool() {
+ return PythonTypes.BOOL.toPython(false);
+
+ }
+
+ /**
+ * Returns the bool type object
+ * @return
+ */
+ public static PythonObject boolType() {
+ return attr("bool");
+ }
+
+ /**
+ * Returns an integer from a number or a string.
+ * @param pythonObject
+ * @return
+ */
+ public static PythonObject int_(PythonObject pythonObject) {
+ return PythonTypes.INT.toPython(PythonTypes.INT.toJava(pythonObject));
+ }
+
+ /**
+ * Returns 0
+ * @return
+ */
+ public static PythonObject int_() {
+ return PythonTypes.INT.toPython(0L);
+
+ }
+
+ /**
+ * Returns the int type object
+ * @return
+ */
+ public static PythonObject intType() {
+ return attr("int");
+ }
+
+ /**
+ * Takes sequence types and converts them to lists.
+ * @param pythonObject
+ * @return
+ */
+ public static PythonObject list(PythonObject pythonObject) {
+ PythonGIL.assertThreadSafe();
+ try (PythonGC _ = PythonGC.watch()) {
+ PythonObject listF = attr("list");
+ PythonObject ret = listF.call(pythonObject);
+ if (ret.isNone()) {
+ throw new PythonException("Object is not iterable: " + pythonObject.toString());
+ }
+ return ret;
+ }
+ }
+
+ /**
+ * Returns empty list.
+ * @return
+ */
+ public static PythonObject list() {
+ return PythonTypes.LIST.toPython(Collections.emptyList());
+ }
+
+ /**
+ * Returns list type object.
+ * @return
+ */
+ public static PythonObject listType() {
+ return attr("list");
+ }
+
+ /**
+ * Creates a dictionary.
+ * @param pythonObject
+ * @return
+ */
+ public static PythonObject dict(PythonObject pythonObject) {
+ PythonObject dictF = attr("dict");
+ PythonObject ret = dictF.call(pythonObject);
+ if (ret.isNone()) {
+ throw new PythonException("Cannot build dict from object: " + pythonObject.toString());
+ }
+ dictF.del();
+ return ret;
+ }
+
+ /**
+ * Returns empty dict
+ * @return
+ */
+ public static PythonObject dict() {
+ return PythonTypes.DICT.toPython(Collections.emptyMap());
+ }
+
+ /**
+ * Returns dict type object.
+ * @return
+ */
+ public static PythonObject dictType() {
+ return attr("dict");
+ }
+
+ /**
+ * Creates a set.
+ * @param pythonObject
+ * @return
+ */
+ public static PythonObject set(PythonObject pythonObject) {
+ PythonObject setF = attr("set");
+ PythonObject ret = setF.call(pythonObject);
+ if (ret.isNone()) {
+ throw new PythonException("Cannot build set from object: " + pythonObject.toString());
+ }
+ setF.del();
+ return ret;
+ }
+
+ /**
+ * Returns empty set.
+ * @return
+ */
+ public static PythonObject set() {
+ PythonObject setF = attr("set");
+ PythonObject ret;
+ ret = setF.call();
+ setF.del();
+ return ret;
+ }
+
+ /**
+ * Returns empty set.
+ * @return
+ */
+ public static PythonObject setType() {
+ return attr("set");
+ }
+
+ /**
+ * Creates a bytearray.
+ * @param pythonObject
+ * @return
+ */
+ public static PythonObject bytearray(PythonObject pythonObject) {
+ PythonObject baF = attr("bytearray");
+ PythonObject ret = baF.call(pythonObject);
+ if (ret.isNone()) {
+ throw new PythonException("Cannot build bytearray from object: " + pythonObject.toString());
+ }
+ baF.del();
+ return ret;
+ }
+
+ /**
+ * Returns empty bytearray.
+ * @return
+ */
+ public static PythonObject bytearray() {
+ PythonObject baF = attr("bytearray");
+ PythonObject ret;
+ ret = baF.call();
+ baF.del();
+ return ret;
+ }
+
+ /**
+ * Returns bytearray type object
+ * @return
+ */
+ public static PythonObject bytearrayType() {
+ return attr("bytearray");
+ }
+
+ /**
+ * Creates a memoryview.
+ * @param pythonObject
+ * @return
+ */
+ public static PythonObject memoryview(PythonObject pythonObject) {
+ PythonObject mvF = attr("memoryview");
+ PythonObject ret = mvF.call(pythonObject);
+ if (ret.isNone()) {
+ throw new PythonException("Cannot build memoryview from object: " + pythonObject.toString());
+ }
+ mvF.del();
+ return ret;
+ }
+
+ /**
+ * Returns memoryview type object.
+ * @return
+ */
+ public static PythonObject memoryviewType() {
+ return attr("memoryview");
+ }
+
+ /**
+ * Creates a byte string.
+ * @param pythonObject
+ * @return
+ */
+ public static PythonObject bytes(PythonObject pythonObject) {
+ PythonObject bytesF = attr("bytes");
+ PythonObject ret = bytesF.call(pythonObject);
+ if (ret.isNone()) {
+ throw new PythonException("Cannot build bytes from object: " + pythonObject.toString());
+ }
+ bytesF.del();
+ return ret;
+ }
+
+ /**
+ * Returns empty byte string.
+ * @return
+ */
+ public static PythonObject bytes() {
+ PythonObject bytesF = attr("bytes");
+ PythonObject ret;
+ ret = bytesF.call();
+ bytesF.del();
+ return ret;
+ }
+
+ /**
+ * Returns bytes type object
+ * @return
+ */
+ public static PythonObject bytesType() {
+ return attr("bytes");
+ }
+
+ /**
+ * Creates a tuple.
+ * @param pythonObject
+ * @return
+ */
+ public static PythonObject tuple(PythonObject pythonObject) {
+ PythonObject tupleF = attr("tupleF");
+ PythonObject ret = tupleF.call(pythonObject);
+ if (ret.isNone()) {
+ throw new PythonException("Cannot build tuple from object: " + pythonObject.toString());
+ }
+ tupleF.del();
+ return ret;
+ }
+
+ /**
+ * Returns empty tuple.
+ * @return
+ */
+ public static PythonObject tuple() {
+ PythonObject tupleF = attr("tuple");
+ PythonObject ret;
+ ret = tupleF.call();
+ tupleF.del();
+ return ret;
+ }
+
+ /**
+ * Returns tuple type object
+ * @return
+ */
+ public static PythonObject tupleType() {
+ return attr("tuple");
+ }
+
+ /**
+ * Creates an Exception
+ * @param pythonObject
+ * @return
+ */
+ public static PythonObject Exception(PythonObject pythonObject) {
+ PythonObject excF = attr("Exception");
+ PythonObject ret = excF.call(pythonObject);
+ excF.del();
+ return ret;
+ }
+
+ /**
+ * Creates an Exception
+ * @return
+ */
+ public static PythonObject Exception() {
+ PythonObject excF = attr("Exception");
+ PythonObject ret;
+ ret = excF.call();
+ excF.del();
+ return ret;
+ }
+
+ /**
+ * Returns Exception type object
+ * @return
+ */
+ public static PythonObject ExceptionType() {
+ return attr("Exception");
+ }
+
+
+ /**
+ * Returns the globals dictionary.
+ * @return
+ */
+ public static PythonObject globals() {
+ PythonGIL.assertThreadSafe();
+ PyObject main = PyImport_ImportModule("__main__");
+ PyObject globals = PyModule_GetDict(main);
+ Py_DecRef(main);
+ return new PythonObject(globals, false);
+ }
+
+ /**
+ * Returns the type of an object.
+ * @param pythonObject
+ * @return
+ */
+ public static PythonObject type(PythonObject pythonObject) {
+ PythonObject typeF = attr("type");
+ PythonObject ret = typeF.call(pythonObject);
+ typeF.del();
+ return ret;
+ }
+
+ /**
+ * Returns True if the specified object is of the specified type, otherwise False.
+ * @param obj
+ * @param type
+ * @return
+ */
+ public static boolean isinstance(PythonObject obj, PythonObject... type) {
+ PythonGIL.assertThreadSafe();
+ PyObject argsTuple = PyTuple_New(type.length);
+ try {
+ for (int i = 0; i < type.length; i++) {
+ PythonObject x = type[i];
+ Py_IncRef(x.getNativePythonObject());
+ PyTuple_SetItem(argsTuple, i, x.getNativePythonObject());
+ }
+ return PyObject_IsInstance(obj.getNativePythonObject(), argsTuple) != 0;
+ } finally {
+ Py_DecRef(argsTuple);
+ }
+
+ }
+
+ /**
+ * Evaluates the specified expression.
+ * @param expression
+ * @return
+ */
+ public static PythonObject eval(String expression) {
+
+ PythonGIL.assertThreadSafe();
+ PyObject compiledCode = Py_CompileString(expression, "", Py_eval_input);
+ PyObject main = PyImport_ImportModule("__main__");
+ PyObject globals = PyModule_GetDict(main);
+ PyObject locals = PyDict_New();
+ try {
+ return new PythonObject(PyEval_EvalCode(compiledCode, globals, locals));
+ } finally {
+ Py_DecRef(main);
+ Py_DecRef(locals);
+ Py_DecRef(compiledCode);
+ }
+
+ }
+
+ /**
+ * Returns the builtins module
+ * @return
+ */
+ public static PythonObject builtins() {
+ return importModule("builtins");
+
+ }
+
+ /**
+ * Returns None.
+ * @return
+ */
+ public static PythonObject None() {
+ return eval("None");
+ }
+
+ /**
+ * Returns True.
+ * @return
+ */
+ public static PythonObject True() {
+ return eval("True");
+ }
+
+ /**
+ * Returns False.
+ * @return
+ */
+ public static PythonObject False() {
+ return eval("False");
+ }
+
+ /**
+ * Returns True if the object passed is callable callable, otherwise False.
+ * @param pythonObject
+ * @return
+ */
+ public static boolean callable(PythonObject pythonObject) {
+ PythonGIL.assertThreadSafe();
+ return PyCallable_Check(pythonObject.getNativePythonObject()) == 1;
+ }
+
+
+ public static void setContext(String context){
+ PythonContextManager.setContext(context);
+ }
+
+ public static String getCurrentContext() {
+ return PythonContextManager.getCurrentContext();
+ }
+
+ public static void deleteContext(String context){
+ PythonContextManager.deleteContext(context);
+ }
+ public static void resetContext() {
+ PythonContextManager.reset();
+ }
+
+ /**
+ * Executes a string of code.
+ * @param code
+ * @throws PythonException
+ */
+ public static void exec(String code) throws PythonException {
+ PythonExecutioner.exec(code);
+ }
+
+ /**
+ * Executes a string of code.
+ * @param code
+ * @param inputs
+ * @param outputs
+ */
+ public static void exec(String code, List inputs, List outputs){
+ PythonExecutioner.exec(code, inputs, outputs);
+ }
+
+
+}
diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonContextManager.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonContextManager.java
new file mode 100644
index 000000000..a34d8a239
--- /dev/null
+++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonContextManager.java
@@ -0,0 +1,241 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Konduit K.K.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+package org.eclipse.python4j;
+
+import javax.lang.model.SourceVersion;
+
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Emulates multiples interpreters in a single interpreter.
+ * This works by simply obfuscating/de-obfuscating variable names
+ * such that only the required subset of the global namespace is "visible"
+ * at any given time.
+ * By default, there exists a "main" context emulating the default interpreter
+ *
+ * @author Fariz Rahman
+ */
+
+
+public class PythonContextManager {
+
+ private static Set contexts = new HashSet<>();
+ private static AtomicBoolean init = new AtomicBoolean(false);
+ private static String currentContext;
+ private static final String MAIN_CONTEXT = "main";
+ private static final String COLLAPSED_KEY = "__collapsed__";
+
+ static {
+ init();
+ }
+
+ private static void init() {
+ if (init.get()) return;
+ new PythonExecutioner();
+ init.set(true);
+ currentContext = MAIN_CONTEXT;
+ contexts.add(currentContext);
+ }
+
+
+ /**
+ * Adds a new context.
+ * @param contextName
+ */
+ public static void addContext(String contextName) {
+ if (!validateContextName(contextName)) {
+ throw new PythonException("Invalid context name: " + contextName);
+ }
+ contexts.add(contextName);
+ }
+
+ /**
+ * Returns true if context exists, else false.
+ * @param contextName
+ * @return
+ */
+ public static boolean hasContext(String contextName) {
+ return contexts.contains(contextName);
+ }
+
+ private static boolean validateContextName(String s) {
+ return SourceVersion.isIdentifier(s) && !s.startsWith(COLLAPSED_KEY);
+ }
+
+ private static String getContextPrefix(String contextName) {
+ return COLLAPSED_KEY + contextName + "__";
+ }
+
+ private static String getCollapsedVarNameForContext(String varName, String contextName) {
+ return getContextPrefix(contextName) + varName;
+ }
+
+ private static String expandCollapsedVarName(String varName, String contextName) {
+ String prefix = COLLAPSED_KEY + contextName + "__";
+ return varName.substring(prefix.length());
+
+ }
+
+ private static void collapseContext(String contextName) {
+ try (PythonGC _ = PythonGC.watch()) {
+ PythonObject globals = Python.globals();
+ PythonObject pop = globals.attr("pop");
+ PythonObject keysF = globals.attr("keys");
+ PythonObject keys = keysF.call();
+ PythonObject keysList = Python.list(keys);
+ int numKeys = Python.len(keysList).toInt();
+ for (int i = 0; i < numKeys; i++) {
+ PythonObject key = keysList.get(i);
+ String keyStr = key.toString();
+ if (!((keyStr.startsWith("__") && keyStr.endsWith("__")) || keyStr.startsWith("__collapsed_"))) {
+ String collapsedKey = getCollapsedVarNameForContext(keyStr, contextName);
+ PythonObject val = pop.call(key);
+
+ PythonObject pyNewKey = new PythonObject(collapsedKey);
+ globals.set(pyNewKey, val);
+ }
+ }
+ } catch (Exception pe) {
+ throw new RuntimeException(pe);
+ }
+ }
+
+ private static void expandContext(String contextName) {
+ try (PythonGC _ = PythonGC.watch()) {
+ String prefix = getContextPrefix(contextName);
+ PythonObject globals = Python.globals();
+ PythonObject pop = globals.attr("pop");
+ PythonObject keysF = globals.attr("keys");
+
+ PythonObject keys = keysF.call();
+
+ PythonObject keysList = Python.list(keys);
+ try (PythonGC __ = PythonGC.pause()) {
+ int numKeys = Python.len(keysList).toInt();
+
+ for (int i = 0; i < numKeys; i++) {
+ PythonObject key = keysList.get(i);
+ String keyStr = key.toString();
+ if (keyStr.startsWith(prefix)) {
+ String expandedKey = expandCollapsedVarName(keyStr, contextName);
+ PythonObject val = pop.call(key);
+ PythonObject newKey = new PythonObject(expandedKey);
+ globals.set(newKey, val);
+ }
+ }
+ }
+ }
+ }
+
+
+ /**
+ * Activates the specified context
+ * @param contextName
+ */
+ public static void setContext(String contextName) {
+ if (contextName.equals(currentContext)) {
+ return;
+ }
+ if (!hasContext(contextName)) {
+ addContext(contextName);
+ }
+
+
+ collapseContext(currentContext);
+
+ expandContext(contextName);
+ currentContext = contextName;
+
+ }
+
+ /**
+ * Activates the main context
+ */
+ public static void setMainContext() {
+ setContext(MAIN_CONTEXT);
+
+ }
+
+ /**
+ * Returns the current context's name.
+ * @return
+ */
+ public static String getCurrentContext() {
+ return currentContext;
+ }
+
+ /**
+ * Resets the current context.
+ */
+ public static void reset() {
+ String tempContext = "___temp__context___";
+ String currContext = currentContext;
+ setContext(tempContext);
+ deleteContext(currContext);
+ setContext(currContext);
+ }
+
+ /**
+ * Deletes the specified context.
+ * @param contextName
+ */
+ public static void deleteContext(String contextName) {
+ if (contextName.equals(currentContext)) {
+ throw new PythonException("Cannot delete current context!");
+ }
+ if (!contexts.contains(contextName)) {
+ return;
+ }
+ String prefix = getContextPrefix(contextName);
+ PythonObject globals = Python.globals();
+ PythonObject keysList = Python.list(globals.attr("keys").call());
+ int numKeys = Python.len(keysList).toInt();
+ for (int i = 0; i < numKeys; i++) {
+ PythonObject key = keysList.get(i);
+ String keyStr = key.toString();
+ if (keyStr.startsWith(prefix)) {
+ globals.attr("__delitem__").call(key);
+ }
+ }
+ contexts.remove(contextName);
+ }
+
+ /**
+ * Deletes all contexts except the main context.
+ */
+ public static void deleteNonMainContexts() {
+ setContext(MAIN_CONTEXT); // will never fail
+ for (String c : contexts.toArray(new String[0])) {
+ if (!c.equals(MAIN_CONTEXT)) {
+ deleteContext(c); // will never fail
+ }
+ }
+
+ }
+
+ /**
+ * Returns the names of all contexts.
+ * @return
+ */
+ public String[] getContexts() {
+ return contexts.toArray(new String[0]);
+ }
+
+}
diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonException.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonException.java
new file mode 100644
index 000000000..a9bbf596c
--- /dev/null
+++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonException.java
@@ -0,0 +1,52 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Konduit K.K.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+package org.eclipse.python4j;
+
+
+/**
+ * Thrown when an exception occurs in python land
+ */
+public class PythonException extends RuntimeException {
+ public PythonException(String message) {
+ super(message);
+ }
+
+ private static String getExceptionString(PythonObject exception) {
+ try (PythonGC gc = PythonGC.watch()) {
+ if (Python.isinstance(exception, Python.ExceptionType())) {
+ String exceptionClass = Python.type(exception).attr("__name__").toString();
+ String message = exception.toString();
+ return exceptionClass + ": " + message;
+ }
+ return exception.toString();
+ } catch (Exception e) {
+ throw new RuntimeException("An error occurred while trying to create a PythonException.", e);
+ }
+ }
+
+ public PythonException(PythonObject exception) {
+ this(getExceptionString(exception));
+ }
+
+ public PythonException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public PythonException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonExecutioner.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonExecutioner.java
new file mode 100644
index 000000000..57e1a22ae
--- /dev/null
+++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonExecutioner.java
@@ -0,0 +1,342 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Konduit K.K.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+
+package org.eclipse.python4j;
+
+import org.bytedeco.cpython.PyObject;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.io.IOUtils;
+import org.bytedeco.cpython.global.python;
+
+import static org.bytedeco.cpython.global.python.*;
+import static org.bytedeco.cpython.global.python.PyImport_ImportModule;
+import static org.bytedeco.cpython.helper.python.Py_SetPath;
+
+
+public class PythonExecutioner {
+ private final static String PYTHON_EXCEPTION_KEY = "__python_exception__";
+ private static AtomicBoolean init = new AtomicBoolean(false);
+ private final static String DEFAULT_PYTHON_PATH_PROPERTY = "org.eclipse.python4j.path";
+ private final static String JAVACPP_PYTHON_APPEND_TYPE = "org.eclipse.python4j.path.append";
+ private final static String DEFAULT_APPEND_TYPE = "before";
+
+ static {
+ init();
+ }
+
+ private static synchronized void init() {
+ if (init.get()) {
+ return;
+ }
+ init.set(true);
+ initPythonPath();
+ PyEval_InitThreads();
+ Py_InitializeEx(0);
+ }
+
+ /**
+ * Sets a variable.
+ *
+ * @param name
+ * @param value
+ */
+ public static void setVariable(String name, PythonObject value) {
+ PythonGIL.assertThreadSafe();
+ PyObject main = PyImport_ImportModule("__main__");
+ PyObject globals = PyModule_GetDict(main);
+ PyDict_SetItemString(globals, name, value.getNativePythonObject());
+ Py_DecRef(main);
+
+ }
+
+ /**
+ * Sets given list of PythonVariables in the interpreter.
+ *
+ * @param pyVars
+ */
+ public static void setVariables(List pyVars) {
+ for (PythonVariable pyVar : pyVars)
+ setVariable(pyVar.getName(), pyVar.getPythonObject());
+ }
+
+ /**
+ * Sets given list of PythonVariables in the interpreter.
+ *
+ * @param pyVars
+ */
+ public static void setVariables(PythonVariable... pyVars) {
+ setVariables(Arrays.asList(pyVars));
+ }
+
+ /**
+ * Gets the given list of PythonVariables from the interpreter.
+ *
+ * @param pyVars
+ */
+ public static void getVariables(List pyVars) {
+ for (PythonVariable pyVar : pyVars)
+ pyVar.setValue(getVariable(pyVar.getName(), pyVar.getType()).getValue());
+ }
+
+ /**
+ * Gets the given list of PythonVariables from the interpreter.
+ *
+ * @param pyVars
+ */
+ public static void getVariables(PythonVariable... pyVars) {
+ getVariables(Arrays.asList(pyVars));
+ }
+
+ /**
+ * Gets the variable with the given name from the interpreter.
+ *
+ * @param name
+ * @return
+ */
+ public static PythonObject getVariable(String name) {
+ PythonGIL.assertThreadSafe();
+ PyObject main = PyImport_ImportModule("__main__");
+ PyObject globals = PyModule_GetDict(main);
+ PyObject pyName = PyUnicode_FromString(name);
+ try {
+ if (PyDict_Contains(globals, pyName) == 1) {
+ return new PythonObject(PyObject_GetItem(globals, pyName), false);
+ }
+ } finally {
+ Py_DecRef(main);
+ //Py_DecRef(globals);
+ Py_DecRef(pyName);
+ }
+ return new PythonObject(null);
+ }
+
+ /**
+ * Gets the variable with the given name from the interpreter.
+ *
+ * @param name
+ * @return
+ */
+ public static PythonVariable getVariable(String name, PythonType type) {
+ PythonObject val = getVariable(name);
+ return new PythonVariable<>(name, type, type.toJava(val));
+ }
+
+ /**
+ * Executes a string of code
+ *
+ * @param code
+ */
+ public static synchronized void simpleExec(String code) {
+ PythonGIL.assertThreadSafe();
+ int result = PyRun_SimpleStringFlags(code, null);
+ if (result != 0) {
+ throw new PythonException("Execution failed, unable to retrieve python exception.");
+ }
+ }
+
+ private static void throwIfExecutionFailed() {
+ PythonObject ex = getVariable(PYTHON_EXCEPTION_KEY);
+ if (ex != null && !ex.isNone() && !ex.toString().isEmpty()) {
+ setVariable(PYTHON_EXCEPTION_KEY, PythonTypes.STR.toPython(""));
+ throw new PythonException(ex);
+ }
+ }
+
+
+ private static String getWrappedCode(String code) {
+
+ try (InputStream is = PythonExecutioner.class
+ .getResourceAsStream("pythonexec/pythonexec.py")) {
+ String base = IOUtils.toString(is, StandardCharsets.UTF_8);
+ String indentedCode = " " + code.replace("\n", "\n ");
+ String out = base.replace(" pass", indentedCode);
+ return out;
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to read python code!", e);
+ }
+
+ }
+
+ /**
+ * Executes a string of code. Throws PythonException if execution fails.
+ *
+ * @param code
+ */
+ public static void exec(String code) {
+ simpleExec(getWrappedCode(code));
+ throwIfExecutionFailed();
+ }
+
+ public static void exec(String code, List inputs, List outputs) {
+ if (inputs != null) {
+ setVariables(inputs.toArray(new PythonVariable[0]));
+ }
+ exec(code);
+ if (outputs != null) {
+ getVariables(outputs.toArray(new PythonVariable[0]));
+ }
+ }
+
+ /**
+ * Return list of all supported variables in the interpreter.
+ *
+ * @return
+ */
+ public static List getAllVariables() {
+ PythonGIL.assertThreadSafe();
+ List ret = new ArrayList<>();
+ PyObject main = PyImport_ImportModule("__main__");
+ PyObject globals = PyModule_GetDict(main);
+ PyObject keys = PyDict_Keys(globals);
+ PyObject keysIter = PyObject_GetIter(keys);
+ try {
+
+ long n = PyObject_Size(globals);
+ for (int i = 0; i < n; i++) {
+ PyObject pyKey = PyIter_Next(keysIter);
+ try {
+ if (!new PythonObject(pyKey, false).toString().startsWith("_")) {
+
+ PyObject pyVal = PyObject_GetItem(globals, pyKey); // TODO check ref count
+ PythonType pt;
+ try {
+ pt = PythonTypes.getPythonTypeForPythonObject(new PythonObject(pyVal, false));
+
+ } catch (PythonException pe) {
+ pt = null;
+ }
+ if (pt != null) {
+ ret.add(
+ new PythonVariable<>(
+ new PythonObject(pyKey, false).toString(),
+ pt,
+ pt.toJava(new PythonObject(pyVal, false))
+ )
+ );
+ }
+ }
+ } finally {
+ Py_DecRef(pyKey);
+ }
+ }
+ } finally {
+ Py_DecRef(keysIter);
+ Py_DecRef(keys);
+ Py_DecRef(main);
+ return ret;
+ }
+
+ }
+
+
+ /**
+ * Executes a string of code and returns a list of all supported variables.
+ *
+ * @param code
+ * @param inputs
+ * @return
+ */
+ public static List execAndReturnAllVariables(String code, List inputs) {
+ setVariables(inputs);
+ simpleExec(getWrappedCode(code));
+ return getAllVariables();
+ }
+
+ /**
+ * Executes a string of code and returns a list of all supported variables.
+ *
+ * @param code
+ * @return
+ */
+ public static List execAndReturnAllVariables(String code) {
+ simpleExec(getWrappedCode(code));
+ return getAllVariables();
+ }
+
+ private static synchronized void initPythonPath() {
+ try {
+ String path = System.getProperty(DEFAULT_PYTHON_PATH_PROPERTY);
+ if (path == null) {
+ File[] packages = cachePackages();
+
+ //// TODO: fix in javacpp
+ File sitePackagesWindows = new File(python.cachePackage(), "site-packages");
+ File[] packages2 = new File[packages.length + 1];
+ for (int i = 0; i < packages.length; i++) {
+ //System.out.println(packages[i].getAbsolutePath());
+ packages2[i] = packages[i];
+ }
+ packages2[packages.length] = sitePackagesWindows;
+ //System.out.println(sitePackagesWindows.getAbsolutePath());
+ packages = packages2;
+ //////////
+
+ Py_SetPath(packages);
+ } else {
+ StringBuffer sb = new StringBuffer();
+ File[] packages = cachePackages();
+ JavaCppPathType pathAppendValue = JavaCppPathType.valueOf(System.getProperty(JAVACPP_PYTHON_APPEND_TYPE, DEFAULT_APPEND_TYPE).toUpperCase());
+ switch (pathAppendValue) {
+ case BEFORE:
+ for (File cacheDir : packages) {
+ sb.append(cacheDir);
+ sb.append(java.io.File.pathSeparator);
+ }
+
+ sb.append(path);
+ break;
+ case AFTER:
+ sb.append(path);
+
+ for (File cacheDir : packages) {
+ sb.append(cacheDir);
+ sb.append(java.io.File.pathSeparator);
+ }
+ break;
+ case NONE:
+ sb.append(path);
+ break;
+ }
+
+ Py_SetPath(sb.toString());
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private enum JavaCppPathType {
+ BEFORE, AFTER, NONE
+ }
+
+ private static File[] cachePackages() throws IOException {
+ File[] path = org.bytedeco.cpython.global.python.cachePackages();
+ path = Arrays.copyOf(path, path.length + 1);
+ path[path.length - 1] = cachePackage();
+ return path;
+ }
+
+}
diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonGC.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonGC.java
new file mode 100644
index 000000000..5531b67d3
--- /dev/null
+++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonGC.java
@@ -0,0 +1,137 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Konduit K.K.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+
+package org.eclipse.python4j;
+
+import org.bytedeco.cpython.PyObject;
+import org.bytedeco.javacpp.Pointer;
+
+import java.io.Closeable;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.bytedeco.cpython.global.python.*;
+
+/**
+ * Wrap your code in a try-with-PythonGC block for automatic GC:
+ * ```
+ * try(PythonGC gc = PythonGC.lock()){
+ * // your code here
+ * }
+ *
+ * If a PythonObject created inside such a block has to be used outside
+ * the block, use PythonGC.keep() to exclude that object from GC.
+ *
+ * ```
+ * PythonObject pyObj;
+ *
+ * try(PythonGC gc = PythonG.lock()){
+ * // do stuff
+ * pyObj = someFunction();
+ * PythonGC.keep(pyObj);
+ * }
+ *
+ */
+public class PythonGC implements Closeable {
+
+ private PythonGC previousFrame = null;
+ private boolean active = true;
+ private static PythonGC currentFrame = new PythonGC();
+
+ private Set objects = new HashSet<>();
+
+ private boolean alreadyRegistered(PyObject pyObject) {
+ if (objects.contains(pyObject)) {
+ return true;
+ }
+ if (previousFrame == null) {
+ return false;
+ }
+ return previousFrame.alreadyRegistered(pyObject);
+
+ }
+
+ private void addObject(PythonObject pythonObject) {
+ if (!active) return;
+ if (Pointer.isNull(pythonObject.getNativePythonObject()))return;
+ if (alreadyRegistered(pythonObject.getNativePythonObject())) {
+ return;
+ }
+ objects.add(pythonObject.getNativePythonObject());
+ }
+
+ public static void register(PythonObject pythonObject) {
+ currentFrame.addObject(pythonObject);
+ }
+
+ public static void keep(PythonObject pythonObject) {
+ currentFrame.objects.remove(pythonObject.getNativePythonObject());
+ if (currentFrame.previousFrame != null) {
+ currentFrame.previousFrame.addObject(pythonObject);
+ }
+ }
+
+ private PythonGC() {
+ }
+
+ public static PythonGC watch() {
+ PythonGC ret = new PythonGC();
+ ret.previousFrame = currentFrame;
+ ret.active = currentFrame.active;
+ currentFrame = ret;
+ return ret;
+ }
+
+ private void collect() {
+ for (PyObject pyObject : objects) {
+ // TODO find out how globals gets collected here
+ if (pyObject.equals(Python.globals().getNativePythonObject())) continue;
+// try{
+// System.out.println(PythonTypes.STR.toJava(new PythonObject(pyObject, false)));
+// }catch (Exception e){}
+ Py_DecRef(pyObject);
+
+ }
+ this.objects = new HashSet<>();
+ }
+
+ @Override
+ public void close() {
+ if (active) collect();
+ currentFrame = previousFrame;
+ }
+
+ public static boolean isWatching() {
+ if (!currentFrame.active) return false;
+ return currentFrame.previousFrame != null;
+ }
+
+ public static PythonGC pause() {
+ PythonGC pausedFrame = new PythonGC();
+ pausedFrame.active = false;
+ pausedFrame.previousFrame = currentFrame;
+ currentFrame = pausedFrame;
+ return pausedFrame;
+ }
+
+ public static void resume() {
+ if (currentFrame.active) {
+ throw new RuntimeException("GC not paused!");
+ }
+ currentFrame = currentFrame.previousFrame;
+ }
+}
diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonGIL.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonGIL.java
new file mode 100644
index 000000000..46b3db431
--- /dev/null
+++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonGIL.java
@@ -0,0 +1,93 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Konduit K.K.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+package org.eclipse.python4j;
+
+
+import org.bytedeco.cpython.PyThreadState;
+import org.omg.SendingContext.RunTime;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.bytedeco.cpython.global.python.*;
+
+
+public class PythonGIL implements AutoCloseable {
+ private static PyThreadState mainThreadState;
+ private static final AtomicBoolean acquired = new AtomicBoolean();
+ private boolean acquiredByMe = false;
+ private static long defaultThreadId = -1;
+
+ public static void assertThreadSafe() {
+ if (acquired.get()) {
+ return;
+ }
+ if (defaultThreadId == -1) {
+ defaultThreadId = Thread.currentThread().getId();
+ } else if (defaultThreadId != Thread.currentThread().getId()) {
+ throw new RuntimeException("Attempt to use Python4j from multiple threads without " +
+ "acquiring GIL. Enclose your code in a try(PythonGIL gil = PythonGIL.lock()){...}" +
+ " block to ensure that GIL is acquired in multi-threaded environments.");
+ }
+
+
+ }
+
+ static {
+ new PythonExecutioner();
+ }
+
+ private PythonGIL() {
+ while (acquired.get()) {
+ try {
+ Thread.sleep(10);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+ acquire();
+ acquired.set(true);
+ acquiredByMe = true;
+
+ }
+
+ @Override
+ public void close() {
+ if (acquiredByMe) {
+ release();
+ acquired.set(false);
+ acquiredByMe = false;
+ }
+
+ }
+
+ public static synchronized PythonGIL lock() {
+ return new PythonGIL();
+ }
+
+ private static synchronized void acquire() {
+ mainThreadState = PyEval_SaveThread();
+ PyThreadState ts = PyThreadState_New(mainThreadState.interp());
+ PyEval_RestoreThread(ts);
+ PyThreadState_Swap(ts);
+ }
+
+ private static void release() { // do not synchronize!
+ PyEval_SaveThread();
+ PyEval_RestoreThread(mainThreadState);
+ }
+}
diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonJob.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonJob.java
new file mode 100644
index 000000000..cdbb1b81d
--- /dev/null
+++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonJob.java
@@ -0,0 +1,175 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Konduit K.K.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+package org.eclipse.python4j;
+
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+
+
+@Data
+@NoArgsConstructor
+/**
+ * PythonJob is the right abstraction for executing multiple python scripts
+ * in a multi thread stateful environment. The setup-and-run mode allows your
+ * "setup" code (imports, model loading etc) to be executed only once.
+ */
+public class PythonJob {
+
+ private String code;
+ private String name;
+ private String context;
+ private boolean setupRunMode;
+ private PythonObject runF;
+
+ static {
+ new PythonExecutioner();
+ }
+
+ @Builder
+ /**
+ * @param name Name for the python job.
+ * @param code Python code.
+ * @param setupRunMode If true, the python code is expected to have two methods: setup(), which takes no arguments,
+ * and run() which takes some or no arguments. setup() method is executed once,
+ * and the run() method is called with the inputs(if any) per transaction, and is expected to return a dictionary
+ * mapping from output variable names (str) to output values.
+ * If false, the full script is run on each transaction and the output variables are obtained from the global namespace
+ * after execution.
+ */
+ public PythonJob(@Nonnull String name, @Nonnull String code, boolean setupRunMode){
+ this.name = name;
+ this.code = code;
+ this.setupRunMode = setupRunMode;
+ context = "__job_" + name;
+ if (PythonContextManager.hasContext(context)) {
+ throw new PythonException("Unable to create python job " + name + ". Context " + context + " already exists!");
+ }
+ if (setupRunMode) setup();
+ }
+
+
+ /**
+ * Clears all variables in current context and calls setup()
+ */
+ public void clearState(){
+ String context = this.context;
+ PythonContextManager.setContext("main");
+ PythonContextManager.deleteContext(context);
+ this.context = context;
+ setup();
+ }
+
+ public void setup(){
+ try (PythonGIL gil = PythonGIL.lock()) {
+ PythonContextManager.setContext(context);
+ PythonObject runF = PythonExecutioner.getVariable("run");
+ if (runF == null || runF.isNone() || !Python.callable(runF)) {
+ PythonExecutioner.exec(code);
+ runF = PythonExecutioner.getVariable("run");
+ }
+ if (runF.isNone() || !Python.callable(runF)) {
+ throw new PythonException("run() method not found! " +
+ "If a PythonJob is created with 'setup and run' " +
+ "mode enabled, the associated python code is " +
+ "expected to contain a run() method " +
+ "(with or without arguments).");
+ }
+ this.runF = runF;
+ PythonObject setupF = PythonExecutioner.getVariable("setup");
+ if (!setupF.isNone()) {
+ setupF.call();
+ }
+ }
+ }
+
+ public void exec(List inputs, List outputs) {
+ try (PythonGIL gil = PythonGIL.lock()) {
+ try (PythonGC _ = PythonGC.watch()) {
+ PythonContextManager.setContext(context);
+
+ if (!setupRunMode) {
+
+ PythonExecutioner.exec(code, inputs, outputs);
+
+ return;
+ }
+ PythonExecutioner.setVariables(inputs);
+
+ PythonObject inspect = Python.importModule("inspect");
+ PythonObject getfullargspec = inspect.attr("getfullargspec");
+ PythonObject argspec = getfullargspec.call(runF);
+ PythonObject argsList = argspec.attr("args");
+ PythonObject runargs = Python.dict();
+ int argsCount = Python.len(argsList).toInt();
+ for (int i = 0; i < argsCount; i++) {
+ PythonObject arg = argsList.get(i);
+ PythonObject val = Python.globals().get(arg);
+ if (val.isNone()) {
+ throw new PythonException("Input value not received for run() argument: " + arg.toString());
+ }
+ runargs.set(arg, val);
+ }
+ PythonObject outDict = runF.callWithKwargs(runargs);
+ PythonObject globals = Python.globals();
+ PythonObject updateF = globals.attr("update");
+ updateF.call(outDict);
+ PythonExecutioner.getVariables(outputs);
+ }
+ }
+
+ }
+
+ public List execAndReturnAllVariables(List inputs){
+ try (PythonGIL gil = PythonGIL.lock()) {
+ try (PythonGC _ = PythonGC.watch()) {
+ PythonContextManager.setContext(context);
+ if (!setupRunMode) {
+ return PythonExecutioner.execAndReturnAllVariables(code, inputs);
+ }
+ PythonExecutioner.setVariables(inputs);
+ PythonObject inspect = Python.importModule("inspect");
+ PythonObject getfullargspec = inspect.attr("getfullargspec");
+ PythonObject argspec = getfullargspec.call(runF);
+ PythonObject argsList = argspec.attr("args");
+ PythonObject runargs = Python.dict();
+ int argsCount = Python.len(argsList).toInt();
+ for (int i = 0; i < argsCount; i++) {
+ PythonObject arg = argsList.get(i);
+ PythonObject val = Python.globals().get(arg);
+ if (val.isNone()) {
+ throw new PythonException("Input value not received for run() argument: " + arg.toString());
+ }
+ runargs.set(arg, val);
+ }
+
+ PythonObject outDict = runF.callWithKwargs(runargs);
+ PythonObject globals = Python.globals();
+ PythonObject updateF = globals.attr("update");
+ updateF.call(outDict);
+ return PythonExecutioner.getAllVariables();
+ }
+
+ }
+ }
+
+
+}
diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonObject.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonObject.java
new file mode 100644
index 000000000..f8ec17ed9
--- /dev/null
+++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonObject.java
@@ -0,0 +1,244 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Konduit K.K.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+package org.eclipse.python4j;
+
+
+import org.bytedeco.cpython.PyObject;
+import org.bytedeco.javacpp.Pointer;
+
+import java.util.*;
+
+import static org.bytedeco.cpython.global.python.*;
+
+public class PythonObject {
+
+ static {
+ new PythonExecutioner();
+ }
+
+ private boolean owned = true;
+ private PyObject nativePythonObject;
+
+
+ public PythonObject(PyObject nativePythonObject, boolean owned) {
+ PythonGIL.assertThreadSafe();
+ this.nativePythonObject = nativePythonObject;
+ this.owned = owned;
+ if (owned && nativePythonObject != null) {
+ PythonGC.register(this);
+ }
+ }
+
+ public PythonObject(PyObject nativePythonObject) {
+ PythonGIL.assertThreadSafe();
+ this.nativePythonObject = nativePythonObject;
+ if (nativePythonObject != null) {
+ PythonGC.register(this);
+ }
+
+ }
+
+ public PyObject getNativePythonObject() {
+ return nativePythonObject;
+ }
+
+ public String toString() {
+ return PythonTypes.STR.toJava(this);
+
+ }
+
+ public boolean isNone() {
+ if (nativePythonObject == null || Pointer.isNull(nativePythonObject)) {
+ return true;
+ }
+ try (PythonGC _ = PythonGC.pause()) {
+ PythonObject type = Python.type(this);
+ boolean ret = Python.type(this).toString().equals("") && toString().equals("None");
+ Py_DecRef(type.nativePythonObject);
+ return ret;
+ }
+ }
+
+ public void del() {
+ PythonGIL.assertThreadSafe();
+ if (owned && nativePythonObject != null && !PythonGC.isWatching()) {
+ Py_DecRef(nativePythonObject);
+ nativePythonObject = null;
+ }
+ }
+
+ public PythonObject callWithArgs(PythonObject args) {
+ return callWithArgsAndKwargs(args, null);
+ }
+
+ public PythonObject callWithKwargs(PythonObject kwargs) {
+ if (!Python.callable(this)) {
+ throw new PythonException("Object is not callable: " + toString());
+ }
+ PyObject tuple = PyTuple_New(0);
+ PyObject dict = kwargs.nativePythonObject;
+ if (PyObject_IsInstance(dict, new PyObject(PyDict_Type())) != 1) {
+ throw new PythonException("Expected kwargs to be dict. Received: " + kwargs.toString());
+ }
+ PythonObject ret = new PythonObject(PyObject_Call(nativePythonObject, tuple, dict));
+ Py_DecRef(tuple);
+ return ret;
+ }
+
+ public PythonObject callWithArgsAndKwargs(PythonObject args, PythonObject kwargs) {
+ PythonGIL.assertThreadSafe();
+ PyObject tuple = null;
+ boolean ownsTuple = false;
+ try {
+ if (!Python.callable(this)) {
+ throw new PythonException("Object is not callable: " + toString());
+ }
+
+ if (PyObject_IsInstance(args.nativePythonObject, new PyObject(PyTuple_Type())) == 1) {
+ tuple = args.nativePythonObject;
+ } else if (PyObject_IsInstance(args.nativePythonObject, new PyObject(PyList_Type())) == 1) {
+ tuple = PyList_AsTuple(args.nativePythonObject);
+ ownsTuple = true;
+ } else {
+ throw new PythonException("Expected args to be tuple or list. Received: " + args.toString());
+ }
+ if (kwargs != null && PyObject_IsInstance(kwargs.nativePythonObject, new PyObject(PyDict_Type())) != 1) {
+ throw new PythonException("Expected kwargs to be dict. Received: " + kwargs.toString());
+ }
+ return new PythonObject(PyObject_Call(nativePythonObject, tuple, kwargs == null ? null : kwargs.nativePythonObject));
+ } finally {
+ if (ownsTuple) Py_DecRef(tuple);
+ }
+
+ }
+
+
+ public PythonObject call(Object... args) {
+ return callWithArgsAndKwargs(Arrays.asList(args), null);
+ }
+
+ public PythonObject callWithArgs(List args) {
+ return call(args, null);
+ }
+
+ public PythonObject callWithKwargs(Map kwargs) {
+ return call(null, kwargs);
+ }
+
+ public PythonObject callWithArgsAndKwargs(List args, Map kwargs) {
+ PythonGIL.assertThreadSafe();
+ try (PythonGC _ = PythonGC.watch()) {
+ if (!Python.callable(this)) {
+ throw new PythonException("Object is not callable: " + toString());
+ }
+ PythonObject pyArgs;
+ PythonObject pyKwargs;
+ if (args == null) {
+ pyArgs = new PythonObject(PyTuple_New(0));
+ } else {
+ PythonObject argsList = PythonTypes.convert(args);
+ pyArgs = new PythonObject(PyList_AsTuple(argsList.getNativePythonObject()));
+ }
+ if (kwargs == null) {
+ pyKwargs = null;
+ } else {
+ pyKwargs = PythonTypes.convert(kwargs);
+ }
+ PythonObject ret = new PythonObject(
+ PyObject_Call(
+ nativePythonObject,
+ pyArgs.nativePythonObject,
+ pyKwargs == null ? null : pyKwargs.nativePythonObject
+ )
+ );
+ PythonGC.keep(ret);
+ return ret;
+ }
+
+ }
+
+
+ public PythonObject attr(String attrName) {
+ PythonGIL.assertThreadSafe();
+ return new PythonObject(PyObject_GetAttrString(nativePythonObject, attrName));
+ }
+
+
+ public PythonObject(Object javaObject) {
+ PythonGIL.assertThreadSafe();
+ if (javaObject instanceof PythonObject) {
+ owned = false;
+ nativePythonObject = ((PythonObject) javaObject).nativePythonObject;
+ } else {
+ try (PythonGC _ = PythonGC.pause()) {
+ nativePythonObject = PythonTypes.convert(javaObject).getNativePythonObject();
+ }
+ PythonGC.register(this);
+ }
+
+ }
+
+ public int toInt() {
+ return PythonTypes.INT.toJava(this).intValue();
+ }
+
+ public long toLong() {
+ return PythonTypes.INT.toJava(this);
+ }
+
+ public float toFloat() {
+ return PythonTypes.FLOAT.toJava(this).floatValue();
+ }
+
+ public double toDouble() {
+ return PythonTypes.FLOAT.toJava(this);
+ }
+
+ public boolean toBoolean() {
+ return PythonTypes.BOOL.toJava(this);
+
+ }
+
+ public List toList() {
+ return PythonTypes.LIST.toJava(this);
+ }
+
+ public Map toMap() {
+ return PythonTypes.DICT.toJava(this);
+ }
+
+ public PythonObject get(int key) {
+ PythonGIL.assertThreadSafe();
+ return new PythonObject(PyObject_GetItem(nativePythonObject, PyLong_FromLong(key)));
+ }
+
+ public PythonObject get(String key) {
+ PythonGIL.assertThreadSafe();
+ return new PythonObject(PyObject_GetItem(nativePythonObject, PyUnicode_FromString(key)));
+ }
+
+ public PythonObject get(PythonObject key) {
+ PythonGIL.assertThreadSafe();
+ return new PythonObject(PyObject_GetItem(nativePythonObject, key.nativePythonObject));
+ }
+
+ public void set(PythonObject key, PythonObject value) {
+ PythonGIL.assertThreadSafe();
+ PyObject_SetItem(nativePythonObject, key.nativePythonObject, value.nativePythonObject);
+ }
+
+}
diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonType.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonType.java
new file mode 100644
index 000000000..b4806aa37
--- /dev/null
+++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonType.java
@@ -0,0 +1,47 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Konduit K.K.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+package org.eclipse.python4j;
+
+
+public abstract class PythonType {
+
+ private final String name;
+ private final Class javaType;
+
+ public PythonType(String name, Class javaType) {
+ this.name = name;
+ this.javaType = javaType;
+ }
+
+ public T adapt(Object javaObject) throws PythonException {
+ return (T) javaObject;
+ }
+
+ public abstract T toJava(PythonObject pythonObject);
+
+ public abstract PythonObject toPython(T javaObject);
+
+ public boolean accepts(Object javaObject) {
+ return javaType.isAssignableFrom(javaObject.getClass());
+ }
+
+ public String getName() {
+ return name;
+ }
+
+
+}
diff --git a/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonTypes.java b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonTypes.java
new file mode 100644
index 000000000..0dc20f712
--- /dev/null
+++ b/python4j/python4j-core/src/main/java/org/eclipse/python4j/PythonTypes.java
@@ -0,0 +1,344 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Konduit K.K.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+package org.eclipse.python4j;
+
+
+import org.bytedeco.cpython.PyObject;
+
+import java.util.*;
+
+import static org.bytedeco.cpython.global.python.*;
+import static org.bytedeco.cpython.global.python.Py_DecRef;
+
+public class PythonTypes {
+
+
+ private static List getPrimitiveTypes() {
+ return Arrays.asList(STR, INT, FLOAT, BOOL);
+ }
+
+ private static List getCollectionTypes() {
+ return Arrays.asList(LIST, DICT);
+ }
+
+ private static List getExternalTypes() {
+ //TODO service loader
+ return new ArrayList<>();
+ }
+
+ public static List get() {
+ List ret = new ArrayList<>();
+ ret.addAll(getPrimitiveTypes());
+ ret.addAll(getCollectionTypes());
+ ret.addAll(getExternalTypes());
+ return ret;
+ }
+
+ public static PythonType get(String name) {
+ for (PythonType pt : get()) {
+ if (pt.getName().equals(name)) { // TODO use map instead?
+ return pt;
+ }
+ }
+ throw new PythonException("Unknown python type: " + name);
+ }
+
+ public static PythonType getPythonTypeForJavaObject(Object javaObject) {
+ for (PythonType pt : get()) {
+ if (pt.accepts(javaObject)) {
+ return pt;
+ }
+ }
+ throw new PythonException("Unable to find python type for java type: " + javaObject.getClass());
+ }
+
+ public static PythonType getPythonTypeForPythonObject(PythonObject pythonObject) {
+ PyObject pyType = PyObject_Type(pythonObject.getNativePythonObject());
+ try {
+ String pyTypeStr = PythonTypes.STR.toJava(new PythonObject(pyType, false));
+
+ for (PythonType pt : get()) {
+ String pyTypeStr2 = "";
+ if (pyTypeStr.equals(pyTypeStr2)) {
+ return pt;
+ }
+ }
+ throw new PythonException("Unable to find converter for python object of type " + pyTypeStr);
+ } finally {
+ Py_DecRef(pyType);
+ }
+
+
+ }
+
+ public static PythonObject convert(Object javaObject) {
+ PythonType pt = getPythonTypeForJavaObject(javaObject);
+ return pt.toPython(pt.adapt(javaObject));
+ }
+
+ public static final PythonType STR = new PythonType("str", String.class) {
+
+ @Override
+ public String adapt(Object javaObject) {
+ if (javaObject instanceof String) {
+ return (String) javaObject;
+ }
+ throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to String");
+ }
+
+ @Override
+ public String toJava(PythonObject pythonObject) {
+ PythonGIL.assertThreadSafe();
+ PyObject repr = PyObject_Str(pythonObject.getNativePythonObject());
+ PyObject str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~");
+ String jstr = PyBytes_AsString(str).getString();
+ Py_DecRef(repr);
+ Py_DecRef(str);
+ return jstr;
+ }
+
+ @Override
+ public PythonObject toPython(String javaObject) {
+ return new PythonObject(PyUnicode_FromString(javaObject));
+ }
+ };
+
+ public static final PythonType INT = new PythonType("int", Long.class) {
+ @Override
+ public Long adapt(Object javaObject) {
+ if (javaObject instanceof Number) {
+ return ((Number) javaObject).longValue();
+ }
+ throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to Long");
+ }
+
+ @Override
+ public Long toJava(PythonObject pythonObject) {
+ PythonGIL.assertThreadSafe();
+ long val = PyLong_AsLong(pythonObject.getNativePythonObject());
+ if (val == -1 && PyErr_Occurred() != null) {
+ throw new PythonException("Could not convert value to int: " + pythonObject.toString());
+ }
+ return val;
+ }
+
+ @Override
+ public boolean accepts(Object javaObject) {
+ return (javaObject instanceof Integer) || (javaObject instanceof Long);
+ }
+
+ @Override
+ public PythonObject toPython(Long javaObject) {
+ return new PythonObject(PyLong_FromLong(javaObject));
+ }
+ };
+
+ public static final PythonType FLOAT = new PythonType("float", Double.class) {
+
+ @Override
+ public Double adapt(Object javaObject) {
+ if (javaObject instanceof Number) {
+ return ((Number) javaObject).doubleValue();
+ }
+ throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to Long");
+ }
+
+ @Override
+ public Double toJava(PythonObject pythonObject) {
+ PythonGIL.assertThreadSafe();
+ double val = PyFloat_AsDouble(pythonObject.getNativePythonObject());
+ if (val == -1 && PyErr_Occurred() != null) {
+ throw new PythonException("Could not convert value to float: " + pythonObject.toString());
+ }
+ return val;
+ }
+
+ @Override
+ public boolean accepts(Object javaObject) {
+ return (javaObject instanceof Float) || (javaObject instanceof Double);
+ }
+
+ @Override
+ public PythonObject toPython(Double javaObject) {
+ return new PythonObject(PyFloat_FromDouble(javaObject));
+ }
+ };
+
+
+ public static final PythonType BOOL = new PythonType("bool", Boolean.class) {
+
+ @Override
+ public Boolean adapt(Object javaObject) {
+ if (javaObject instanceof Boolean) {
+ return (Boolean) javaObject;
+ }
+ throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to Boolean");
+ }
+
+ @Override
+ public Boolean toJava(PythonObject pythonObject) {
+ PythonGIL.assertThreadSafe();
+ PyObject builtins = PyImport_ImportModule("builtins");
+ PyObject boolF = PyObject_GetAttrString(builtins, "bool");
+
+ PythonObject bool = new PythonObject(boolF, false).call(pythonObject);
+ boolean ret = PyLong_AsLong(bool.getNativePythonObject()) > 0;
+ bool.del();
+ Py_DecRef(boolF);
+ Py_DecRef(builtins);
+ return ret;
+ }
+
+ @Override
+ public PythonObject toPython(Boolean javaObject) {
+ return new PythonObject(PyBool_FromLong(javaObject ? 1 : 0));
+ }
+ };
+
+
+ public static final PythonType LIST = new PythonType("list", List.class) {
+
+ @Override
+ public List adapt(Object javaObject) {
+ if (javaObject instanceof List) {
+ return (List) javaObject;
+ } else if (javaObject instanceof Object[]) {
+ return Arrays.asList((Object[]) javaObject);
+ } else {
+ throw new PythonException("Cannot cast object of type " + javaObject.getClass().getName() + " to List");
+ }
+ }
+
+ @Override
+ public List toJava(PythonObject pythonObject) {
+ PythonGIL.assertThreadSafe();
+ List ret = new ArrayList();
+ long n = PyObject_Size(pythonObject.getNativePythonObject());
+ if (n < 0) {
+ throw new PythonException("Object cannot be interpreted as a List");
+ }
+ for (long i = 0; i < n; i++) {
+ PyObject pyIndex = PyLong_FromLong(i);
+ PyObject pyItem = PyObject_GetItem(pythonObject.getNativePythonObject(),
+ pyIndex);
+ Py_DecRef(pyIndex);
+ PythonType pyItemType = getPythonTypeForPythonObject(new PythonObject(pyItem, false));
+ ret.add(pyItemType.toJava(new PythonObject(pyItem, false)));
+ Py_DecRef(pyItem);
+ }
+ return ret;
+ }
+
+ @Override
+ public PythonObject toPython(List javaObject) {
+ PythonGIL.assertThreadSafe();
+ PyObject pyList = PyList_New(javaObject.size());
+ for (int i = 0; i < javaObject.size(); i++) {
+ Object item = javaObject.get(i);
+ PythonObject pyItem;
+ boolean owned;
+ if (item instanceof PythonObject) {
+ pyItem = (PythonObject) item;
+ owned = false;
+ } else if (item instanceof PyObject) {
+ pyItem = new PythonObject((PyObject) item, false);
+ owned = false;
+ } else {
+ pyItem = PythonTypes.convert(item);
+ owned = true;
+ }
+ Py_IncRef(pyItem.getNativePythonObject()); // reference will be stolen by PyList_SetItem()
+ PyList_SetItem(pyList, i, pyItem.getNativePythonObject());
+ if (owned) pyItem.del();
+ }
+ return new PythonObject(pyList);
+ }
+ };
+
+ public static final PythonType