Implement python backend

6 files changed, 514 insertions, 3 deletions

diff --git a/.travis.yml b/.travis.yml
index 2900b96..d9c03d3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,6 +23,7 @@ addons:
      - libola-dev
      - libjack-jackd2-dev
      - liblua5.3-dev
+     - python3-dev
      - libssl-dev
      - lintian
     packages: &core_build_gpp_latest
@@ -143,7 +144,7 @@ before_install:
  - git pull --tags
  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
 # 'brew install' sometimes returns non-zero for some arcane reason. Executing 'true' resets the exit code and allows Travis to continue building...
- - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install ccache ola lua openssl jack; true; fi
+ - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install ccache ola lua openssl jack python3; true; fi
 # OpenSSL is not a proper install due to some Apple bull, so provide additional locations via the environment...
 # Additionally, newer versions of this "recipe" seem to use the name 'openssl@1.1' instead of plain 'openssl' and there seems to be
 # no way to programmatically get the link and include paths. Genius! Hardcoding the new version for the time being...
diff --git a/README.md b/README.md
index 27629f2..19fb62c 100644
--- a/README.md
+++ b/README.md
@@ -22,6 +22,7 @@ Currently, the MIDIMonster supports the following protocols:
 | MA Lighting Web Remote	| Linux, Windows, OSX	| GrandMA2 and dot2 (incl. OnPC)	| [`maweb`](backends/maweb.md)	|
 | JACK/LV2 Control Voltage (CV)	| Linux, OSX		|				| [`jack`](backends/jack.md)		|
 | Lua Scripting			| Linux, Windows, OSX	|				| [`lua`](backends/lua.md)		|
+| Python Scripting		| Linux, OSX		|				| [`python`](backends/python.md)	|
 | Loopback			| Linux, Windows, OSX	|				| [`loopback`](backends/loopback.md)	|
 
 With these features, the MIDIMonster allows users to control any channel on any of these protocols, and translate any channel on
@@ -146,6 +147,7 @@ special information. These documentation files are located in the `backends/` di
 * [`osc` backend documentation](backends/osc.md)
 * [`openpixelcontrol` backend documentation](backends/openpixelcontrol.md)
 * [`lua` backend documentation](backends/lua.md)
+* [`python` backend documentation](backends/python.md)
 * [`maweb` backend documentation](backends/maweb.md)
 
 ## Installation
@@ -192,8 +194,9 @@ support for the protocols to translate.
 * `liblua5.3-dev` (for the lua backend)
 * `libola-dev` (for the optional OLA backend)
 * `libjack-jackd2-dev` (for the JACK backend)
-* `pkg-config` (as some projects and systems like to spread their files around)
 * `libssl-dev` (for the MA Web Remote backend)
+* `python3-dev` (for the Python backend)
+* `pkg-config` (as some projects and systems like to spread their files around)
 * A C compiler
 * GNUmake
 
diff --git a/backends/Makefile b/backends/Makefile
index 191a495..366d4b4 100644
--- a/backends/Makefile
+++ b/backends/Makefile
@@ -1,7 +1,7 @@
 .PHONY: all clean full
 LINUX_BACKENDS = midi.so evdev.so
 WINDOWS_BACKENDS = artnet.dll osc.dll loopback.dll sacn.dll maweb.dll winmidi.dll openpixelcontrol.dll
-BACKENDS = artnet.so osc.so loopback.so sacn.so lua.so maweb.so jack.so openpixelcontrol.so
+BACKENDS = artnet.so osc.so loopback.so sacn.so lua.so maweb.so jack.so openpixelcontrol.so python.so
 OPTIONAL_BACKENDS = ola.so
 BACKEND_LIB = libmmbackend.o
 
diff --git a/backends/python.c b/backends/python.c
new file mode 100644
index 0000000..9fad0ae
--- /dev/null
+++ b/backends/python.c
@@ -0,0 +1,419 @@
+#define BACKEND_NAME "python"
+
+#define PY_SSIZE_T_CLEAN
+#include <string.h>
+#include <Python.h>
+#include "python.h"
+
+#define MMPY_INSTANCE_KEY "midimonster_instance"
+
+/*
+ * TODO might want to export the full MM_API set to python at some point
+ */
+
+static PyThreadState* python_main = NULL;
+static wchar_t* program_name = NULL;
+
+MM_PLUGIN_API int init(){
+	backend python = {
+		.name = BACKEND_NAME,
+		.conf = python_configure,
+		.create = python_instance,
+		.conf_instance = python_configure_instance,
+		.channel = python_channel,
+		.handle = python_set,
+		.process = python_handle,
+		.start = python_start,
+		.shutdown = python_shutdown
+	};
+
+	//register backend
+	if(mm_backend_register(python)){
+		LOG("Failed to register backend");
+		return 1;
+	}
+	return 0;
+}
+
+static int python_configure(char* option, char* value){
+	LOG("No backend configuration possible");
+	return 1;
+}
+
+static int python_prepend_str(PyObject* list, char* str){
+	if(!list || !str){
+		return 1;
+	}
+
+	PyObject* item = PyUnicode_FromString(str);
+	if(!item){
+		return 1;
+	}
+
+	if(PyList_Insert(list, 0, item) < 0){
+		Py_DECREF(item);
+		return 1;
+	}
+	Py_DECREF(item);
+	return 0;
+}
+
+static PyObject* mmpy_output(PyObject* self, PyObject* args){
+	instance* inst = *((instance**) PyModule_GetState(self));
+	python_instance_data* data = (python_instance_data*) inst->impl;
+	const char* channel_name = NULL;
+	channel* chan = NULL;
+	channel_value val = {
+		0
+	};
+	size_t u;
+
+	if(!PyArg_ParseTuple(args, "sd", &channel_name, &val.normalised)){
+		return NULL;
+	}
+
+	val.normalised = clamp(val.normalised, 1.0, 0.0);
+
+	for(u = 0; u < data->channels; u++){
+		if(!strcmp(data->channel[u].name, channel_name)){
+			DBGPF("Setting channel %s.%s to %f", inst->name, channel_name, val.normalised);
+			chan = mm_channel(inst, u, 0);
+			//this should never happen
+			if(!chan){
+				LOGPF("Failed to fetch parsed channel %s.%s", inst->name, channel_name);
+				break;
+			}
+			data->channel[u].out = val.normalised;
+			mm_channel_event(chan, val);
+			break;
+		}
+	}
+
+	if(u == data->channels){
+		DBGPF("Output on unknown channel %s.%s, no event pushed", inst->name, channel_name);
+	}
+
+	Py_INCREF(Py_None);
+	return Py_None;
+}
+
+static PyObject* mmpy_channel_value(PyObject* self, PyObject* args, uint8_t in){
+	instance* inst = *((instance**) PyModule_GetState(self));
+	python_instance_data* data = (python_instance_data*) inst->impl;
+	const char* channel_name = NULL;
+	size_t u;
+
+	if(!PyArg_ParseTuple(args, "s", &channel_name)){
+		return NULL;
+	}
+
+	for(u = 0; u < data->channels; u++){
+		if(!strcmp(data->channel[u].name, channel_name)){
+			return PyFloat_FromDouble(in ? data->channel[u].in : data->channel[u].out);
+		}
+	}
+
+	Py_INCREF(Py_None);
+	return Py_None;
+}
+
+static PyObject* mmpy_current_handler(PyObject* self, PyObject* args){
+	instance* inst = *((instance**) PyModule_GetState(self));
+	python_instance_data* data = (python_instance_data*) inst->impl;
+
+	if(data->current_channel){
+		return PyUnicode_FromString(data->current_channel->name);
+	}
+
+	Py_INCREF(Py_None);
+	return Py_None;
+}
+
+static PyObject* mmpy_output_value(PyObject* self, PyObject* args){
+	return mmpy_channel_value(self, args, 0);
+}
+
+static PyObject* mmpy_input_value(PyObject* self, PyObject* args){
+	return mmpy_channel_value(self, args, 1);
+}
+
+static int mmpy_exec(PyObject* module) {
+	instance** inst = (instance**) PyModule_GetState(module);
+	//FIXME actually use interpreter dict (from python 3.8) here at some point
+	PyObject* capsule = PyDict_GetItemString(PyThreadState_GetDict(), MMPY_INSTANCE_KEY);
+	if(capsule && inst){
+		*inst = PyCapsule_GetPointer(capsule, NULL);
+		return 0;
+	}
+
+	//TODO raise exception
+	return -1;
+}
+
+static int python_configure_instance(instance* inst, char* option, char* value){
+	python_instance_data* data = (python_instance_data*) inst->impl;
+	PyObject* module = NULL;
+
+	//load python script
+	if(!strcmp(option, "module")){
+		//swap to interpreter
+		PyEval_RestoreThread(data->interpreter);
+		//import the module
+		module = PyImport_ImportModule(value);
+		if(!module){
+			LOGPF("Failed to import module %s to instance %s", value, inst->name);
+			PyErr_Print();
+		}
+		Py_XDECREF(module);
+		PyEval_ReleaseThread(data->interpreter);
+		return 0;
+	}
+
+	LOGPF("Unknown instance parameter %s for instance %s", option, inst->name);
+	return 1;
+}
+
+static PyObject* mmpy_init(){
+	static PyModuleDef_Slot mmpy_slots[] = {
+		{Py_mod_exec, (void*) mmpy_exec},
+		{0}
+	};
+
+	static PyMethodDef mmpy_methods[] = {
+		{"output", mmpy_output, METH_VARARGS, "Output a channel event"},
+		{"inputvalue", mmpy_input_value, METH_VARARGS, "Get last input value for a channel"},
+		{"outputvalue", mmpy_output_value, METH_VARARGS, "Get the last output value for a channel"},
+		{"current", mmpy_current_handler, METH_VARARGS, "Get the name of the currently executing channel handler"},
+		{0}
+	};
+
+	static struct PyModuleDef mmpy = {
+		PyModuleDef_HEAD_INIT,
+		"midimonster",
+		NULL, /*doc size*/
+		sizeof(instance*),
+		mmpy_methods,
+		mmpy_slots
+	};
+
+	//single-phase init
+	//return PyModule_Create(&mmpy);
+
+	//multi-phase init
+	return PyModuleDef_Init(&mmpy);
+}
+
+static int python_instance(instance* inst){
+	python_instance_data* data = calloc(1, sizeof(python_instance_data));
+	PyObject* interpreter_dict = NULL;
+	char current_directory[8192];
+	if(!data){
+		LOG("Failed to allocate memory");
+		return 1;
+	}
+
+	//lazy-init because we need the interpreter running before _start,
+	//but don't want it running if no instances are defined
+	if(!python_main){
+		LOG("Initializing main python interpreter");
+		if(PyImport_AppendInittab("midimonster", &mmpy_init)){
+			LOG("Failed to extend python inittab for main interpreter");
+		}
+		program_name = Py_DecodeLocale("midimonster", NULL);
+		Py_SetProgramName(program_name);
+		//initialize python
+		Py_InitializeEx(0);
+		//create, acquire and release the GIL
+		PyEval_InitThreads();
+		python_main = PyEval_SaveThread();
+	}
+
+	//acquire the GIL before creating a new interpreter
+	PyEval_RestoreThread(python_main);
+	//create subinterpreter for new instance
+	data->interpreter = Py_NewInterpreter();
+
+	//push cwd as import path
+	if(getcwd(current_directory, sizeof(current_directory))){
+		if(python_prepend_str(PySys_GetObject("path"), current_directory)){
+			LOG("Failed to push current working directory to python");
+			goto bail;
+		}
+	}
+
+	//push the instance pointer for later module initialization
+	//FIXME python 3.8 introduces interpreter_dict = PyInterpreterState_GetDict(data->interpreter->interp);
+	//for now use thread state...
+	interpreter_dict = PyThreadState_GetDict();
+	if(!interpreter_dict){
+		LOG("Failed to access per-interpreter data storage");
+		goto bail;
+	}
+	//FIXME this might leak a reference to the capsule
+	if(PyDict_SetItemString(interpreter_dict, MMPY_INSTANCE_KEY, PyCapsule_New(inst, NULL, NULL))){
+		LOG("Failed to set per-interpreter instance pointer");
+		goto bail;
+	}
+
+	//NewInterpreter leaves us with the GIL, drop it
+	PyEval_ReleaseThread(data->interpreter);
+	inst->impl = data;
+	return 0;
+
+bail:
+	if(data->interpreter){
+		PyEval_ReleaseThread(data->interpreter);
+	}
+	free(data);
+	return 1;
+}
+
+static channel* python_channel(instance* inst, char* spec, uint8_t flags){
+	python_instance_data* data = (python_instance_data*) inst->impl;
+	size_t u;
+
+	for(u = 0; u < data->channels; u++){
+		if(!strcmp(data->channel[u].name, spec)){
+			break;
+		}
+	}
+
+	if(u == data->channels){
+		data->channel = realloc(data->channel, (data->channels + 1) * sizeof(mmpython_channel));
+		if(!data->channel){
+			data->channels = 0;
+			LOG("Failed to allocate memory");
+			return NULL;
+		}
+		memset(data->channel + u, 0, sizeof(mmpython_channel));
+
+		data->channel[u].name = strdup(spec);
+		if(!data->channel[u].name){
+			LOG("Failed to allocate memory");
+			return NULL;
+		}
+		data->channels++;
+	}
+
+	return mm_channel(inst, u, 1);
+}
+
+static int python_set(instance* inst, size_t num, channel** c, channel_value* v){
+	python_instance_data* data = (python_instance_data*) inst->impl;
+	mmpython_channel* chan = NULL;
+	PyObject* result = NULL;
+	size_t u;
+
+	//swap to interpreter
+	PyEval_RestoreThread(data->interpreter);
+
+	for(u = 0; u < num; u++){
+		chan = data->channel + c[u]->ident;
+
+		//update input value buffer
+		chan->in = v[u].normalised;
+
+		//call handler if present
+		if(chan->handler){
+			DBGPF("Calling handler for %s.%s", inst->name, chan->name);
+			data->current_channel = chan;
+			result = PyObject_CallFunction(chan->handler, "d", chan->in);
+			Py_XDECREF(result);
+			data->current_channel = NULL;
+			DBGPF("Done with handler for %s.%s", inst->name, chan->name);
+		}
+	}
+
+	PyEval_ReleaseThread(data->interpreter);
+	return 0;
+}
+
+static int python_handle(size_t num, managed_fd* fds){
+	//TODO implement some kind of intervaling functionality before people get it in their heads to start `import threading`
+	return 0;
+}
+
+static int python_start(size_t n, instance** inst){
+	python_instance_data* data = NULL;
+	PyObject* module = NULL;
+	size_t u, p;
+	char* module_name = NULL, *channel_name = NULL;
+
+	//resolve channel references to handler functions
+	for(u = 0; u < n; u++){
+		data = (python_instance_data*) inst[u]->impl;
+
+		//switch to interpreter
+		PyEval_RestoreThread(data->interpreter);
+		for(p = 0; p < data->channels; p++){
+			module = PyImport_AddModule("__main__");
+			channel_name = data->channel[p].name;
+			module_name = strchr(channel_name, '.');
+			if(module_name){
+				*module_name = 0;
+				//returns borrowed reference
+				module = PyImport_AddModule(channel_name);
+
+				if(!module){
+					LOGPF("Module %s for qualified channel %s.%s is not loaded on instance %s", channel_name, channel_name, module_name + 1, inst[u]->name);
+					return 1;
+				}
+
+				*module_name = '.';
+				channel_name = module_name + 1;
+			}
+
+			//returns new reference
+			data->channel[p].handler = PyObject_GetAttrString(module, channel_name);
+		}
+
+		//release interpreter
+		PyEval_ReleaseThread(data->interpreter);
+	}
+	return 0;
+}
+
+static int python_shutdown(size_t n, instance** inst){
+	size_t u, p;
+	python_instance_data* data = NULL;
+
+	//clean up channels
+	//this needs to be done before stopping the interpreters,
+	//because the handler references are refcounted
+	for(u = 0; u < n; u++){
+		data = (python_instance_data*) inst[u]->impl;
+		for(p = 0; p < data->channels; p++){
+			free(data->channel[p].name);
+			Py_XDECREF(data->channel[p].handler);
+		}
+		free(data->channel);
+		//do not free data here, needed for shutting down interpreters
+	}
+
+	if(python_main){
+		//just used to lock the GIL
+		PyEval_RestoreThread(python_main);
+
+		for(u = 0; u < n; u++){
+			data = (python_instance_data*) inst[u]->impl;
+			DBGPF("Shutting down interpreter for instance %s", inst[u]->name);
+			//swap to interpreter and end it, GIL is held after this but state is NULL
+			PyThreadState_Swap(data->interpreter);
+			PyErr_Clear();
+			//PyThreadState_Clear(data->interpreter);
+			Py_EndInterpreter(data->interpreter);
+
+			free(data);
+		}
+
+		//shut down main interpreter
+		PyThreadState_Swap(python_main);
+		if(Py_FinalizeEx()){
+			LOG("Failed to destroy python interpreters");
+		}
+		PyMem_RawFree(program_name);
+	}
+
+	LOG("Backend shut down");
+	return 0;
+}
diff --git a/backends/python.h b/backends/python.h
new file mode 100644
index 0000000..10411ca
--- /dev/null
+++ b/backends/python.h
@@ -0,0 +1,25 @@
+#include "midimonster.h"
+
+MM_PLUGIN_API int init();
+static int python_configure(char* option, char* value);
+static int python_configure_instance(instance* inst, char* option, char* value);
+static int python_instance(instance* inst);
+static channel* python_channel(instance* inst, char* spec, uint8_t flags);
+static int python_set(instance* inst, size_t num, channel** c, channel_value* v);
+static int python_handle(size_t num, managed_fd* fds);
+static int python_start(size_t n, instance** inst);
+static int python_shutdown(size_t n, instance** inst);
+
+typedef struct /*_python_channel_data*/ {
+	char* name;
+	PyObject* handler;
+	double in;
+	double out;
+} mmpython_channel;
+
+typedef struct /*_python_instance_data*/ {
+	PyThreadState* interpreter;
+	size_t channels;
+	mmpython_channel* channel;
+	mmpython_channel* current_channel;
+} python_instance_data;
diff --git a/backends/python.md b/backends/python.md
new file mode 100644
index 0000000..5f81e70
--- /dev/null
+++ b/backends/python.md
@@ -0,0 +1,63 @@
+### The `python` backend
+
+The `python` backend provides a flexible programming environment, allowing users
+to route, generate and manipulate channel events using the Python 3 scripting languge.
+
+Every instance has its own interpreter, which can be loaded with multiple Python modules.
+These modules may contain member functions accepting a single `float` parameter, which can
+then be used as target channels. For each incoming event, the handler function is called.
+
+To interact with the MIDIMonster core, import the `midimonster` module from within your module.
+
+The `midimonster` module provides the following functions:
+
+| Function			| Usage example				| Description					|
+|-------------------------------|---------------------------------------|-----------------------------------------------|
+| `output(string, float)`	| `midimonster.output("foo", 0.75)`	| Output a value event to a channel		|
+| `inputvalue(string)`		| `midimonster.inputvalue("foo")`	| Get the last input value on a channel		|
+| `outputvalue(string)`		| `midimonster.outputvalue("bar")`	| Get the last output value on a channel 	|
+| `current()`			| `print(midimonster.current())`	| Returns the name of the input channel whose handler function is currently running or `None` if the interpreter was called from another context |
+
+Example Python module:
+```
+import midimonster
+
+def in1(value):
+	midimonster.output("out1", 1 - value)
+```
+
+Input values range between 0.0 and 1.0, output values are clamped to the same range.
+
+#### Global configuration
+
+The `python` backend does not take any global configuration.
+
+#### Instance configuration
+
+| Option	| Example value		| Default value 	| Description					|
+|---------------|-----------------------|-----------------------|-----------------------------------------------|
+| `module`	| `my_handlers.py`	| none			| (Path to) Python module source file, relative to configuration file location |
+
+A single instance may have multiple `module` options specified. This will make all handlers available within their
+module namespaces (see the section on channel specification).
+
+#### Channel specification
+
+Channel names may be any valid Python function name. To call handler functions in a module,
+specify the channel as the functions qualified path (by prefixing it with the module name and a dot).
+
+Example mappings:
+```
+py1.my_handlers.in1 < py1.foo
+py1.out1 > py2.module.handler 
+```
+
+#### Known bugs / problems
+
+Output values will not trigger corresponding input event handlers unless the channel is mapped
+back in the MIDIMonster configuration. This is intentional.
+
+Importing a Python module named `midimonster` may cause problems and is unsupported.
+
+There is currently no functionality for cyclic execution. This may be implemented in a future
+release.