From 8b016f61a4b3d3be0c7b1e311209ab991276af0c Mon Sep 17 00:00:00 2001
From: cbdev <cb@cbcdn.com>
Date: Thu, 22 Aug 2019 21:13:48 +0200
Subject: Implement input for the maweb backend (with a few limitations)

---
 README.md               |   2 +-
 backends/Makefile       |   4 +-
 backends/evdev.h        |   3 +-
 backends/libmmbackend.c | 241 ++++++++++++++++++++++++++++++--
 backends/libmmbackend.h |  33 ++---
 backends/maweb.c        | 358 ++++++++++++++++++++++++++++++++++++++++--------
 backends/maweb.h        |  14 +-
 backends/maweb.md       |  36 +++--
 backends/midi.h         |   3 +-
 backends/osc.h          |   3 +-
 10 files changed, 582 insertions(+), 115 deletions(-)

diff --git a/README.md b/README.md
index 4d8fe18..40f8fbb 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,7 @@ on any other (or the same) supported protocol, for example to:
 * Translate MIDI Control Changes into Notes ([Example configuration](configs/unifest-17.cfg))
 * Translate MIDI Notes into ArtNet or sACN ([Example configuration](configs/launchctl-sacn.cfg))
 * Translate OSC messages into MIDI ([Example configuration](configs/midi-osc.cfg))
-* Dynamically route and modify events using the Lua programming language ([Example configuration](configs/lua.cfg) and [Script](configs/demo.lua)) to create your own lighting controller or run effects on TouchOSC (Flying faders demo [configuration](configs/flying-faders.cfg) and [script](configs/flying-faders.lua))
+* Dynamically generate, route and modify events using the Lua programming language ([Example configuration](configs/lua.cfg) and [Script](configs/demo.lua)) to create your own lighting controller or run effects on TouchOSC (Flying faders demo [configuration](configs/flying-faders.cfg) and [script](configs/flying-faders.lua))
 * Use an OSC app as a simple lighting controller via ArtNet or sACN
 * Visualize ArtNet data using OSC tools
 * Control lighting fixtures or DAWs using gamepad controllers, trackballs, etc ([Example configuration](configs/evdev.cfg))
diff --git a/backends/Makefile b/backends/Makefile
index 582655c..5c5b677 100644
--- a/backends/Makefile
+++ b/backends/Makefile
@@ -1,7 +1,7 @@
 .PHONY: all clean full
 LINUX_BACKENDS = midi.so evdev.so
-WINDOWS_BACKENDS = artnet.dll osc.dll loopback.dll sacn.dll
-BACKENDS = artnet.so osc.so loopback.so sacn.so lua.so
+WINDOWS_BACKENDS = artnet.dll osc.dll loopback.dll sacn.dll maweb.dll
+BACKENDS = artnet.so osc.so loopback.so sacn.so lua.so maweb.so
 OPTIONAL_BACKENDS = ola.so
 BACKEND_LIB = libmmbackend.o
 
diff --git a/backends/evdev.h b/backends/evdev.h
index b26664b..48bd0ab 100644
--- a/backends/evdev.h
+++ b/backends/evdev.h
@@ -52,4 +52,5 @@ typedef union {
 		uint16_t code;
 	} fields;
 	uint64_t label;
-} evdev_channel_ident;
\ No newline at end of file
+} evdev_channel_ident;
+
diff --git a/backends/libmmbackend.c b/backends/libmmbackend.c
index c98cfe3..ccbeb52 100644
--- a/backends/libmmbackend.c
+++ b/backends/libmmbackend.c
@@ -208,10 +208,11 @@ size_t json_validate(char* json, size_t length){
 size_t json_validate_string(char* json, size_t length){
 	size_t string_length = 0, offset;
 
-	for(offset = 0; json[offset] && offset < length && json[offset] != '"'; offset++){
+	//skip leading whitespace
+	for(offset = 0; json[offset] && offset < length && isspace(json[offset]); offset++){
 	}
 
-	if(offset == length){
+	if(offset == length || json[offset] != '"'){
 		return 0;
 	}
 
@@ -230,17 +231,122 @@ size_t json_validate_string(char* json, size_t length){
 }
 
 size_t json_validate_array(char* json, size_t length){
-	//TODO
+	size_t offset = 0;
+
+	//skip leading whitespace
+	for(offset = 0; json[offset] && offset < length && isspace(json[offset]); offset++){
+	}
+
+	if(offset == length || json[offset] != '['){
+		return 0;
+	}
+
+	for(offset++; offset < length; offset++){
+		offset += json_validate(json + offset, length - offset);
+
+		//skip trailing whitespace, find terminator
+		for(; offset < length && isspace(json[offset]); offset++){
+		}
+
+		if(json[offset] == ','){
+			continue;
+		}
+
+		if(json[offset] == ']'){
+			return offset + 1;
+		}
+
+		break;
+	}
+
 	return 0;
 }
 
 size_t json_validate_object(char* json, size_t length){
-	//TODO
+	size_t offset = 0;
+
+	//skip whitespace
+	for(offset = 0; json[offset] && isspace(json[offset]); offset++){
+	}
+
+	if(offset == length || json[offset] != '{'){
+		return 0;
+	}
+
+	for(offset++; offset < length; offset++){
+		if(json_identify(json + offset, length - offset) != JSON_STRING){
+			//still could be an empty object...
+			for(; offset < length && isspace(json[offset]); offset++){
+			}
+			if(json[offset] == '}'){
+				return offset + 1;
+			}
+			return 0;
+		}
+		offset += json_validate(json + offset, length - offset);
+
+		//find value separator
+		for(; offset < length && isspace(json[offset]); offset++){
+		}
+
+		if(json[offset] != ':'){
+			return 0;
+		}
+
+		offset++;
+		offset += json_validate(json + offset, length - offset);
+
+		//skip trailing whitespace
+		for(; json[offset] && isspace(json[offset]); offset++){
+		}
+
+		if(json[offset] == '}'){
+			return offset + 1;
+		}
+		else if(json[offset] != ','){
+			return 0;
+		}
+	}
 	return 0;
 }
 
 size_t json_validate_value(char* json, size_t length){
-	//TODO
+	size_t offset = 0, value_length;
+
+	//skip leading whitespace
+	for(offset = 0; json[offset] && offset < length && isspace(json[offset]); offset++){
+	}
+
+	if(offset == length){
+		return 0;
+	}
+
+	//match complete values
+	if(length - offset >= 4 && !strncmp(json + offset, "null", 4)){
+		return offset + 4;
+	}
+	else if(length - offset >= 4 && !strncmp(json + offset, "true", 4)){
+		return offset + 4;
+	}
+	else if(length - offset >= 5 && !strncmp(json + offset, "false", 5)){
+		return offset + 5;
+	}
+
+	if(json[offset] == '-' || isdigit(json[offset])){
+		//json number parsing is dumb.
+		for(value_length = 1; offset + value_length < length &&
+					(isdigit(json[offset + value_length])
+					|| json[offset + value_length] == '+'
+					|| json[offset + value_length] == '-'
+					|| json[offset + value_length] == '.'
+					|| tolower(json[offset + value_length]) == 'e'); value_length++){
+		}
+
+		if(value_length > 0){
+			return offset + value_length;
+		}
+	}
+
 	return 0;
 }
 
@@ -284,13 +390,51 @@ size_t json_obj_offset(char* json, char* key){
 		//add length of value
 		offset += json_validate(json + offset, strlen(json + offset));
 
-		//find comma or closing brace
-		for(; json[offset] && json[offset] != ',' && json[offset] != '}'; offset++){
+		//skip trailing whitespace
+		for(; json[offset] && isspace(json[offset]); offset++){
 		}
 
 		if(json[offset] == ','){
 			offset++;
+			continue;
+		}
+
+		break;
+	}
+
+	return 0;
+}
+
+size_t json_array_offset(char* json, uint64_t key){
+	size_t offset = 0, index = 0;
+
+	//skip leading whitespace
+	for(offset = 0; json[offset] && isspace(json[offset]); offset++){
+	}
+
+	if(json[offset] != '['){
+		return 0;
+	}
+
+	for(offset++; index <= key; offset++){
+		//skip whitespace
+		for(; json[offset] && isspace(json[offset]); offset++){
+		}
+
+		if(index == key){
+			return offset;
+		}
+
+		offset += json_validate(json + offset, strlen(json + offset));
+
+		//skip trailing whitespace, find terminator
+		for(; json[offset] && isspace(json[offset]); offset++){
 		}
+
+		if(json[offset] != ','){
+			break;
+		}
+		index++;
 	}
 
 	return 0;
@@ -304,6 +448,14 @@ json_type json_obj(char* json, char* key){
 	return JSON_INVALID;
 }
 
+json_type json_array(char* json, uint64_t key){
+	size_t offset = json_array_offset(json, key);
+	if(offset){
+		return json_identify(json + offset, strlen(json + offset));
+	}
+	return JSON_INVALID;
+}
+
 uint8_t json_obj_bool(char* json, char* key, uint8_t fallback){
 	size_t offset = json_obj_offset(json, key);
 	if(offset){
@@ -317,6 +469,19 @@ uint8_t json_obj_bool(char* json, char* key, uint8_t fallback){
 	return fallback;
 }
 
+uint8_t json_array_bool(char* json, uint64_t key, uint8_t fallback){
+	size_t offset = json_array_offset(json, key);
+	if(offset){
+		if(!strncmp(json + offset, "true", 4)){
+			return 1;
+		}
+		if(!strncmp(json + offset, "false", 5)){
+			return 0;
+		}
+	}
+	return fallback;
+}
+
 int64_t json_obj_int(char* json, char* key, int64_t fallback){
 	char* next_token = NULL;
 	int64_t result;
@@ -332,7 +497,7 @@ int64_t json_obj_int(char* json, char* key, int64_t fallback){
 
 double json_obj_double(char* json, char* key, double fallback){
 	char* next_token = NULL;
-	int64_t result;
+	double result;
 	size_t offset = json_obj_offset(json, key);
 	if(offset){
 		result = strtod(json + offset, &next_token);
@@ -343,6 +508,32 @@ double json_obj_double(char* json, char* key, double fallback){
 	return fallback;
 }
 
+int64_t json_array_int(char* json, uint64_t key, int64_t fallback){
+	char* next_token = NULL;
+	int64_t result;
+	size_t offset = json_array_offset(json, key);
+	if(offset){
+		result = strtol(json + offset, &next_token, 10);
+		if(next_token != json + offset){
+			return result;
+		}
+	}
+	return fallback;
+}
+
+double json_array_double(char* json, uint64_t key, double fallback){
+	char* next_token = NULL;
+	double result;
+	size_t offset = json_array_offset(json, key);
+	if(offset){
+		result = strtod(json + offset, &next_token);
+		if(next_token != json + offset){
+			return result;
+		}
+	}
+	return fallback;
+}
+
 char* json_obj_str(char* json, char* key, size_t* length){
 	size_t offset = json_obj_offset(json, key), raw_length;
 	if(offset){
@@ -356,15 +547,37 @@ char* json_obj_str(char* json, char* key, size_t* length){
 }
 
 char* json_obj_strdup(char* json, char* key){
-	size_t offset = json_obj_offset(json, key), raw_length;
-	char* rv = NULL;
+	size_t len = 0;
+	char* value = json_obj_str(json, key, &len), *rv = NULL;
+	if(len){
+		rv = calloc(len + 1, sizeof(char));
+		if(rv){
+			memcpy(rv, value, len);
+		}
+	}
+	return rv;
+}
+
+char* json_array_str(char* json, uint64_t key, size_t* length){
+	size_t offset = json_array_offset(json, key), raw_length;
 	if(offset){
 		raw_length = json_validate_string(json + offset, strlen(json + offset));
-		rv = calloc(raw_length - 1, sizeof(char));
-		if(rv){
-			memcpy(rv, json + offset + 1, raw_length - 2);
+		if(length){
+			*length = raw_length - 2;
 		}
-		return rv;
+		return json + offset + 1;
 	}
 	return NULL;
 }
+
+char* json_array_strdup(char* json, uint64_t key){
+	size_t len = 0;
+	char* value = json_array_str(json, key, &len), *rv = NULL;
+	if(len){
+		rv = calloc(len + 1, sizeof(char));
+		if(rv){
+			memcpy(rv, value, len);
+		}
+	}
+	return rv;
+}
diff --git a/backends/libmmbackend.h b/backends/libmmbackend.h
index aa0ac0c..5749119 100644
--- a/backends/libmmbackend.h
+++ b/backends/libmmbackend.h
@@ -78,49 +78,50 @@ json_type json_identify(char* json, size_t length);
  * Returns the length of a detected JSON document, 0 otherwise (ie. parse failures)
  */
 size_t json_validate(char* json, size_t length);
-
 size_t json_validate_string(char* json, size_t length);
-
 size_t json_validate_array(char* json, size_t length);
-
 size_t json_validate_object(char* json, size_t length);
-
 size_t json_validate_value(char* json, size_t length);
 
 /* 
  * Calculate offset for value of `key`
- * Assumes a zero-terminated, validated JSON object as input
+ * Assumes a zero-terminated, validated JSON object / array as input
  * Returns offset on success, 0 on failure
  */
 size_t json_obj_offset(char* json, char* key);
+size_t json_array_offset(char* json, uint64_t key);
 
 /*
- * Check for for a key within a JSON object
- * Assumes a zero-terminated, validated JSON object as input
+ * Check for for a key within a JSON object / index within an array
+ * Assumes a zero-terminated, validated JSON object / array as input
  * Returns type of value
  */
 json_type json_obj(char* json, char* key);
-
-//json_type json_array(char* json, size_t index)
+json_type json_array(char* json, uint64_t key);
 
 /*
- * Fetch boolean value for an object key
- * Assumes a zero-terminated, validated JSON object as input
+ * Fetch boolean value for an object / array key
+ * Assumes a zero-terminated, validated JSON object / array as input
  */
 uint8_t json_obj_bool(char* json, char* key, uint8_t fallback);
+uint8_t json_array_bool(char* json, uint64_t key, uint8_t fallback);
 
 /*
- * Fetch integer/double value for an object key
- * Assumes a zero-terminated validated JSON object as input
+ * Fetch integer/double value for an object / array key
+ * Assumes a zero-terminated validated JSON object / array as input
  */
 int64_t json_obj_int(char* json, char* key, int64_t fallback);
 double json_obj_double(char* json, char* key, double fallback);
+int64_t json_array_int(char* json, uint64_t key, int64_t fallback);
+double json_array_double(char* json, uint64_t key, double fallback);
 
 /* 
- * Fetch a string value for an object key
- * Assumes a zero-terminated validated JSON object as input
- * json_obj_strdup returns a newly-allocated buffer containing
+ * Fetch a string value for an object / array key
+ * Assumes a zero-terminated validated JSON object / array as input
+ * json_*_strdup returns a newly-allocated buffer containing
  * only the requested value
  */
 char* json_obj_str(char* json, char* key, size_t* length);
 char* json_obj_strdup(char* json, char* key);
+char* json_array_str(char* json, uint64_t key, size_t* length);
+char* json_array_strdup(char* json, uint64_t key);
diff --git a/backends/maweb.c b/backends/maweb.c
index 79e223f..07595be 100644
--- a/backends/maweb.c
+++ b/backends/maweb.c
@@ -14,7 +14,13 @@
 #define WS_FLAG_FIN 0x80
 #define WS_FLAG_MASK 0x80
 
+//TODO test using different pages simultaneously
+//TODO test dot2 button virtual faders in fader view
+
 static uint64_t last_keepalive = 0;
+static uint64_t update_interval = 50;
+static uint64_t last_update = 0;
+static uint64_t updates_inflight = 0;
 
 static char* cmdline_keys[] = {
 	"SET",
@@ -94,7 +100,8 @@ int init(){
 		.handle = maweb_set,
 		.process = maweb_handle,
 		.start = maweb_start,
-		.shutdown = maweb_shutdown
+		.shutdown = maweb_shutdown,
+		.interval = maweb_interval
 	};
 
 	if(sizeof(maweb_channel_ident) != sizeof(uint64_t)){
@@ -110,8 +117,27 @@ int init(){
 	return 0;
 }
 
+static int channel_comparator(const void* raw_a, const void* raw_b){
+	maweb_channel_ident* a = (maweb_channel_ident*) raw_a;
+	maweb_channel_ident* b = (maweb_channel_ident*) raw_b;
+
+	if(a->fields.page != b->fields.page){
+		return a->fields.page - b->fields.page;
+	}
+	return a->fields.index - b->fields.index;
+}
+
+static uint32_t maweb_interval(){
+	return update_interval - (last_update % update_interval);
+}
+
 static int maweb_configure(char* option, char* value){
-	fprintf(stderr, "The maweb backend does not take any global configuration\n");
+	if(!strcmp(option, "interval")){
+		update_interval = strtoul(value, NULL, 10);
+		return 0;
+	}
+
+	fprintf(stderr, "Unknown maweb backend configuration option %s\n", option);
 	return 1;
 }
 
@@ -187,6 +213,7 @@ static instance* maweb_instance(){
 }
 
 static channel* maweb_channel(instance* inst, char* spec){
+	maweb_instance_data* data = (maweb_instance_data*) inst->impl;
 	maweb_channel_ident ident = {
 		.label = 0
 	};
@@ -214,7 +241,7 @@ static channel* maweb_channel(instance* inst, char* spec){
 			next_token += 5;
 		}
 		else if(!strncmp(next_token, "flash", 5)){
-			ident.fields.type = exec_flash;
+			ident.fields.type = exec_button;
 			next_token += 5;
 		}
 		else if(!strncmp(next_token, "button", 6)){
@@ -238,6 +265,24 @@ static channel* maweb_channel(instance* inst, char* spec){
 		//actually, those are zero-indexed...
 		ident.fields.index--;
 		ident.fields.page--;
+
+		//check if the channel is already known
+		for(n = 0; n < data->input_channels; n++){
+			if(data->input_channel[n].label == ident.label){
+				break;
+			}
+		}
+
+		if(n == data->input_channels){
+			data->input_channel = realloc(data->input_channel, (data->input_channels + 1) * sizeof(maweb_channel_ident));
+			if(!data->input_channel){
+				fprintf(stderr, "Failed to allocate memory\n");
+				return NULL;
+			}
+			data->input_channel[n].label = ident.label;
+			data->input_channels++;
+		}
+
 		return mm_channel(inst, ident.label, 1);
 	}
 	fprintf(stderr, "Failed to parse maweb channel spec %s\n", spec);
@@ -276,11 +321,216 @@ static int maweb_send_frame(instance* inst, maweb_operation op, uint8_t* payload
 	return 0;
 }
 
+static int maweb_process_playback(instance* inst, int64_t page, maweb_channel_type metatype, char* payload, size_t payload_length){
+	size_t exec_blocks = json_obj_offset(payload, (metatype == 2) ? "executorBlocks" : "bottomButtons"), offset, block = 0, control;
+	channel* chan = NULL;
+	channel_value evt;	
+	maweb_channel_ident ident = {
+		.fields.page = page,
+		.fields.index = json_obj_int(payload, "iExec", 191)
+	};
+
+	if(!exec_blocks){
+		if(metatype == 3){
+			//ignore unused buttons
+			return 0;
+		}
+		fprintf(stderr, "maweb missing exec block data on exec %d\n", ident.fields.index);
+		return 1;
+	}
+
+	if(metatype == 3){
+		exec_blocks += json_obj_offset(payload + exec_blocks, "items");
+	}
+
+	//TODO detect unused faders
+	//TODO state tracking for fader values / exec run state
+
+	//iterate over executor blocks
+	for(offset = json_array_offset(payload + exec_blocks, block); offset; offset = json_array_offset(payload + exec_blocks, block)){
+		control = exec_blocks + offset + json_obj_offset(payload + exec_blocks + offset, "fader");
+		ident.fields.type = exec_fader;
+		chan = mm_channel(inst, ident.label, 0);
+		if(chan){
+			evt.normalised = json_obj_double(payload + control, "v", 0.0);
+			mm_channel_event(chan, evt);
+		}
+
+		ident.fields.type = exec_button;
+		chan = mm_channel(inst, ident.label, 0);
+		if(chan){
+			evt.normalised = json_obj_int(payload, "isRun", 0);
+			mm_channel_event(chan, evt);
+		}
+
+		//printf("maweb page %ld exec %d value %f running %lu\n", page, ident.fields.index, json_obj_double(payload + control, "v", 0.0), json_obj_int(payload, "isRun", 0));
+		ident.fields.index++;
+		block++;
+	}
+
+	return 0;
+}
+
+static int maweb_process_playbacks(instance* inst, int64_t page, char* payload, size_t payload_length){
+	size_t base_offset = json_obj_offset(payload, "itemGroups"), group_offset, subgroup_offset, item_offset;
+	uint64_t group = 0, subgroup, item, metatype;
+
+	if(!page){
+		fprintf(stderr, "maweb received playbacks for invalid page\n");
+		return 0;
+	}
+
+	if(!base_offset){
+		fprintf(stderr, "maweb playback data missing item key\n");
+		return 0;
+	}
+
+	//iterate .itemGroups
+	for(group_offset = json_array_offset(payload + base_offset, group);
+			group_offset;
+			group_offset = json_array_offset(payload + base_offset, group)){
+		metatype = json_obj_int(payload + base_offset + group_offset, "itemsType", 0);
+		//iterate .itemGroups.items
+		//FIXME this is problematic if there is no "items" key
+		group_offset = group_offset + json_obj_offset(payload + base_offset + group_offset, "items");
+		if(group_offset){
+			subgroup = 0;
+			group_offset += base_offset;
+			for(subgroup_offset = json_array_offset(payload + group_offset, subgroup);
+					subgroup_offset;
+					subgroup_offset = json_array_offset(payload + group_offset, subgroup)){
+				//iterate .itemGroups.items[n]
+				item = 0;
+				subgroup_offset += group_offset;
+				for(item_offset = json_array_offset(payload + subgroup_offset, item);
+						item_offset;
+						item_offset = json_array_offset(payload + subgroup_offset, item)){
+					maweb_process_playback(inst, page, metatype,
+							payload + subgroup_offset + item_offset,
+							payload_length - subgroup_offset - item_offset);
+					item++;
+				}
+				subgroup++;
+			}
+		}
+		group++;
+	}
+	updates_inflight--;
+	fprintf(stderr, "maweb playback message processing done, %lu updates inflight\n", updates_inflight);
+	return 0;
+}
+
+static int maweb_request_playbacks(instance* inst){
+	maweb_instance_data* data = (maweb_instance_data*) inst->impl;
+	char xmit_buffer[MAWEB_XMIT_CHUNK];
+	int rv = 0;
+
+	char item_indices[1024] = "[0,100,200]", item_counts[1024] = "[21,21,21]", item_types[1024] = "[2,3,3]";
+	//char item_indices[1024] = "[300,400]", item_counts[1024] = "[18,18]", item_types[1024] = "[3,3]";
+	size_t page_index = 0, view = 2, channel = 0, offsets[3], channel_offset, channels;
+
+	if(updates_inflight){
+		fprintf(stderr, "maweb skipping update request, %lu updates still inflight\n", updates_inflight);
+		return 0;
+	}
+
+	for(channel = 0; channel < data->input_channels; channel++){
+		offsets[0] = offsets[1] = offsets[2] = 0;
+		page_index = data->input_channel[channel].fields.page;
+		if(data->peer_type == peer_dot2){
+			//TODO implement poll segmentation for dot
+			//"\"startIndex\":[0,100,200],"
+			//"\"itemsCount\":[21,21,21],"
+			//"\"itemsType\":[2,3,3],"
+			//"\"view\":2,"
+			//view = (data->input_channel[channel].fields.index >= 300) ? 3 : 2;
+			//observed
+			//"startIndex":[300,400,500,600,700,800],
+			//"itemsCount":[13,13,13,13,13,13]
+			//"itemsType":[3,3,3,3,3,3]
+			/*fprintf(stderr, "range start at %lu.%lu (%lu/%lu) end at %lu.%lu (%lu/%lu)\n", 
+					page_index, 
+					data->input_channel[channel].fields.index,
+					channel,
+					data->input_channels,
+					page_index,
+					data->input_channel[channel + channel_offset - 1].fields.index,
+					channel + channel_offset - 1,
+					data->input_channels
+					);*/
+			//only send one request currently
+			channel = data->input_channels;
+		}
+		else{
+			view = (data->input_channel[channel].fields.index >= 100) ? 3 : 2;
+			//for the ma, the view equals the exec type
+			snprintf(item_types, sizeof(item_types), "[%lu]", view);
+			//this channel must be included, so it must be in range for the first startindex
+			snprintf(item_indices, sizeof(item_indices), "[%d]", (data->input_channel[channel].fields.index / 5) * 5);
+
+			for(channel_offset = 1; channel + channel_offset < data->input_channels
+					&& data->input_channel[channel].fields.page == data->input_channel[channel + channel_offset].fields.page
+					&& data->input_channel[channel].fields.index / 100 == data->input_channel[channel + channel_offset].fields.index / 100; channel_offset++){
+			}
+
+			channels = data->input_channel[channel + channel_offset - 1].fields.index - (data->input_channel[channel].fields.index / 5) * 5;
+
+
+			snprintf(item_counts, sizeof(item_indices), "[%lu]", ((channels / 5) * 5 + 5));
+			channel += channel_offset - 1;
+		}
+		snprintf(xmit_buffer, sizeof(xmit_buffer),
+				"{"
+				"\"requestType\":\"playbacks\","
+				"\"startIndex\":%s,"
+				"\"itemsCount\":%s,"
+				"\"pageIndex\":%lu,"
+				"\"itemsType\":%s,"
+				"\"view\":%lu,"
+				"\"execButtonViewMode\":2,"	//extended
+				"\"buttonsViewMode\":0,"	//get vfader for button execs
+				"\"session\":%lu"
+				"}",
+				item_indices,
+				item_counts,
+				page_index,
+				item_types,
+				view,
+				data->session);
+		rv |= maweb_send_frame(inst, ws_text, (uint8_t*) xmit_buffer, strlen(xmit_buffer));
+		//fprintf(stderr, "req: %s\n", xmit_buffer);		
+		updates_inflight++;
+	}
+
+	return rv;
+}
+
 static int maweb_handle_message(instance* inst, char* payload, size_t payload_length){
 	char xmit_buffer[MAWEB_XMIT_CHUNK];
 	char* field;
 	maweb_instance_data* data = (maweb_instance_data*) inst->impl;
 
+	//query this early to save on unnecessary parser passes with stupid-huge data messages
+	if(json_obj(payload, "responseType") == JSON_STRING){
+		field = json_obj_str(payload, "responseType", NULL);
+		if(!strncmp(field, "login", 5)){
+			if(json_obj_bool(payload, "result", 0)){
+				fprintf(stderr, "maweb login successful\n");
+				data->login = 1;
+			}
+			else{
+				fprintf(stderr, "maweb login failed\n");
+				data->login = 0;
+			}
+		}
+		if(!strncmp(field, "playbacks", 9)){
+			if(maweb_process_playbacks(inst, json_obj_int(payload, "iPage", 0), payload, payload_length)){
+				fprintf(stderr, "maweb failed to handle/request input data\n");
+			}
+			return 0;
+		}
+	}
+
 	fprintf(stderr, "maweb message (%lu): %s\n", payload_length, payload);
 	if(json_obj(payload, "session") == JSON_NUMBER){
 		data->session = json_obj_int(payload, "session", data->session);
@@ -294,7 +544,6 @@ static int maweb_handle_message(instance* inst, char* payload, size_t payload_le
 				(data->peer_type == peer_dot2) ? "remote" : data->user, data->pass, data->session);
 		maweb_send_frame(inst, ws_text, (uint8_t*) xmit_buffer, strlen(xmit_buffer));
 	}
-
 	if(json_obj(payload, "status") && json_obj(payload, "appType")){
 		fprintf(stderr, "maweb connection established\n");
 		field = json_obj_str(payload, "appType", NULL);
@@ -307,31 +556,6 @@ static int maweb_handle_message(instance* inst, char* payload, size_t payload_le
 		maweb_send_frame(inst, ws_text, (uint8_t*) "{\"session\":0}", 13);
 	}
 
-	if(json_obj(payload, "responseType") == JSON_STRING){
-		field = json_obj_str(payload, "responseType", NULL);
-		if(!strncmp(field, "login", 5)){
-			if(json_obj_bool(payload, "result", 0)){
-				fprintf(stderr, "maweb login successful\n");
-				data->login = 1;
-			}
-			else{
-				fprintf(stderr, "maweb login failed\n");
-				data->login = 0;
-			}
-		}
-		else if(!strncmp(field, "getdata", 7)){
-			//FIXME stupid keepalive logic
-			snprintf(xmit_buffer, sizeof(xmit_buffer),
-					"{\"requestType\":\"getdata\","
-					"\"data\":\"set,clear,solo,high\","
-					"\"realtime\":true,"
-					"\"maxRequests\":10,"
-					",\"session\":%ld}",
-					data->session);
-			maweb_send_frame(inst, ws_text, (uint8_t*) xmit_buffer, strlen(xmit_buffer));
-		}
-	}
-
 	return 0;
 }
 
@@ -376,7 +600,7 @@ static ssize_t maweb_handle_lines(instance* inst, ssize_t bytes_read){
 	maweb_instance_data* data = (maweb_instance_data*) inst->impl;
 	size_t n, begin = 0;
 
-	for(n = 0; n < bytes_read - 2; n++){
+	for(n = 0; n < bytes_read - 1; n++){
 		if(!strncmp((char*) data->buffer + data->offset + n, "\r\n", 2)){
 			if(data->state == ws_new){
 				if(!strncmp((char*) data->buffer, "HTTP/1.1 101", 12)){
@@ -397,7 +621,7 @@ static ssize_t maweb_handle_lines(instance* inst, ssize_t bytes_read){
 		}
 	}
 
-	return begin;
+	return data->offset + begin;
 }
 
 static ssize_t maweb_handle_ws(instance* inst, ssize_t bytes_read){
@@ -507,6 +731,7 @@ static int maweb_handle_fd(instance* inst){
 
 		if(bytes_handled < 0){
 			bytes_handled = data->offset + bytes_read;
+			data->offset = 0;
 			//TODO close, reopen
 			fprintf(stderr, "maweb failed to handle incoming data\n");
 			return 1;
@@ -517,8 +742,6 @@ static int maweb_handle_fd(instance* inst){
 
 		memmove(data->buffer, data->buffer + bytes_handled, (data->offset + bytes_read) - bytes_handled);
 
-		//FIXME this might be somewhat borked
-		bytes_read -= data->offset;
 		bytes_handled -= data->offset;
 		bytes_read -= bytes_handled;
 		data->offset = 0;
@@ -551,30 +774,10 @@ static int maweb_set(instance* inst, size_t num, channel** c, channel_value* v){
 						"\"type\":1,"
 						"\"session\":%ld"
 						"}", ident.fields.index, ident.fields.page, v[n].normalised, data->session);
-				fprintf(stderr, "maweb out %s\n", xmit_buffer);
 				maweb_send_frame(inst, ws_text, (uint8_t*) xmit_buffer, strlen(xmit_buffer));
 				break;
 			case exec_upper:
 			case exec_lower:
-			case exec_flash:
-				snprintf(xmit_buffer, sizeof(xmit_buffer),
-						"{\"requestType\":\"playbacks_userInput\","
-						//"\"cmdline\":\"\","
-						"\"execIndex\":%d,"
-						"\"pageIndex\":%d,"
-						"\"buttonId\":%d,"
-						"\"pressed\":%s,"
-						"\"released\":%s,"
-						"\"type\":0,"
-						"\"session\":%ld"
-						"}", ident.fields.index, ident.fields.page,
-						(data->peer_type == peer_dot2) ? (ident.fields.type - 3) : (exec_flash - ident.fields.type),
-						(v[n].normalised > 0.9) ? "true" : "false",
-						(v[n].normalised > 0.9) ? "false" : "true",
-						data->session);
-				fprintf(stderr, "maweb out %s\n", xmit_buffer);
-				maweb_send_frame(inst, ws_text, (uint8_t*) xmit_buffer, strlen(xmit_buffer));
-				break;
 			case exec_button:
 				snprintf(xmit_buffer, sizeof(xmit_buffer),
 						"{\"requestType\":\"playbacks_userInput\","
@@ -586,13 +789,11 @@ static int maweb_set(instance* inst, size_t num, channel** c, channel_value* v){
 						"\"released\":%s,"
 						"\"type\":0,"
 						"\"session\":%ld"
-						"}", ident.fields.index,
-						ident.fields.page,
-						0,
+						"}", ident.fields.index, ident.fields.page,
+						(data->peer_type == peer_dot2 && ident.fields.type == exec_upper) ? 0 : (ident.fields.type - exec_button),
 						(v[n].normalised > 0.9) ? "true" : "false",
 						(v[n].normalised > 0.9) ? "false" : "true",
 						data->session);
-				fprintf(stderr, "maweb out %s\n", xmit_buffer);
 				maweb_send_frame(inst, ws_text, (uint8_t*) xmit_buffer, strlen(xmit_buffer));
 				break;
 			case cmdline_button:
@@ -602,7 +803,6 @@ static int maweb_set(instance* inst, size_t num, channel** c, channel_value* v){
 						"\"value\":%d"
 						"}", cmdline_keys[ident.fields.index],
 						(v[n].normalised > 0.9) ? 1 : 0);
-				fprintf(stderr, "maweb out %s\n", xmit_buffer);
 				maweb_send_frame(inst, ws_text, (uint8_t*) xmit_buffer, strlen(xmit_buffer));
 				break;
 			default:
@@ -638,6 +838,29 @@ static int maweb_keepalive(){
 	return 0;
 }
 
+static int maweb_poll(){
+	size_t n, u;
+	instance** inst = NULL;
+	maweb_instance_data* data = NULL;
+
+	//fetch all defined instances
+	if(mm_backend_instances(BACKEND_NAME, &n, &inst)){
+		fprintf(stderr, "Failed to fetch instance list\n");
+		return 1;
+	}
+
+	//send data polls for logged-in instances
+	for(u = 0; u < n; u++){
+		data = (maweb_instance_data*) inst[u]->impl;
+		if(data->login){
+			maweb_request_playbacks(inst[u]);
+		}
+	}
+
+	free(inst);
+	return 0;
+}
+
 static int maweb_handle(size_t num, managed_fd* fds){
 	size_t n = 0;
 	int rv = 0;
@@ -646,17 +869,24 @@ static int maweb_handle(size_t num, managed_fd* fds){
 		rv |= maweb_handle_fd((instance*) fds[n].impl);
 	}
 
+	//FIXME all keepalive processing allocates temporary buffers, this might an optimization target
 	if(last_keepalive && mm_timestamp() - last_keepalive >= MAWEB_CONNECTION_KEEPALIVE){
 		rv |= maweb_keepalive();
 		last_keepalive = mm_timestamp();
 	}
 
+	if(last_update && mm_timestamp() - last_update >= update_interval){
+		rv |= maweb_poll();
+		last_update = mm_timestamp();
+	}
+
 	return rv;
 }
 
 static int maweb_start(){
 	size_t n, u;
 	instance** inst = NULL;
+	maweb_instance_data* data = NULL;
 
 	//fetch all defined instances
 	if(mm_backend_instances(BACKEND_NAME, &n, &inst)){
@@ -665,6 +895,10 @@ static int maweb_start(){
 	}
 
 	for(u = 0; u < n; u++){
+		//sort channels
+		data = (maweb_instance_data*) inst[u]->impl;
+		qsort(data->input_channel, data->input_channels, sizeof(maweb_channel_ident), channel_comparator);
+
 		if(maweb_connect(inst[u])){
 			fprintf(stderr, "Failed to open connection to MA Web Remote for instance %s\n", inst[u]->name);
 			return 1;
@@ -678,8 +912,8 @@ static int maweb_start(){
 
 	fprintf(stderr, "maweb backend registering %lu descriptors to core\n", n);
 
-	//initialize keepalive timeout
-	last_keepalive = mm_timestamp();
+	//initialize timeouts
+	last_keepalive = last_update = mm_timestamp();
 	return 0;
 }
 
@@ -713,6 +947,10 @@ static int maweb_shutdown(){
 
 		data->offset = data->allocated = 0;
 		data->state = ws_new;
+
+		free(data->input_channel);
+		data->input_channel = NULL;
+		data->input_channels = 0;
 	}
 
 	free(inst);
diff --git a/backends/maweb.h b/backends/maweb.h
index 5f59cc1..a868426 100644
--- a/backends/maweb.h
+++ b/backends/maweb.h
@@ -9,22 +9,22 @@ static int maweb_set(instance* inst, size_t num, channel** c, channel_value* v);
 static int maweb_handle(size_t num, managed_fd* fds);
 static int maweb_start();
 static int maweb_shutdown();
+static uint32_t maweb_interval();
 
 //Default login password: MD5("midimonster")
 #define MAWEB_DEFAULT_PASSWORD "2807623134739142b119aff358f8a219"
 #define MAWEB_DEFAULT_PORT "80"
 #define MAWEB_RECV_CHUNK 1024
-#define MAWEB_XMIT_CHUNK 2048
+#define MAWEB_XMIT_CHUNK 4096
 #define MAWEB_FRAME_HEADER_LENGTH 16
 #define MAWEB_CONNECTION_KEEPALIVE 10000
 
 typedef enum /*_maweb_channel_type*/ {
 	type_unset = 0,
 	exec_fader = 1,
-	exec_button = 2,
-	exec_upper = 3,
-	exec_lower = 4,
-	exec_flash = 5,
+	exec_button = 2, //gma: 0 dot: 0
+	exec_lower = 3, //gma: 1 dot: 1
+	exec_upper = 4, //gma: 2 dot: 0
 	cmdline_button
 } maweb_channel_type;
 
@@ -69,6 +69,10 @@ typedef struct /*_maweb_instance_data*/ {
 	int64_t session;
 	maweb_peer_type peer_type;
 
+	//need to keep an internal registry to optimize data polls
+	size_t input_channels;
+	maweb_channel_ident* input_channel;
+
 	int fd;
 	maweb_state state;
 	size_t offset;
diff --git a/backends/maweb.md b/backends/maweb.md
index d713d82..fe430db 100644
--- a/backends/maweb.md
+++ b/backends/maweb.md
@@ -2,8 +2,7 @@
 
 This backend connects directly with the integrated *MA Web Remote* of MA Lighting consoles and OnPC
 instances (GrandMA2 / GrandMA2 OnPC / GrandMA Dot2 / GrandMA Dot2 OnPC).
-It grants read-write access to the console's playback faders and buttons as well as write access to
-the command line buttons.
+It grants read-write access to the console's playback controls as well as write access to the command line.
 
 #### Setting up the console
 
@@ -16,7 +15,9 @@ Web Remote. Set a web remote password using the option below the activation sett
 
 #### Global configuration
 
-The `maweb` backend does not take any global configuration.
+| Option	| Example value		| Default value		| Description							|
+|---------------|-----------------------|-----------------------|---------------------------------------------------------------|
+| `interval`	| `100`			| `50`			| Query interval for input data polling (in msec)		|
 
 #### Instance configuration
 
@@ -28,39 +29,44 @@ The `maweb` backend does not take any global configuration.
 
 #### Channel specification
 
-Currently, three types of channels can be assigned
+Currently, three types of MA controls can be assigned, with each having some subcontrols
+
+* Fader executor
+* Button executor
+* Command line buttons
 
 ##### Executors
 
 * For the GrandMA2, executors are arranged in pages, with each page having 90 fader executors (numbered 1 through 90)
 	and 90 button executors (numbered 101 through 190).
-	* A fader executor consists of a `fader`, two buttons above it (`upper`, `lower`) and one `flash` button below it.
-	* A button executor consists of a `button` control.
+	* A fader executor consists of a `fader`, two buttons above it (`upper`, `lower`) and one `button` below it.
+	* A button executor consists of a `button` control and a virtual `fader` (visible on the console in the "Action Buttons" view).
 * For the dot2, executors are also arranged in pages, but the controls are non-obviously numbered.
 	* For the faders, they are numerically right-to-left from the Core Fader section (Faders 6 to 1) over the F-Wing 1 (Faders 13 to 6) to
 	F-Wing 2 (Faders 21 to 14).
 	* Above the fader sections are two rows of 21 `button` executors, numbered 122 through 101 (upper row) and 222 through 201 (lower row),
 		in the same order as the faders are.
 	* Fader executors have two buttons below them (`upper` and `lower`).
-	* The button executor section consists of six rows of 18 buttons, divided into two button wings. Buttons on the wings
+	* The button executor section consists of six rows of 16 buttons, divided into two button wings. Buttons on the wings
 		are once again numbered right-to-left.
-		* B-Wing 1 has `button` executors 308 to 301 (top row), 408 to 401 (second row), and so on until 808 through 801 (bottom row)
+		* B-Wing 1 has `button` controls 308 to 301 (top row), 408 to 401 (second row), and so on until 808 through 801 (bottom row)
 		* B-Wing 2 has 316 to 309 (top row) through 816 to 809 (bottom row)
 
 When creating a new show, only the first page is created and active. Additional pages have to be created explicitly within
-the console before being usable.
+the console before being usable. `fader` controls, when mapped as outputs from the MA, output their value, `button` controls
+output 1 when the corresponding executor is running, 0 otherwise.
 
 These controls can be addressed like
 
 ```
 mw1.page1.fader5 > mw1.page1.upper5
-mw1.page3.lower3 > mw1.page2.flash2
+mw1.page3.lower3 > mw1.page2.button2
 ```
 
 A button executor can likewise be mapped using the syntax
 
 ```
-mw1.page2.button103 > mw1.page3.button101
+mw1.page2.button103 > mw1.page3.fader101
 mw1.page2.button803 > mw1.page3.button516
 ```
 
@@ -99,10 +105,12 @@ Since this may be a problem on some platforms, the backend can be built with thi
 to set arbitrary passwords. The backend will always try to log in with the default password `midimonster` in this case. The user name is still
 configurable.
 
+Data input from the console is done by actively querying the state of all mapped controls, which is resource-intensive if done
+at low latency. A lower input interval value will produce data with lower latency, at the cost of network & CPU usage.
+Higher values will make the input "step" more, but will not consume as many CPU cycles and network bandwidth.
+
 This backend is currently in active development. It therefore has some limitations:
 
 * It outputs a lot of debug information
-* It currently is write-only, channel events are only sent to the MA, not generated by it
-* Fader executors (and their buttons) seem to work, I haven't tested button executors yet.
 * Command line events are sent, but I'm not sure they're being handled yet
-* I have so far only tested it with GradMA2 OnPC
+* For the dot2, currently only the Core & F-Wings are supported for input from the console, not the B-Wings
diff --git a/backends/midi.h b/backends/midi.h
index 5ec17ea..6c3fcf9 100644
--- a/backends/midi.h
+++ b/backends/midi.h
@@ -24,4 +24,5 @@ typedef union {
 		uint8_t control;
 	} fields;
 	uint64_t label;
-} midi_channel_ident;
\ No newline at end of file
+} midi_channel_ident;
+
diff --git a/backends/osc.h b/backends/osc.h
index ab19463..dd5afb0 100644
--- a/backends/osc.h
+++ b/backends/osc.h
@@ -73,4 +73,5 @@ typedef union {
 		uint32_t parameter;
 	} fields;
 	uint64_t label;
-} osc_channel_ident;
\ No newline at end of file
+} osc_channel_ident;
+
-- 
cgit v1.2.3