send processor matches and players seperately; refactor

3 files changed, 95 insertions, 12 deletions

diff --git a/jsonapi.js b/jsonapi.js
new file mode 100644
index 0000000..fa3b6ef
--- /dev/null
+++ b/jsonapi.js
@@ -0,0 +1,64 @@
+#!/usr/bin/node
+/* https://github.com/alex94puchades/superagent-jsonapify/blob/master/common.js */
+
+"use strict";
+
+var _isUndefined = require('lodash/isUndefined');
+var _isArray = require('lodash/isArray');
+var _map = require('lodash/map');
+var _partial = require('lodash/partial');
+var _each = require('lodash/each');
+var _camelCase = require('lodash/camelCase');
+var _memoize = require('lodash/memoize');
+var _chain = require('lodash/chain');
+var _find = require('lodash/find');
+var _clone = require('lodash/clone');
+var _compact = require('lodash/compact');
+
+exports.parse = (obj) => {
+    let response = obj,
+        data = obj.data;
+    if (!data) {
+        return data;
+    } else if (_isArray(data)) {
+        return _map(data, _partial(parseResourceDataObject, response));
+    } else {
+        return parseResourceDataObject(response, data);
+    }
+}
+
+function parseResourceDataObject(response, data) {
+    var result = _clone(data);
+    _each(data.attributes, function(value, name) {
+        Object.defineProperty(result, _camelCase(name), { value: value, enumerable: true });
+    });
+    _each(data.relationships, function(value, name) {
+        if (_isArray(value.data)) {
+            Object.defineProperty(result, _camelCase(name), {
+                get: _memoize(function() {
+                    const related = _map(value.data, function(related) {
+                        var resdata = _find(response.included, function(included) {
+                            return included.id === related.id && included.type === related.type;
+                        });
+                        if(resdata)
+                            return parseResourceDataObject(response, resdata);
+                    });
+                    return _compact(related);
+                }),
+                enumerable: true
+            });
+        } else if (value.data) {
+            Object.defineProperty(result, _camelCase(name), {
+                get: _memoize(function() {
+                    var resdata = _find(response.included, function(included) {
+                        return included.id === value.data.id && included.type === value.data.type;
+                    });
+                    return resdata ? parseResourceDataObject(response, resdata)
+                                   : null;
+                }),
+                enumerable: true
+            });
+        }
+    });
+    return result;
+}
diff --git a/package.json b/package.json
index 36d37f1..2c49a8f 100644
--- a/package.json
+++ b/package.json
@@ -5,11 +5,11 @@
   "main": "worker.js",
   "dependencies": {
     "amqplib": "^0.5.1",
-    "json_": "^1.0.0",
     "request": "^2.81.0",
     "request-promise": "^4.2.0",
-    "superagent-jsonapify": "^1.4.4",
-    "sleep-promise": "^2.0.0"
+    "sleep-promise": "^2.0.0",
+    "snakecase-keys": "^1.1.0",
+    "superagent-jsonapify": "^1.4.4"
   },
   "devDependencies": {},
   "scripts": {
diff --git a/worker.js b/worker.js
index 9ad227f..3618c01 100644
--- a/worker.js
+++ b/worker.js
@@ -5,9 +5,8 @@
 var amqp = require("amqplib"),
     request = require("request-promise"),
     sleep = require("sleep-promise"),
-    Bluebird = require("bluebird"),
-    jsonapi = Bluebird.promisifyAll(require("superagent-jsonapify/common")),
-    JSON_ = require("json_");
+    snakeCaseKeys = require("snakecase-keys"),
+    jsonapi = require("./jsonapi");
 
 var MADGLORY_TOKEN = process.env.MADGLORY_TOKEN,
     RABBITMQ_URI = process.env.RABBITMQ_URI || "amqp://localhost";
@@ -40,12 +39,32 @@ if (MADGLORY_TOKEN == undefined) throw "Need an API token";
             opts.qs = payload.params;
             try {
                 console.log("API request: %j", opts.qs);
-                let res = await request(opts);
-                // JSON_: stringify snakeCase -> camel_case
-                let matches = await jsonapi.parse(JSON_.stringify(res));  // TODO parse, stringify, parse, stringify… -> inefficient
-                matches.data.forEach(async (match) => {
-                    await ch.sendToQueue("process", new Buffer(JSON.stringify(match)), { persistent: true });
-                });
+                let data = await request(opts),
+                    matches = jsonapi.parse(data);
+                // send match structure
+                await Promise.all(matches
+                    .map(async (match) => await ch.sendToQueue("process",
+                        new Buffer(JSON.stringify(snakeCaseKeys(match))), {
+                            persistent: true,
+                            type: "match",
+                            headers: {
+                                shard: payload.region
+                            }
+                        })
+                ));
+                // send players and teams, they are duplicated in the above structure
+                // and will be inserted seperately
+                await Promise.all(data.included
+                    .filter((o) => o.type == "player" || o.type == "team")
+                    .map(async (o) => await ch.sendToQueue("process",
+                        new Buffer(JSON.stringify(snakeCaseKeys(o))), {
+                            persistent: true,
+                            type: o.type,
+                            headers: {
+                                shard: payload.region
+                            }
+                        })
+                ));
             } catch (err) {
                 if (err.statusCode == 429) {
                     await sleep(1000);