implement sample downloads

2 files changed, 55 insertions, 15 deletions

diff --git a/package.json b/package.json
index 2c49a8f..c989c6b 100644
--- a/package.json
+++ b/package.json
@@ -4,6 +4,7 @@
   "description": "",
   "main": "worker.js",
   "dependencies": {
+    "adm-zip": "^0.4.7",
     "amqplib": "^0.5.1",
     "request": "^2.81.0",
     "request-promise": "^4.2.0",
diff --git a/worker.js b/worker.js
index a43879a..6d6e206 100644
--- a/worker.js
+++ b/worker.js
@@ -5,7 +5,8 @@
 var amqp = require("amqplib"),
     request = require("request-promise"),
     sleep = require("sleep-promise"),
-    jsonapi = require("./jsonapi");
+    jsonapi = require("./jsonapi"),
+    AdmZip = require("adm-zip");
 
 var MADGLORY_TOKEN = process.env.MADGLORY_TOKEN,
     RABBITMQ_URI = process.env.RABBITMQ_URI || "amqp://localhost";
@@ -29,14 +30,27 @@ if (MADGLORY_TOKEN == undefined) throw "Need an API token";
 
     await ch.prefetch(1);
     ch.consume("grab", async (msg) => {
-        let exhausted = false,
-            payload = JSON.parse(msg.content);
+        let payload = JSON.parse(msg.content.toString());
+        if (msg.properties.type == "matches")
+            await getAPI(payload, "matches");
+        if (msg.properties.type == "samples")
+            await getAPI(payload, "samples");
+        if (msg.properties.type == "sample")
+            await getSample(payload);
+
+        console.log("done");
+        ch.ack(msg);
+    }, { noAck: false });
+
+    // loop over API data pages
+    async function getAPI(payload, where) {
+        let exhausted = false;
         payload.params["page[limit]"] = payload.params["page[limit]"] || 50;
         payload.params["page[offset]"] = payload.params["page[offset]"] || 0;
 
         while (!exhausted) {
             let opts = {
-                uri: "https://api.dc01.gamelockerapp.com/shards/" + payload.region + "/matches",
+                uri: "https://api.dc01.gamelockerapp.com/shards/" + payload.region + "/" + where,
                 headers: {
                     "X-Title-ID": "semc-vainglory",
                     "Authorization": MADGLORY_TOKEN
@@ -48,14 +62,24 @@ if (MADGLORY_TOKEN == undefined) throw "Need an API token";
             try {
                 console.log("API request: %j", opts.qs);
                 let data = await request(opts),
-                    matches = jsonapi.parse(data);
-                // send match structure
-                await Promise.all(matches
-                    .map((match) => ch.sendToQueue("process",
-                        new Buffer(JSON.stringify(match)),
-                        { persistent: true, type: "match" })
-                ));
-                if (matches.length < 50) exhausted = true;
+                    datas = jsonapi.parse(data);
+                if (where == "matches") {
+                    // send match structure
+                    await Promise.all(datas
+                        .map((match) => ch.sendToQueue("process",
+                            new Buffer(JSON.stringify(match)),
+                            { persistent: true, type: "match" })
+                    ));
+                }
+                if (where == "samples") {
+                    // send to self
+                    await Promise.all(datas
+                        .map((sample) => ch.sendToQueue("grab",
+                            new Buffer(JSON.stringify(sample.attributes.URL)),
+                            { persistent: true, type: "sample" })
+                    ));
+                }
+                if (datas.length < 50) exhausted = true;
             } catch (err) {
                 if (err.statusCode == 429) {
                     await sleep(1000);
@@ -73,8 +97,23 @@ if (MADGLORY_TOKEN == undefined) throw "Need an API token";
             if (!failed)
                 payload.params["page[offset]"] += payload.params["page[limit]"]
         }
+    }
 
-        console.log("done");
-        ch.ack(msg);
-    }, { noAck: false });
+    // download a sample ZIP and send to processor
+    async function getSample(url) {
+        console.log("downloading sample", url);
+        let zipdata = await request({
+            uri: url,
+            encoding: null
+        }),
+            zip = new AdmZip(zipdata);
+        await Promise.all(zip.getEntries().map(async (entry) => {
+            if (entry.isDirectory) return;
+            let match = jsonapi.parse(JSON.parse(entry.getData().toString("utf8")));
+            await ch.sendToQueue("process",
+                new Buffer(JSON.stringify(match)),
+                { persistent: true, type: "match" })
+        }));
+        console.log("sample processed", url);
+    }
 })();