move api logic to orm

2 files changed, 90 insertions, 1 deletions

diff --git a/api.js b/api.js
new file mode 100644
index 0000000..2bd63a3
--- /dev/null
+++ b/api.js
@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+/* jshint esnext: true */
+"use strict";
+const Promise = require("bluebird"),
+    request = require("request-promise"),
+    sleep = require("sleep-promise"),
+    jsonapi = require("./jsonapi");
+
+const MADGLORY_TOKEN = process.env.MADGLORY_TOKEN;
+if (MADGLORY_TOKEN == undefined) throw "Need an API token";
+
+const api = module.exports;
+
+// send a request to url with options, log data about it
+// return [jsonapi parsed data, jsonapi links]
+module.exports.request = async (url, options, logger) => {
+    let response;
+    while (true) {
+        try {
+            const opts = {
+                uri: url,
+                headers: {
+                    "X-Title-Id": "semc-vainglory",
+                    "Authorization": MADGLORY_TOKEN
+                },
+                qs: options,
+                json: true,
+                gzip: true,
+                time: true,
+                forever: true,
+                strictSSL: true,
+                resolveWithFullResponse: true
+            };
+            logger.info("API request", {
+                uri: opts.uri,
+                qs: opts.qs
+            });
+            response = await request(opts);
+            return [jsonapi.parse(response.body), response.body.links];
+        } catch (err) {
+            response = err.response;
+            if (err.statusCode == 429) {
+                logger.warn("rate limited, sleeping");
+                await sleep(100);  // no return, no break => retry
+                continue;
+            } else if (err.statusCode != 404) {
+                logger.error("API error, retrying", err);
+                await sleep(100);
+                continue;
+            }
+            logger.warn("not found", {
+                uri: err.options.uri,
+                qs: err.options.qs,
+                error: err.response.body
+            });
+            return [undefined, undefined];
+        } finally {
+            logger.info("API response", {
+                status: response.statusCode,
+                connection_start: response.timings.connect,
+                connection_end: response.timings.end,
+                ratelimit_remaining: parseInt(response.headers["x-ratelimit-remaining"])
+            });
+        }
+    }
+}
+
+// loop over all pages, yield jsonapi parsed data results
+// TODO make it an async generator? coroutine?
+module.exports.requests = async (endpoint, region, options, logger) => {
+    let url = api.url(endpoint, region),
+        result = [];
+    do {
+        const [data, links] = await api.request(url, options, logger);
+        if (data == undefined) break;  // 404
+        if (links == undefined) break;  // 404
+        url = links.next;
+        result.push(...data);
+        if (links.next == undefined) break;  // last page
+    } while (true);
+    return result;
+}
+
+module.exports.url = (endpoint, region) =>
+    "https://api.dc01.gamelockerapp.com/shards/" + region + "/" + endpoint;
diff --git a/package.json b/package.json
index 6d642e3..b3625d9 100644
--- a/package.json
+++ b/package.json
@@ -9,8 +9,12 @@
   "author": "schneefux",
   "license": "UNLICENSED",
   "dependencies": {
+    "bluebird": "^3.5.0",
     "mysql": "^2.13.0",
+    "request": "^2.81.0",
+    "request-promise": "^4.2.1",
     "sequelize-auto": "^0.4.25",
-    "sequelize-json": "^2.1.2"
+    "sequelize-json": "^2.1.2",
+    "sleep-promise": "^2.0.0"
   }
 }