move request param generation to own file, retry failed queries

5 files changed, 75 insertions, 26 deletions

diff --git a/api.js b/api.js
index 3e65d66..30b4937 100644
--- a/api.js
+++ b/api.js
@@ -1,17 +1,17 @@
 #!/usr/bin/node
 
-const requestP = require('request-promise')
-  Future = require('fluture');
+const requestP = require('request-promise'),
+  config = require('./config'),
+  Future = require('fluture'),
+  FutureRetry = require('fluture-retry');
 
-const API_KEY = process.env.API_KEY || require('./secret').apikey,
-  API_BASE = 'https://api.dc01.gamelockerapp.com',
-  API_TIMEOUT = 1; // seconds
+const API_KEY = process.env.API_KEY || require('./secret').apikey;
 
 const requestF = Future.encaseP(requestP);
 
 function apiRequest(path, query) {
-  return requestF({
-    uri: API_BASE + path,
+  return FutureRetry.retryLinearly(requestF({
+    uri: config.api.baseUrl + path,
     headers: {
       'X-Title-Id': 'semc-vainglory',
       'Authorization': API_KEY
@@ -20,20 +20,20 @@ function apiRequest(path, query) {
     json: true,
     gzip: true,
     forever: true,
-    timeout: API_TIMEOUT*1000,
+    timeout: config.api.timeout * 1000,
     strictSSL: true
-  });
+  }));
 }
 
 function awsRequest(url) {
-  return requestF({
+  return FutureRetry.retryLinearly(requestF({
     uri: url,
     json: true,
     gzip: true,
     forever: true,
-    timeout: API_TIMEOUT*1000,
+    timeout: config.api.timeout * 1000,
     strictSSL: true
-  });
+  }));
 }
 
 module.exports = {
diff --git a/config.js b/config.js
index 8710e5d..5b35485 100644
--- a/config.js
+++ b/config.js
@@ -1,6 +1,13 @@
 #!/usr/bin/node
 
 module.exports = {
+  api: {
+    baseUrl: 'https://api.dc01.gamelockerapp.com',
+    timeout: 5,
+    regions: ['sg', 'eu', 'na'],
+    rpm: 50,
+    modes: ['blitz_pvp_ranked'],
+  },
   etl: {
     participant: [ ['Winner', ['attributes', 'stats', 'winner']] ],
     roster: [ ],
diff --git a/index.js b/index.js
index 12decf2..cefd7e9 100644
--- a/index.js
+++ b/index.js
@@ -1,25 +1,20 @@
 #!/usr/bin/node
 
 const R  = require('ramda'),
-  etl    = require('./etl'),
+  Future = require('fluture'),
+  moment = require('moment'),
+  source = require('./source'),
+  params = require('./params'),
   reduce = require('./reduce'),
   config = require('./config');
 
 function main() {
-  const loadFPayloads     = etl.loadFPayloads(config.etl);
-  const aggregatePayloads = reduce.aggregatePayloads(config.reduce);
-  const deriveStatistics  = reduce.deriveStatistics(config.reduce);
+  const paramsForHourSample = params.paramsForHourSample(config.api);
+  const loadFApi            = source.loadFApi(config.etl);
+  const aggregatePayloads   = reduce.aggregatePayloads(config.reduce);
+  const deriveStatistics    = reduce.deriveStatistics(config.reduce);
 
-  let pages   = [0, 5, 10, 15, 20, 25];
-  let regions = ['sg', 'eu', 'na'];
-
-  const params   = R.xprod(regions, pages);
-  const requests = R.map(
-    R.apply((region, page) => loadFPayloads(
-      `/shards/${region}/matches`,
-      { 'sort': '-createdAt', 'filter[gameMode]': 'blitz_pvp_ranked', 'page[limit]': '5', 'page[offset]': page },
-    )),
-    params);
+  const requests = R.map(R.apply(loadFApi), paramsForHourSample(moment().subtract(1, 'days')));
 
   /* main */
   Future.parallel(1, requests)
diff --git a/package-lock.json b/package-lock.json
index b9f2aa0..87de321 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1387,6 +1387,11 @@
         "sanctuary-type-identifiers": "2.0.1"
       }
     },
+    "fluture-retry": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fluture-retry/-/fluture-retry-1.0.0.tgz",
+      "integrity": "sha512-a/PYnJ5rmxgNl94tem0PylMFSA8HpO53to3gbOcgmaRmwrl95x1X3mR6rZHichnxIQ3wiy3jLyIZXQMChM6VVA=="
+    },
     "follow-redirects": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.4.1.tgz",
@@ -3264,6 +3269,11 @@
         "minimist": "0.0.8"
       }
     },
+    "moment": {
+      "version": "2.22.1",
+      "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.1.tgz",
+      "integrity": "sha512-shJkRTSebXvsVqk56I+lkb2latjBs8I+pc2TzWc545y2iFnSjm7Wg0QMh+ZWcdSLQyGEau5jI8ocnmkyTgr9YQ=="
+    },
     "ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
diff --git a/params.js b/params.js
new file mode 100644
index 0000000..b3dcb98
--- /dev/null
+++ b/params.js
@@ -0,0 +1,37 @@
+#!/usr/bin/node
+
+const R = require('ramda');
+
+function paramsForHourSample(config) {
+  const requestArgsForRegionStartEnd = R.curry((region, start, end) => [
+    `/shards/${region}/matches`, {
+      'sort': '-createdAt',
+      'filter[gameMode]': R.join(',', config.modes),
+      'filter[createdAt-start]': start.toISOString(),
+      'filter[createdAt-end]': end.toISOString(),
+      'page[limit]': '5',
+      'page[offset]': '0',
+    }
+  ]);
+
+  const requestsPerMinutePerRegion = Math.floor(config.rpm / config.regions.length);
+  const splitDurationMinutes = Math.floor(60 / requestsPerMinutePerRegion);
+  const splitMoments = (hour) => R.map(
+    (offsetIndex) => [
+      hour.clone().startOf('hour').minutes(offsetIndex * splitDurationMinutes),
+      hour.clone().startOf('hour').minutes((offsetIndex + 1) * splitDurationMinutes),
+    ],
+    R.range(0, requestsPerMinutePerRegion),
+  );
+
+  return R.pipe(
+    splitMoments,
+    R.xprod(config.regions),
+    R.map(R.unnest),
+    R.map(R.apply(requestArgsForRegionStartEnd)),
+  );
+}
+
+module.exports = {
+  paramsForHourSample,
+};