remove rate limit, do not crash on http 4xx

3 files changed, 21 insertions, 11 deletions

diff --git a/config.js b/config.js
index d1601ec..0a7e048 100644
--- a/config.js
+++ b/config.js
@@ -18,29 +18,38 @@ module.exports = {
     timeout: 3,
     /* available: eu, na, sg, cn, sa, ea */
     regions: ['sg', 'eu', 'na'],
-    /* available: casual, ranked, blitz_pvp_ranked, casual_aral */
-    modes: ['blitz_pvp_ranked'],
+    /* available: casual, 5v5_pvp_casual, ranked, 5v5_pvp_ranked, blitz_pvp_ranked, casual_aral */
+    modes: ['blitz_pvp_ranked', 'casual_aral'],
     /* available: 2, 3, 4, 5 */
     matchesPerRequest: 5,
     /* How many equidistant requests to create
-     * matchesPerRequest * requestsPerInterval = sample size for that interval */
-    requestsPerInterval: 5,
+     * matchesPerRequest * requestsPerInterval = sample size for that interval
+     * currently requestsPerInterval <= API key rate limit but should work if it isn't significantly higher
+     * rough estimate of upper limits (na/eu/sg, depends on region and time of day):
+     * blitz: 240
+     * battle royale: 30
+     * 3v3 casual: 30
+     * 3v3 ranked: 100
+     * 5v5 casual: 30
+     * 5v5 ranked: 40
+     * if filtering for multiple game modes, use the sum of the upper limits */
+    requestsPerInterval: 50,
     /* the size of a batch */
     interval: 1,
     intervalUnit: 'hours',
     /* How many batches to request per minute
      * requestsPerInterval * intervalsPer Minute <= API key rate limit */
-    intervalsPerMinute: 10,
+    intervalsPerMinute: 1,
   },
   etl: {
     /* attributes to be extracted and renamed from main API data */
     participant: [ ['Winner', ['attributes', 'stats', 'winner']] ],
     roster: [ ],
-    match: [ ],
+    match: [ ['Mode', ['attributes', 'gameMode']] ],
   },
   reduce: {
     /* criteria or 'filters' */
-    group: ['Actor', 'Talent'],
+    group: ['Actor', 'Talent', 'Mode'],
     /* stats */
     sum: ['Level', 'Winner'],
   },
diff --git a/index.js b/index.js
index 70b9f47..5da0f82 100644
--- a/index.js
+++ b/index.js
@@ -15,7 +15,7 @@ function main() {
   const firstOfMay = moment('2018-05-01');
 
   const later = R.curry((base, hs) => base.clone().add(hs, 'hours'));
-  const hours = R.range(0, 24 * 4);
+  const hours = R.range(0, 24 * 5);
   const laterMoments = R.map(later(firstOfMay), hours);
 
   const futures = R.map(loadFTimestamped, laterMoments);
diff --git a/source.js b/source.js
index 2d02939..0f4fd28 100644
--- a/source.js
+++ b/source.js
@@ -19,9 +19,10 @@ function doFetchProcessStoreHour(config) {
 
   return (timestamp) => Future.parallel(1, requestsForHour(timestamp))
     .map(R.compose(aggregatePayloads, R.unnest))
-    .chain(/*R.map(R.chain(Future.after(intervalSleep),*/ saveFPayloads(timestamp))/*))*/
-    .chain(Future.after(intervalSleep));
-  // TODO sleeps after request; throttling rpm should happen
+    .chainRej((err) => { console.error(err); return Future.of({}); }) // fail silently
+    .chain(saveFPayloads(timestamp));
+    //.chain(Future.after(intervalSleep));
+  // TODO should sleep after request; throttling rpm should happen
   // at a higher level though. CPU / IO / AWS requests stall
   // as well
 }