move request param generation to own file, retry failed queries

1 files changed, 9 insertions, 14 deletions

diff --git a/index.js b/index.js
index 12decf2..cefd7e9 100644
--- a/index.js
+++ b/index.js
@@ -1,25 +1,20 @@
 #!/usr/bin/node
 
 const R  = require('ramda'),
-  etl    = require('./etl'),
+  Future = require('fluture'),
+  moment = require('moment'),
+  source = require('./source'),
+  params = require('./params'),
   reduce = require('./reduce'),
   config = require('./config');
 
 function main() {
-  const loadFPayloads     = etl.loadFPayloads(config.etl);
-  const aggregatePayloads = reduce.aggregatePayloads(config.reduce);
-  const deriveStatistics  = reduce.deriveStatistics(config.reduce);
+  const paramsForHourSample = params.paramsForHourSample(config.api);
+  const loadFApi            = source.loadFApi(config.etl);
+  const aggregatePayloads   = reduce.aggregatePayloads(config.reduce);
+  const deriveStatistics    = reduce.deriveStatistics(config.reduce);
 
-  let pages   = [0, 5, 10, 15, 20, 25];
-  let regions = ['sg', 'eu', 'na'];
-
-  const params   = R.xprod(regions, pages);
-  const requests = R.map(
-    R.apply((region, page) => loadFPayloads(
-      `/shards/${region}/matches`,
-      { 'sort': '-createdAt', 'filter[gameMode]': 'blitz_pvp_ranked', 'page[limit]': '5', 'page[offset]': page },
-    )),
-    params);
+  const requests = R.map(R.apply(loadFApi), paramsForHourSample(moment().subtract(1, 'days')));
 
   /* main */
   Future.parallel(1, requests)