From 63da93e9f443a799208c8e706616a828fa528a3e Mon Sep 17 00:00:00 2001 From: schneefux Date: Sat, 5 May 2018 13:52:50 +0200 Subject: move request param generation to own file, retry failed queries --- index.js | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) (limited to 'index.js') diff --git a/index.js b/index.js index 12decf2..cefd7e9 100644 --- a/index.js +++ b/index.js @@ -1,25 +1,20 @@ #!/usr/bin/node const R = require('ramda'), - etl = require('./etl'), + Future = require('fluture'), + moment = require('moment'), + source = require('./source'), + params = require('./params'), reduce = require('./reduce'), config = require('./config'); function main() { - const loadFPayloads = etl.loadFPayloads(config.etl); - const aggregatePayloads = reduce.aggregatePayloads(config.reduce); - const deriveStatistics = reduce.deriveStatistics(config.reduce); + const paramsForHourSample = params.paramsForHourSample(config.api); + const loadFApi = source.loadFApi(config.etl); + const aggregatePayloads = reduce.aggregatePayloads(config.reduce); + const deriveStatistics = reduce.deriveStatistics(config.reduce); - let pages = [0, 5, 10, 15, 20, 25]; - let regions = ['sg', 'eu', 'na']; - - const params = R.xprod(regions, pages); - const requests = R.map( - R.apply((region, page) => loadFPayloads( - `/shards/${region}/matches`, - { 'sort': '-createdAt', 'filter[gameMode]': 'blitz_pvp_ranked', 'page[limit]': '5', 'page[offset]': page }, - )), - params); + const requests = R.map(R.apply(loadFApi), paramsForHourSample(moment().subtract(1, 'days'))); /* main */ Future.parallel(1, requests) -- cgit v1.3.1