diff options
| -rw-r--r-- | api.js | 24 | ||||
| -rw-r--r-- | config.js | 7 | ||||
| -rw-r--r-- | index.js | 23 | ||||
| -rw-r--r-- | package-lock.json | 10 | ||||
| -rw-r--r-- | params.js | 37 |
5 files changed, 75 insertions, 26 deletions
@@ -1,17 +1,17 @@ #!/usr/bin/node -const requestP = require('request-promise') - Future = require('fluture'); +const requestP = require('request-promise'), + config = require('./config'), + Future = require('fluture'), + FutureRetry = require('fluture-retry'); -const API_KEY = process.env.API_KEY || require('./secret').apikey, - API_BASE = 'https://api.dc01.gamelockerapp.com', - API_TIMEOUT = 1; // seconds +const API_KEY = process.env.API_KEY || require('./secret').apikey; const requestF = Future.encaseP(requestP); function apiRequest(path, query) { - return requestF({ - uri: API_BASE + path, + return FutureRetry.retryLinearly(requestF({ + uri: config.api.baseUrl + path, headers: { 'X-Title-Id': 'semc-vainglory', 'Authorization': API_KEY @@ -20,20 +20,20 @@ function apiRequest(path, query) { json: true, gzip: true, forever: true, - timeout: API_TIMEOUT*1000, + timeout: config.api.timeout * 1000, strictSSL: true - }); + })); } function awsRequest(url) { - return requestF({ + return FutureRetry.retryLinearly(requestF({ uri: url, json: true, gzip: true, forever: true, - timeout: API_TIMEOUT*1000, + timeout: config.api.timeout * 1000, strictSSL: true - }); + })); } module.exports = { @@ -1,6 +1,13 @@ #!/usr/bin/node module.exports = { + api: { + baseUrl: 'https://api.dc01.gamelockerapp.com', + timeout: 5, + regions: ['sg', 'eu', 'na'], + rpm: 50, + modes: ['blitz_pvp_ranked'], + }, etl: { participant: [ ['Winner', ['attributes', 'stats', 'winner']] ], roster: [ ], @@ -1,25 +1,20 @@ #!/usr/bin/node const R = require('ramda'), - etl = require('./etl'), + Future = require('fluture'), + moment = require('moment'), + source = require('./source'), + params = require('./params'), reduce = require('./reduce'), config = require('./config'); function main() { - const loadFPayloads = etl.loadFPayloads(config.etl); - const aggregatePayloads = reduce.aggregatePayloads(config.reduce); - const deriveStatistics = reduce.deriveStatistics(config.reduce); + const paramsForHourSample = params.paramsForHourSample(config.api); + const loadFApi = source.loadFApi(config.etl); + const aggregatePayloads = reduce.aggregatePayloads(config.reduce); + const deriveStatistics = reduce.deriveStatistics(config.reduce); - let pages = [0, 5, 10, 15, 20, 25]; - let regions = ['sg', 'eu', 'na']; - - const params = R.xprod(regions, pages); - const requests = R.map( - R.apply((region, page) => loadFPayloads( - `/shards/${region}/matches`, - { 'sort': '-createdAt', 'filter[gameMode]': 'blitz_pvp_ranked', 'page[limit]': '5', 'page[offset]': page }, - )), - params); + const requests = R.map(R.apply(loadFApi), paramsForHourSample(moment().subtract(1, 'days'))); /* main */ Future.parallel(1, requests) diff --git a/package-lock.json b/package-lock.json index b9f2aa0..87de321 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1387,6 +1387,11 @@ "sanctuary-type-identifiers": "2.0.1" } }, + "fluture-retry": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fluture-retry/-/fluture-retry-1.0.0.tgz", + "integrity": "sha512-a/PYnJ5rmxgNl94tem0PylMFSA8HpO53to3gbOcgmaRmwrl95x1X3mR6rZHichnxIQ3wiy3jLyIZXQMChM6VVA==" + }, "follow-redirects": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.4.1.tgz", @@ -3264,6 +3269,11 @@ "minimist": "0.0.8" } }, + "moment": { + "version": "2.22.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.1.tgz", + "integrity": "sha512-shJkRTSebXvsVqk56I+lkb2latjBs8I+pc2TzWc545y2iFnSjm7Wg0QMh+ZWcdSLQyGEau5jI8ocnmkyTgr9YQ==" + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", diff --git a/params.js b/params.js new file mode 100644 index 0000000..b3dcb98 --- /dev/null +++ b/params.js @@ -0,0 +1,37 @@ +#!/usr/bin/node + +const R = require('ramda'); + +function paramsForHourSample(config) { + const requestArgsForRegionStartEnd = R.curry((region, start, end) => [ + `/shards/${region}/matches`, { + 'sort': '-createdAt', + 'filter[gameMode]': R.join(',', config.modes), + 'filter[createdAt-start]': start.toISOString(), + 'filter[createdAt-end]': end.toISOString(), + 'page[limit]': '5', + 'page[offset]': '0', + } + ]); + + const requestsPerMinutePerRegion = Math.floor(config.rpm / config.regions.length); + const splitDurationMinutes = Math.floor(60 / requestsPerMinutePerRegion); + const splitMoments = (hour) => R.map( + (offsetIndex) => [ + hour.clone().startOf('hour').minutes(offsetIndex * splitDurationMinutes), + hour.clone().startOf('hour').minutes((offsetIndex + 1) * splitDurationMinutes), + ], + R.range(0, requestsPerMinutePerRegion), + ); + + return R.pipe( + splitMoments, + R.xprod(config.regions), + R.map(R.unnest), + R.map(R.apply(requestArgsForRegionStartEnd)), + ); +} + +module.exports = { + paramsForHourSample, +}; |