diff options
Diffstat (limited to 'service_grab.js')
| -rw-r--r-- | service_grab.js | 295 |
1 files changed, 295 insertions, 0 deletions
diff --git a/service_grab.js b/service_grab.js new file mode 100644 index 0000000..96770ce --- /dev/null +++ b/service_grab.js @@ -0,0 +1,295 @@ +#!/usr/bin/env node +/* jshint esnext: true */ +"use strict"; + +const api = require("../orm/api"), + Promise = require("bluebird"), + Seq = require("sequelize"), + Service = require("./service_skeleton.js"); + +const GRABSTART = process.env.GRABSTART || "2017-02-14T00:00:00Z", + BRAWL_RETENTION_DAYS = parseInt(process.env.BRAWL_RETENTION_DAYS) || 3, + PLAYER_PROCESS_QUEUE = process.env.PLAYER_PROCESS_QUEUE || "process", + PLAYER_TOURNAMENT_PROCESS_QUEUE = process.env.PLAYER_TOURNAMENT_PROCESS_QUEUE || "process_tournament", + GRAB_QUEUE = process.env.GRAB_QUEUE || "grab", + GRAB_BRAWL_QUEUE = process.env.GRAB_BRAWL_QUEUE || "grab_brawl", + GRAB_TOURNAMENT_QUEUE = process.env.GRAB_TOURNAMENT_QUEUE || "grab_tournament", + REGIONS = (process.env.REGIONS || "na,eu,sg,sa,ea").split(","), + TOURNAMENT_REGIONS = (process.env.TOURNAMENT_REGIONS || "tournament-na," + + "tournament-eu,tournament-sg,tournament-sa,tournament-ea").split(","); + +module.exports = class Analyzer extends Service { + constructor() { + super(); + + this.setTargets({ + "regular": GRAB_QUEUE, + "brawl": GRAB_BRAWL_QUEUE, + "tournament": GRAB_TOURNAMENT_QUEUE, + "regular_player": PLAYER_PROCESS_QUEUE, + "tournament_player": PLAYER_TOURNAMENT_PROCESS_QUEUE + }); + + this.setRoutes({ + // update a whole region (tournament only) + "/api/match/:region/update": async (req, res) => { + const region = req.params.region; + if (TOURNAMENT_REGIONS.indexOf(region) == -1) { + logger.error("called with unsupported region", { region: region }); + res.sendStatus(404); + return; + } + this.updateRegion(region, "tournament"); // fire away + res.sendStatus(204); + }, + // grab multiple users by IGN + "/api/players/:region/grab/:category*?": async (req, res) => { + let grabstart = new Date(); + grabstart.setTime(grabstart.getTime() - req.query.minutesAgo * 60000); + this.grabPlayers(req.query.names, req.params.region, grabstart, + req.params.category || "regular"); + res.sendStatus(204); + }, + + // find a player / force a player update + "/api/player/:name/search/:category*?": async (req, res) => { + this.searchPlayer(req.params.name, req.params.category || "regular"); + res.sendStatus(204); // notifications will follow + }, + // update a known user + // flow: + // * get from db (depending on category, either regular, brawl or tourn) + // * update lifetime from `/players` via id, catch IGN changes + // * update from `/matches` + "/api/player/:name/update/:category*?": async (req, res) => { + const name = req.params.name, + category = req.params.category || "regular", + db = this.getDatabase(category), + players = await db.Player.findAll({ where: { name: req.params.name } }); + if (players == undefined) { + logger.error("player not found in db", { name: req.params.name }); + res.sendStatus(404); + return; + } + logger.info("player in db, updating", { name: name, category: category }); + players.forEach((player) => this.updatePlayer(player, category)); + res.sendStatus(204); + }, + // update a random user + "/api/player": async (req, res) => { + // TODO this becomes slow + let player = await this.getDatabase("regular").Player.findOne({ order: [ Seq.fn("RAND") ] }); + this.updatePlayer(player, "regular"); + res.json(player); + } + }); + } + + // convenience + getGameModes(category) { + switch (category) { + case "regular": return "casual,ranked"; + case "brawl": return "casual_aral,blitz_pvp_ranked"; + case "tournament": return "private,private_party_draft_match"; + } + logger.error("unsupported game mode category", { category: category }); + } + getRegions(category) { + switch (category) { + case "brawl": + case "regular": return REGIONS; + case "tournament": return TOURNAMENT_REGIONS; + } + logger.error("unsupported region category", { category: category }); + } + // return a default `createdAt-start` based on game mode + getGrabstart(category) { + switch (category) { + case "regular": return GRABSTART; + case "brawl": + let past = new Date(); + past.setDate(past.getDate() - BRAWL_RETENTION_DAYS); + return past.toISOString(); // minimum + case "tournament": return "2017-02-12T00:00:00Z"; // all + } + logger.error("unsupported grabstart category", { category: category }); + } + + // API requires maximum time interval of 4w + // return [payload, payload, …] with fixed createdAt + getSplitGrabs(payload, start, end) { + let part_start = start, part_end = start, + payloads = []; + // loop forwards, adding 4w until we passed NOW + while (part_start < end) { + part_end = new Date(part_end.getTime() + (4 * 60*60*24*7*1000)); // add 4w + if(part_end > end) part_end = end; + let pl = payload; + pl["params"]["filter[createdAt-start]"] = part_start.toISOString(); + pl["params"]["filter[createdAt-end]"] = part_end.toISOString(); + // push a deep clone or the object will have the reference to part_start + // which leads to all start/end dates being the same (?!) + payloads.push(JSON.parse(JSON.stringify(pl))); // JavaScript sucks. + part_start = part_end; + } + return payloads; + } + + // request grab jobs for a player's matches + async grabPlayer(name, region, last_match_created_date, id, category) { + const start = new Date(Date.parse(last_match_created_date)), + end = new Date(); // today + + const payload_template = { + "region": region, + "params": { + "filter[playerIds]": id, + "filter[gameMode]": this.getGameModes(category), + "sort": "createdAt" + } + }; + await Promise.each(this.getSplitGrabs(payload_template, start, end), async (payload) => { + logger.info("requesting update", { name: name, region: region }); + + await this.forward(this.getTarget(category), JSON.stringify(payload), { + persistent: true, + headers: { notify: "player." + name } + }); + }); + } + + // request grab job for multiple players' matches + async grabPlayers(names, region, grabstart, category) { + const payload_template = { + "region": region, + "params": { + "filter[playerNames]": names, + "filter[gameMode]": this.getGameModes(category), + "sort": "createdAt" + } + }, now = new Date(); + logger.info("requesting triple player grab", { + names: names, region: region, category: category + }); + + await Promise.each(this.getSplitGrabs(payload_template, grabstart, now), async (payload) => { + await this.forward(this.getTarget(category), JSON.stringify(payload), { + persistent: true, + headers: { notify: "player." + names } + }); + }); + } + + // request grab jobs for a region's matches + async grabMatches(region, lmcd) { + if (lmcd == undefined) lmcd = new Date(Date.parse(this.getGrabstart("tournament"))); + const now = new Date(); + + // add 1s, because createdAt-start <= x <= createdAt-end + // so without the +1s, we'd always get the last_match_created_date match back + last_match_created_date.setSeconds(last_match_created_date.getSeconds() + 1); + + const payload_template = { + "region": region, + "params": { "sort": "createdAt" } + }; + logger.info("requesting region update", { region: region }); + + await Promise.each(this.getSplitGrabs(payload_template, last_match_created_date, now), async (payload) => { + await this.forward(GRAB_TOURNAMENT_QUEUE, + JSON.stringify(payload), { persistent: true }); + }); + } + + // search for a player name in one region + // request process for lifetime + // return an array (JSONAPI response) + async searchPlayerInRegion(region, name, id, category) { + logger.info("searching", { name: name, id: id, region: region }); + let options = {}; + if (id == undefined) options["filter[playerNames]"] = name + else options["filter[playerIds]"] = id + + // players.length and page length will be 1 in 99.9999% of all cases + // - but just in case. + const players = await Promise.map(await api.requests("players", region, options, logger), async (player) => { + logger.info("found", { name: name, id: id, region: region }); + await this.notify("player." + name, "search_success"); + // send to processor, so the player is in db + // no matter whether we find matches or not + await this.forward(this.getTarget(category + "_player"), JSON.stringify(player), { + persistent: true, type: "player", + headers: { notify: "player." + player.name } + }); + return player; + }); + if (players.length == 0) + logger.warn("not found", { name: name, id: id, region: region }); + + return players; + } + + // search for player name on all shards + // grab player(s) + // send a notification for results and request updates + async searchPlayer(name, category) { + let found = false; + logger.info("searching", { name: name }); + await Promise.map(this.getRegions(category), async (region) => { + const players = await this.searchPlayerInRegion(region, name, undefined, category); + if (players.length > 0) found = true; + + // request grab jobs + await Promise.map(players, async (p) => + await this.grabPlayer(p.name, p.shardId, + this.getGrabstart(category), p.id, category)); + }); + // notify web + if (!found) { + logger.info("search failed", { name: name }); + await this.notify("player." + name, "search_fail"); + } + } + + // update a player from API based on db record + // & grab player + async updatePlayer(player, category) { + // set last_update and request an update job + // if last_update is null, we need that player's full history + let grabstart; + if (player.get("last_update") != null && category == "regular") + // TODO do not fetch everything on category tournament / brawl + grabstart = player.get("created_at"); + if (grabstart == undefined) grabstart = this.getGrabstart(category); + else + // add 1s, because createdAt-start <= x <= createdAt-end + // so without the +1s, we'd always get the last_match_created_date match back + grabstart.setSeconds(grabstart.getSeconds() + 1); + + const players = await this.searchPlayerInRegion( + player.shard_id, player.name, player.api_id, category); + + // will be the same as `player` 99.9% of the time + // but not when the player changed their name + // search happened by ID, so we get only 1 player back + // but player.name != players[0].attributes.name + // (with a name change) + await Promise.map(players, async (p) => + await this.grabPlayer(p.name, p.shardId, grabstart, p.id, category)); + } + + // update a region from API based on db records + async updateRegion(region, category) { + let grabstart; + const last_match = await this.getDatabase(category).Participant.findOne({ + attributes: ["created_at"], + where: { shard_id: region }, + order: [ [seq.col("created_at"), "DESC"] ] + }); + if (last_match == null) grabstart = undefined; + else grabstart = last_match.get("created_at"); + + await this.grabMatches(region, grabstart); + } + +} |