summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
m---------discordbot0
-rw-r--r--updater/.gitignore59
-rw-r--r--updater/api.js331
-rw-r--r--updater/index.html60
-rw-r--r--updater/package.json19
m---------vaindock0
m---------vainsocial0
7 files changed, 469 insertions, 0 deletions
diff --git a/discordbot b/discordbot
-Subproject 5b2c6e3f290379f1124758d77c55db8b79c2df8
+Subproject 8adfdfe94e994821594678625fb3dd6cb31b9d9
diff --git a/updater/.gitignore b/updater/.gitignore
new file mode 100644
index 0000000..00cbbdf
--- /dev/null
+++ b/updater/.gitignore
@@ -0,0 +1,59 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Typescript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+
diff --git a/updater/api.js b/updater/api.js
new file mode 100644
index 0000000..d6525dc
--- /dev/null
+++ b/updater/api.js
@@ -0,0 +1,331 @@
+#!/usr/bin/env node
+/* jshint esnext: true */
+
+var request = require("request-promise");
+
+var pg = require("pg");
+var Pool = pg.Pool;
+var db_config_raw = {
+ user: "vainraw",
+ password: "vainraw",
+ host: "localhost",
+ database: "vainsocial-raw",
+ port: 5433,
+ max: 10
+};
+var db_config_web = {
+ user: "vainweb",
+ password: "vainweb",
+ host: "localhost",
+ database: "vainsocial-web",
+ port: 5432,
+ max: 10
+};
+var pool_raw = new Pool(db_config_raw);
+var pool_web = new Pool(db_config_web);
+
+var app = require("express")();
+var http = require("http").Server(app);
+var io = require("socket.io")(http);
+
+var APITOKEN = process.env.VAINSOCIAL_APITOKEN;
+if (APITOKEN == undefined) throw "Need a valid API token!";
+
+http.listen(8080);
+
+/* API helper */
+/* search for player name across all shards */
+async function api_playerByAttr(attr, val) {
+ var regions = ["na", "eu", "sg"],
+ finds = [],
+ filter = "filter[" + attr + "]";
+
+ for (let region of regions) {
+ var options = {
+ uri: "https://api.dc01.gamelockerapp.com/shards/" + region + "/players",
+ headers: {
+ "X-Title-Id": "semc-vainglory",
+ "Authorization": APITOKEN
+ },
+ qs: {
+ filter: val
+ },
+ json: true,
+ gzip: true
+ };
+ try {
+ res = await request(options);
+ finds.push({
+ "region": res.data[0].attributes.shardId,
+ "id": res.data[0].id,
+ "last_update": res.data[0].attributes.createdAt,
+ "source": "api"
+ });
+ } catch (err) {
+ // TODO
+ }
+ }
+
+ if (finds.length == 0)
+ return undefined;
+
+ // due to an API bug, many players are also present in NA
+ // TODO: get history for all regions in case of region transfer
+ finds.sort((a, b) => { return a.last_update < b.last_update; });
+ return finds[0];
+}
+async function api_playerByName(name) {
+ return await api_playerByAttr("playerNames", name);
+}
+async function api_playerById(id) {
+ return await api_playerByAttr("playerIds", id);
+}
+
+
+/* DB helper */
+/* retry until no serialization error */
+async function db_serialized(con, query, data) {
+ var commit_success = false,
+ res;
+ do {
+ /* transaction begin */
+ try {
+ await con.query("BEGIN");
+ await con.query("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");
+ res = await con.query(query, data);
+ await con.query("COMMIT");
+ commit_success = true;
+ } catch (err) {
+ await con.query("ROLLBACK");
+ // serialization error - expected, else rethrow
+ console.error(err);
+ if (err.code != "40001") throw err;
+ }
+ /* transaction end */
+ } while (!commit_success);
+ return res;
+}
+
+/* find player by id or by name in db */
+async function db_playerByAttr(attr, val) {
+ var web = await pool_web.connect(),
+ player = await web.query(`
+ SELECT name, api_id, shard_id, last_match_created_date
+ FROM player WHERE ` + attr + `=$1
+ `, [val]);
+ web.release();
+
+ if (player.rows.length > 0) {
+ return {
+ "name": player.rows[0].name,
+ "id": player.rows[0].api_id,
+ "region": player.rows[0].shard_id,
+ "last_update": player.rows[0].last_match_created_date,
+ "source": "db"
+ };
+ }
+ return undefined;
+}
+
+async function db_playerByName(name) {
+ return await db_playerByAttr("name", name);
+}
+async function db_playerById(id) {
+ return await db_playerByAttr("api_id", id);
+}
+
+/* returns a player by name from db or API */
+async function playerByName(name) {
+ var player = await db_playerByName(name);
+ if (player != undefined) return player;
+
+ console.log("player '" + name + "' not found in db");
+ player = await api_playerByName(name);
+ if (player != undefined) return player;
+
+ console.log("player '" + name + "' not found in API");
+ return undefined;
+}
+/* returns a player by id from db or API */
+async function playerById(id) {
+ var player = await db_playerById(id);
+ if (player != undefined) return player;
+
+ console.log("player with id '" + name + "' not found in db");
+ player = await api_playerById(id);
+ if (player != undefined) return player;
+
+ console.log("player with id '" + name + "' not found in API");
+ return undefined;
+}
+
+
+/* update request helpers */
+/* upsert a job */
+async function upsertGrabjob(payload) {
+ var raw = await pool_raw.connect(),
+ job;
+
+ // find and prioritize existing jobs
+ job = await db_serialized(raw, `
+ UPDATE jobs SET priority=0
+ WHERE
+ (
+ (type='grab' AND payload=$1) OR
+ (type='process' AND payload->>'playername'=$1->'params'->>'filter[playerNames]') OR
+ (type='compile' AND payload->>'type'='player' AND payload->>'id'=$1->'params'->>'filter[playerIds]')
+ ) AND status<>'finished' AND status<>'failed'
+ RETURNING id
+ `, [payload]);
+ // TODO job dependency information format on jobs is shit
+ // (2.0)
+
+ if (job.rows.length == 0) {
+ // this job is currently not running, insert it
+ job = await db_serialized(raw, `
+ INSERT INTO jobs(type, payload, priority)
+ VALUES('grab', $1, 0)
+ RETURNING id
+ `, [payload]);
+ // wake apigrabber up
+ await raw.query(`NOTIFY grab_open`);
+ console.log("job requested: '%j'", payload);
+ }
+
+ raw.release();
+ return job.rows[0];
+}
+
+async function playerRequestUpdate(name, id) {
+ if (name == undefined && id == undefined) // fail hard and die
+ throw "playerRequestUpdate needs either name or id";
+
+ var player;
+ if (id != undefined) // prefer id over name
+ player = await playerById(id);
+ else
+ player = await playerByName(name);
+ if (player == undefined)
+ return undefined;
+
+ console.log("updating player '%j'", player);
+
+ // if last_update is from our db, use it as a start, else get the whole history
+ if (player.source == "api" || player.last_update == undefined)
+ player.last_update = new Date(value=0); // forever ago
+ /* comment out on shutter's machine
+ if (player.source == "db")
+ player.last_update.setMinutes(player.last_update.getMinutes() - new Date().getTimezoneOffset()); // TODO workaround for my broken db schema
+ */
+
+ // add 1s, because createdAt-start <= x <= createdAt-end
+ // so without the +1s, we'd always get the last_match_created_date match back
+ player.last_update.setSeconds(player.last_update.getSeconds() + 1);
+
+ var timedelta_minutes = ((new Date()) - player.last_update) / 1000 / 60;
+ if (timedelta_minutes < 30) {
+ console.log("player '" + player.name + "' update skipped");
+ return;
+ }
+
+ var payload = {
+ "region": player.region,
+ "params": {
+ "filter[playerIds]": player.id,
+ "filter[playerNames]": player.name, // TODO remove in 2.0 - backwards compat
+ "filter[createdAt-start]": player.last_update.toISOString(),
+ "filter[gameMode]": "casual,ranked"
+ }
+ };
+ job = await upsertGrabjob(payload);
+
+ return player;
+}
+async function playerRequestUpdateByName(name) {
+ return await playerRequestUpdate(name, undefined);
+}
+async function playerRequestUpdateById(id) {
+ return await playerRequestUpdate(undefined, id);
+}
+
+/* routes */
+/* request a grab job */
+app.get("/api/player/name/:name", async (req, res) => {
+ player = await playerRequestUpdateByName(req.params.name);
+ if (player == undefined) res.sendStatus(404);
+ else res.json(player);
+});
+app.get("/api/player/id/:id", async (req, res) => {
+ player = await playerRequestUpdateById(req.params.id);
+ if (player == undefined) res.sendStatus(404);
+ else res.json(player);
+});
+
+/* internal monitoring */
+app.get("/", async (req, res) => {
+ res.sendFile(__dirname + "/index.html");
+});
+
+
+/* notifications from database */
+async function listen() {
+ var client = new pg.Client(db_config_raw);
+ await client.connect();
+
+ /* job status change notification listener */
+ client.on('notification', async (msg) => {
+ var raw = await pool_raw.connect(),
+ jobs;
+
+ // find all interesting jobs, delete them & forward their notification
+ if (msg.channel == "grab_failed") {
+ jobs = await raw.query(`
+ DELETE FROM jobs WHERE
+ type='grab' AND status='failed' AND payload->>'error'='Not Found'
+ RETURNING
+ payload->'params'->>'filter[playerIds]' AS player_id,
+ payload->'params'->>'filter[playerNames]' AS player_name
+ `);
+ }
+ if (msg.channel == "process_finished") {
+ jobs = await raw.query(`
+ DELETE FROM jobs WHERE
+ type='process' AND status='finished'
+ RETURNING payload->>'playername' AS player_name
+ `);
+ }
+ if (msg.channel == "compile_finished") {
+ jobs = await raw.query(`
+ DELETE FROM jobs WHERE
+ type='compile' AND payload->>'type'='player' AND status='finished'
+ RETURNING payload->>'id' AS player_id
+ `);
+ }
+
+ raw.release();
+ if (jobs == undefined) return; // nothing to do
+
+ // forward notification to all playername / playerid channels
+ for (let job of jobs.rows) {
+ var name = job.player_name;
+ var id = job.player_id;
+ var player;
+ if (name == undefined && id == undefined) throw "notification needs either name or ID";
+ if (name == undefined)
+ player = await db_playerById(job.player_id);
+ else
+ player = await db_playerByName(job.player_name);
+ if (player == undefined) throw "player had a job, but doesn't exist";
+ console.log("sending '%s' notification for player '%s'", msg.channel, player.name);
+ io.emit(player.name, msg.channel);
+ io.emit(player.id, msg.channel);
+ }
+ });
+ client.query("LISTEN process_finished");
+ client.query("LISTEN compile_finished");
+ client.query("LISTEN analyze_finished");
+ client.query("LISTEN grab_failed");
+ // keep open forever
+}
+
+listen();
diff --git a/updater/index.html b/updater/index.html
new file mode 100644
index 0000000..b97d287
--- /dev/null
+++ b/updater/index.html
@@ -0,0 +1,60 @@
+<!doctype html>
+<html>
+ <head>
+ <title>Vainsocial backend status</title>
+ </head>
+ <body>
+ <form id="update-form">
+ <input type="text">
+ <input type="submit" value="Update">
+ </form>
+
+ <ul id="updates"></ul>
+ <script src="/socket.io/socket.io.js"></script>
+ <script src="https://code.jquery.com/jquery-3.1.1.min.js"></script>
+ <script>
+ var socket = io();
+ $("#update-form").submit(function(e) {
+ var name = $("input:first").val();
+ $.get("/api/player/name/" + name).done(function() {
+ $("#updates").append(
+ $("<li>")
+ .text("(found)")
+ .css("color", "black")
+ );
+ /* subscribe to socket notifications */
+ socket.on(name, function(msg) {
+ if (msg == "grab_failed") {
+ $("#updates").append(
+ $("<li>")
+ .text(name + ": no new data")
+ .css("color", "red")
+ );
+ }
+ if (msg == "process_finished") {
+ $("#updates").append(
+ $("<li>")
+ .text(name + ": processed")
+ .css("color", "green")
+ );
+ }
+ if (msg == "compile_finished") {
+ $("#updates").append(
+ $("<li>")
+ .text(name + ": compiled")
+ .css("color", "orange")
+ );
+ }
+ });
+ }).fail(function() {
+ $("#updates").append(
+ $("<li>")
+ .text("(not found)")
+ .css("color", "red")
+ );
+ });
+ e.preventDefault();
+ });
+ </script>
+ </body>
+</html>
diff --git a/updater/package.json b/updater/package.json
new file mode 100644
index 0000000..bd514d2
--- /dev/null
+++ b/updater/package.json
@@ -0,0 +1,19 @@
+{
+ "name": "vainsocial-updater",
+ "version": "2.0.0",
+ "description": "Vainsocial frontend to backend communication service",
+ "main": "api.js",
+ "dependencies": {
+ "express": "^4.15.2",
+ "pg": "^6.1.4",
+ "request": "^2.81.0",
+ "request-promise": "^4.2.0",
+ "socketio": "^1.0.0"
+ },
+ "devDependencies": {},
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": "",
+ "license": ""
+}
diff --git a/vaindock b/vaindock
-Subproject 7257eb10f10b9d7ae1d2e326e2401216b645bd6
+Subproject 5cce35f75ba8b42ecbc3debebbe6db750350075
diff --git a/vainsocial b/vainsocial
-Subproject b152e8b807a40c13ae39fc23d44ec2f487dc0b2
+Subproject 25c3927cafadfe7a2a903a79c3dc906b85a4ec2
generated by cgit v1.3.1 (git 2.54.0) at 2026-08-02 04:33:27 +0000