diff options
| -rw-r--r-- | package-lock.json | 205 | ||||
| -rw-r--r-- | package.json | 1 | ||||
| -rw-r--r-- | worker.js | 41 |
3 files changed, 117 insertions, 130 deletions
diff --git a/package-lock.json b/package-lock.json index 0189428..c7b494f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,14 +5,14 @@ "requires": true, "dependencies": { "@types/geojson": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-1.0.2.tgz", - "integrity": "sha1-sC0QqwKOKSisWSoFGqpJgaGUHQM=" + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-1.0.3.tgz", + "integrity": "sha512-unWrGQhXRrNTk/MabfBLCM/rWT6zxR1OSK0GIPxsP8NX8mJcsNWkERPp4z0pTyHLiANy+Nwczf8Q2I16Pth1FA==" }, "@types/node": { - "version": "6.0.82", - "resolved": "https://registry.npmjs.org/@types/node/-/node-6.0.82.tgz", - "integrity": "sha512-uFjwSPo9dyoK0bhxMIuCsqIyQW9EHN2RcadPlA2s67TxzGDL9MJKjNAsi+TCn8ProGHcntLhgvIpbZmbtk2uCQ==" + "version": "8.0.30", + "resolved": "https://registry.npmjs.org/@types/node/-/node-8.0.30.tgz", + "integrity": "sha512-IaQtG3DWe9gRsmk1DqNnYyRVjGDVcBdZywkRVF2f62Boe8XKmlR7lNcwC6pk4V4W8nk+Zu+vdGMsOdRTDj1JPA==" }, "ajv": { "version": "4.11.8", @@ -104,6 +104,31 @@ "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-0.0.2.tgz", "integrity": "sha1-JrOIXRD6E9t/wBquOquHAZngEkw=" }, + "cache-manager": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-2.4.0.tgz", + "integrity": "sha1-Jy5O7jpOzrsoH8sIZJmATlmmolE=", + "requires": { + "async": "1.5.2", + "lru-cache": "4.0.0" + }, + "dependencies": { + "async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" + }, + "lru-cache": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.0.tgz", + "integrity": "sha1-tcvwFVbBaWb+vlTO7A+03JDfbCg=", + "requires": { + "pseudomap": "1.0.2", + "yallist": "2.1.2" + } + } + } + }, "cardinal": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/cardinal/-/cardinal-1.0.0.tgz", @@ -150,25 +175,6 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, - "cross-env": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-3.2.4.tgz", - "integrity": "sha1-ngWF8neGTtQhznVvgamA/w1piro=", - "requires": { - "cross-spawn": "5.1.0", - "is-windows": "1.0.1" - } - }, - "cross-spawn": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", - "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", - "requires": { - "lru-cache": "4.1.1", - "shebang-command": "1.2.0", - "which": "1.2.14" - } - }, "cryptiles": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz", @@ -198,9 +204,9 @@ } }, "debug": { - "version": "2.6.8", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.8.tgz", - "integrity": "sha1-5zFTHKLt4n0YgiJCfaF4IdaP9Pw=", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.0.1.tgz", + "integrity": "sha512-6nVc6S36qbt/mutyt+UGMnawAMrPDZUPQjRZI3FS9tCtDRhvxJbK79unYBLPi+z5SLXQ3ftoVBFCblQtNSls8w==", "requires": { "ms": "2.0.0" } @@ -211,14 +217,14 @@ "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, "denque": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.1.1.tgz", - "integrity": "sha1-ECKcK4juwb0V/4LF/eNW5762254=" + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.2.2.tgz", + "integrity": "sha512-x92Ql74lcTbGylXILO9Xf9S0cMpEPP04zVp2bB9e2C7G/n/Q1SgLl78RaSYEPSgpDX9uLgQXCEGAS5BI5dP3yA==" }, "depd": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz", - "integrity": "sha1-4b2Cxqq2ztlluXuIsX7T5SjKGMM=" + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz", + "integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k=" }, "dottie": { "version": "2.0.0", @@ -234,14 +240,6 @@ "jsbn": "0.1.1" } }, - "env-cmd": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/env-cmd/-/env-cmd-5.1.0.tgz", - "integrity": "sha1-AjbbOTw/AzAFIE/NCpLuQHI6nJ4=", - "requires": { - "cross-spawn": "5.1.0" - } - }, "esprima": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.0.0.tgz", @@ -283,9 +281,9 @@ "integrity": "sha1-aFj+fAlpt9TpCTM3ZHrHn2DfvnQ=" }, "generic-pool": { - "version": "3.1.7", - "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.1.7.tgz", - "integrity": "sha1-2sIrLHp6BOQXMvfY0tJaMDyI9mI=" + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.1.8.tgz", + "integrity": "sha1-CYRLZUW8kXfsIYvTXUrYlMZb4nE=" }, "getpass": { "version": "0.1.7", @@ -343,14 +341,14 @@ } }, "iconv-lite": { - "version": "0.4.18", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.18.tgz", - "integrity": "sha512-sr1ZQph3UwHTR0XftSbK85OvBbxe/abLGzEnPENCQwmHf7sck8Oyu4ob3LgBxWWxRoM+QszeUyl7jbqapu2TqA==" + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", + "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==" }, "inflection": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.10.0.tgz", - "integrity": "sha1-W//LEZetPoEFD44X4hZoCH7p6y8=" + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.12.0.tgz", + "integrity": "sha1-ogCTVlbW9fa8TcdQLhrstwMihBY=" }, "inherits": { "version": "2.0.3", @@ -367,21 +365,11 @@ "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, - "is-windows": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.1.tgz", - "integrity": "sha1-MQ23D3QtJZoWo2kgK1GvhCMzENk=" - }, "isarray": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" }, - "isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" - }, "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", @@ -485,14 +473,14 @@ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, "mysql2": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-1.3.6.tgz", - "integrity": "sha1-1cFJF94LFYAabBiDWZvPb7J8Gh4=", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-1.4.2.tgz", + "integrity": "sha1-teBACf1tY/AHVSU9+IVAolMmQLI=", "requires": { "cardinal": "1.0.0", - "denque": "1.1.1", + "denque": "1.2.2", "generate-function": "2.0.0", - "iconv-lite": "0.4.18", + "iconv-lite": "0.4.19", "long": "3.2.0", "lru-cache": "4.1.1", "named-placeholders": "1.1.1", @@ -641,13 +629,22 @@ } }, "retry-as-promised": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-2.2.0.tgz", - "integrity": "sha1-sEY9f9PPWy/tZFAKtui4pJxbjmw=", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-2.3.1.tgz", + "integrity": "sha1-91BZGD+XMHccCbrR7tV1N5McvJ0=", "requires": { "bluebird": "3.5.0", - "cross-env": "3.2.4", - "debug": "2.6.8" + "debug": "2.6.9" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + } } }, "safe-buffer": { @@ -656,9 +653,9 @@ "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" }, "semver": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", - "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=" + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.4.1.tgz", + "integrity": "sha512-WfG/X9+oATh81XtllIo/I8gOiY9EXRdv1cQdyykeXK17YcUW3EXUAi2To4pcH6nZtJPr7ZOpM5OMyWJZm+8Rsg==" }, "seq-queue": { "version": "0.0.5", @@ -666,43 +663,29 @@ "integrity": "sha1-1WgS4cAXpuTnw+Ojeh2m143TyT4=" }, "sequelize": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-4.3.2.tgz", - "integrity": "sha1-FHrRnW6lk++YME6MsLimkrS50aM=", + "version": "4.10.2", + "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-4.10.2.tgz", + "integrity": "sha512-G4MX19PFQ1J5D3CstjMyJAkaE+9+q9VLGuE10w4H/ALrBemH5uVsKLPQqfNOkokHFE3bWQsqdFLGE3+K8Dy99A==", "requires": { "bluebird": "3.5.0", "cls-bluebird": "2.0.1", - "debug": "2.6.8", - "depd": "1.1.0", + "debug": "3.0.1", + "depd": "1.1.1", "dottie": "2.0.0", - "env-cmd": "5.1.0", - "generic-pool": "3.1.7", - "inflection": "1.10.0", + "generic-pool": "3.1.8", + "inflection": "1.12.0", "lodash": "4.17.4", "moment": "2.18.1", "moment-timezone": "0.5.13", - "retry-as-promised": "2.2.0", - "semver": "5.3.0", + "retry-as-promised": "2.3.1", + "semver": "5.4.1", "terraformer-wkt-parser": "1.1.2", "toposort-class": "1.0.1", "uuid": "3.1.0", - "validator": "6.3.0", - "wkx": "0.4.1" - } - }, - "shebang-command": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", - "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", - "requires": { - "shebang-regex": "1.0.0" + "validator": "8.2.0", + "wkx": "0.4.2" } }, - "shebang-regex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", - "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=" - }, "shimmer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.1.0.tgz", @@ -763,7 +746,7 @@ "resolved": "https://registry.npmjs.org/terraformer/-/terraformer-1.0.8.tgz", "integrity": "sha1-UeCtiXRvzyFh3G9lqnDkI3fItZM=", "requires": { - "@types/geojson": "1.0.2" + "@types/geojson": "1.0.3" } }, "terraformer-wkt-parser": { @@ -817,9 +800,9 @@ "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==" }, "validator": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/validator/-/validator-6.3.0.tgz", - "integrity": "sha1-R84j7Y1Ord+p1LjvAHG2zxB418g=" + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/validator/-/validator-8.2.0.tgz", + "integrity": "sha512-Yw5wW34fSv5spzTXNkokD6S6/Oq92d8q/t14TqsS3fAiA1RYnxSFSIZ+CY3n6PGGRCq5HhJTSepQvFUS2QUDxA==" }, "verror": { "version": "1.3.6", @@ -829,14 +812,6 @@ "extsprintf": "1.0.2" } }, - "which": { - "version": "1.2.14", - "resolved": "https://registry.npmjs.org/which/-/which-1.2.14.tgz", - "integrity": "sha1-mofEN48D6CfOyvGs31bHNsAcFOU=", - "requires": { - "isexe": "2.0.0" - } - }, "winston": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/winston/-/winston-2.3.1.tgz", @@ -860,11 +835,11 @@ } }, "wkx": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.1.tgz", - "integrity": "sha1-L8FxtenLVcYlb+9L3h8hvkE77+4=", + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.2.tgz", + "integrity": "sha1-d201pjSlwi5lbkdEvetU+D/Szo0=", "requires": { - "@types/node": "6.0.82" + "@types/node": "8.0.30" } }, "yallist": { diff --git a/package.json b/package.json index 09e6984..43e460f 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "main": "worker.js", "dependencies": { "amqplib": "^0.5.1", + "cache-manager": "^2.4.0", "mysql2": "^1.4.0", "sequelize": "^4.4.2", "winston": "^2.3.1", @@ -9,6 +9,7 @@ const amqp = require("amqplib"), winston = require("winston"), loggly = require("winston-loggly-bulk"), Seq = require("sequelize"), + cacheManager = require("cache-manager"), api_name_mappings = require("../orm/mappings").map, isAbility = require("../orm/mappings").isAbility; @@ -20,7 +21,7 @@ const RABBITMQ_URI = process.env.RABBITMQ_URI, BATCHSIZE = parseInt(process.env.BATCHSIZE) || 5 * (50 + 1), // maximum number of elements to be inserted in one statement CHUNKSIZE = parseInt(process.env.CHUNKSIZE) || 100, - MAXCONNS = parseInt(process.env.MAXCONNS) || 10, // how many concurrent actions + MAXCONNS = parseInt(process.env.MAXCONNS) || 1, // how many concurrent actions DOANALYZEMATCH = process.env.DOANALYZEMATCH == "true", ANALYZE_QUEUE = process.env.ANALYZE_QUEUE || "analyze", LOAD_TIMEOUT = parseFloat(process.env.LOAD_TIMEOUT) || 5000, // ms @@ -64,6 +65,8 @@ amqp.connect(RABBITMQ_URI).then(async (rabbit) => { } }); + const cache = cacheManager.caching({store: "memory", max: 1000, ttl: 60 }); + const ch = await rabbit.createChannel(); await ch.assertQueue(QUEUE, { durable: true }); await ch.assertQueue(QUEUE + "_failed", { durable: true }); @@ -187,24 +190,32 @@ amqp.connect(RABBITMQ_URI).then(async (rabbit) => { // data from Telemetry, one phase (early/mid/late/…) per obj await Promise.map(telemetry_objects, async (telemetry) => { + let dbpreload_profiler = logger.startTimer(); + // api -> telemetry format const sideToTeam = (s) => s == "left/blue"? "Left" : "Right", // yes there is yet another format and yes it's strings sideToTeamNo = (s) => s == "left/blue"? "1" : "2"; // get match participant references - const participants = await model.Participant.findAll({ - where: { match_api_id: telemetry.match_api_id }, - include: [ { // TODO rm once pushed to participant - model: model.Roster, - attributes: [ "side" ] - } ] - }).map((p) => { return { - api_id: p.api_id, - player_api_id: p.player_api_id, - actor: p.actor, - team: sideToTeam(p.roster.side), - teamNo: sideToTeamNo(p.roster.side) - } }); + const participants = + (await cache.wrap(telemetry.match_api_id, async () => + await model.Participant.findAll({ + where: { match_api_id: telemetry.match_api_id }, + attributes: [ "api_id", "player_api_id", "actor" ], + include: [ { // TODO rm once pushed to participant + model: model.Roster, + attributes: [ "side" ] + } ] + }) + ) ).map((p) => { return { + api_id: p.api_id, + player_api_id: p.player_api_id, + actor: p.actor, + team: sideToTeam(p.roster.side), + teamNo: sideToTeamNo(p.roster.side) + } }); + + dbpreload_profiler.done("loading relationships from db"); // seconds since epoch; first spawn time const matchstart = new Date(Date.parse(telemetry.match_start)).getTime() / 1000; @@ -510,7 +521,7 @@ amqp.connect(RABBITMQ_URI).then(async (rabbit) => { } }); participant_phase_records = participant_phase_records.concat( participants_phase); // TODO calc stats - }); + }, { concurrency: MAXCONNS }); let transaction_profiler = logger.startTimer(); // now access db |