summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorschneefux <schneefux+commit@schneefux.xyz>2017-07-14 18:56:30 +0200
committerschneefux <schneefux+commit@schneefux.xyz>2017-07-14 18:56:30 +0200
commitdd6c4c39440982e2943906f2fa30dbbb8260540b (patch)
tree19e6e1732c4b2cb368d9f1a61e65472a6878143d
parent40afdde86f8b9369c2f3e1ef73176c347ccef5f6 (diff)
downloadbridge-dd6c4c39440982e2943906f2fa30dbbb8260540b.tar.gz
bridge-dd6c4c39440982e2943906f2fa30dbbb8260540b.zip
2.14.0 rewrite: modularization
Diffstat
-rw-r--r--package-lock.json1286
-rw-r--r--package.json11
-rw-r--r--server.js716
-rw-r--r--service_analyze.js83
-rw-r--r--service_crunch.js133
-rw-r--r--service_grab.js295
-rw-r--r--service_skeleton.js73
-rw-r--r--service_telemetry.js43
8 files changed, 1957 insertions, 683 deletions
diff --git a/package-lock.json b/package-lock.json
new file mode 100644
index 0000000..bc06092
--- /dev/null
+++ b/package-lock.json
@@ -0,0 +1,1286 @@
+{
+ "name": "bridge",
+ "version": "2.0.0",
+ "lockfileVersion": 1,
+ "requires": true,
+ "dependencies": {
+ "@types/geojson": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-1.0.2.tgz",
+ "integrity": "sha1-sC0QqwKOKSisWSoFGqpJgaGUHQM="
+ },
+ "@types/node": {
+ "version": "6.0.82",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-6.0.82.tgz",
+ "integrity": "sha512-uFjwSPo9dyoK0bhxMIuCsqIyQW9EHN2RcadPlA2s67TxzGDL9MJKjNAsi+TCn8ProGHcntLhgvIpbZmbtk2uCQ=="
+ },
+ "accepts": {
+ "version": "1.3.3",
+ "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.3.tgz",
+ "integrity": "sha1-w8p0NJOGSMPg2cHjKN1otiLChMo=",
+ "requires": {
+ "mime-types": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.15.tgz",
+ "negotiator": "0.6.1"
+ }
+ },
+ "amqplib": {
+ "version": "0.5.1",
+ "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.5.1.tgz",
+ "integrity": "sha1-fMz+ur5WwumE6noiQ/fO/m+/xs8=",
+ "requires": {
+ "bitsyntax": "0.0.4",
+ "bluebird": "3.5.0",
+ "buffer-more-ints": "0.0.2",
+ "readable-stream": "1.1.14"
+ },
+ "dependencies": {
+ "bitsyntax": {
+ "version": "0.0.4",
+ "resolved": "https://registry.npmjs.org/bitsyntax/-/bitsyntax-0.0.4.tgz",
+ "integrity": "sha1-6xDMb4K4xJDj6FaY8H6D1G4MuoI=",
+ "requires": {
+ "buffer-more-ints": "0.0.2"
+ }
+ },
+ "buffer-more-ints": {
+ "version": "0.0.2",
+ "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-0.0.2.tgz",
+ "integrity": "sha1-JrOIXRD6E9t/wBquOquHAZngEkw="
+ },
+ "core-util-is": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+ "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
+ },
+ "inherits": {
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+ "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
+ },
+ "isarray": {
+ "version": "0.0.1",
+ "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
+ "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
+ },
+ "readable-stream": {
+ "version": "1.1.14",
+ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
+ "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
+ "requires": {
+ "core-util-is": "1.0.2",
+ "inherits": "2.0.3",
+ "isarray": "0.0.1",
+ "string_decoder": "0.10.31"
+ }
+ },
+ "string_decoder": {
+ "version": "0.10.31",
+ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
+ "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
+ }
+ }
+ },
+ "ansicolors": {
+ "version": "0.2.1",
+ "resolved": "https://registry.npmjs.org/ansicolors/-/ansicolors-0.2.1.tgz",
+ "integrity": "sha1-vgiVmQl7dKXJxKhKDNvNtivYeu8="
+ },
+ "array-flatten": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
+ "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
+ },
+ "bluebird": {
+ "version": "3.5.0",
+ "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.0.tgz",
+ "integrity": "sha1-eRQg1/VR7qKJdFOop3ZT+WYG1nw="
+ },
+ "cardinal": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/cardinal/-/cardinal-1.0.0.tgz",
+ "integrity": "sha1-UOIcGwqjdyn5N33vGWtanOyTLuk=",
+ "requires": {
+ "ansicolors": "0.2.1",
+ "redeyed": "1.0.1"
+ }
+ },
+ "cls-bluebird": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/cls-bluebird/-/cls-bluebird-2.0.1.tgz",
+ "integrity": "sha1-wlmkgK4CwOUGE0MHuxPbMERu4uc=",
+ "requires": {
+ "is-bluebird": "1.0.2",
+ "shimmer": "1.1.0"
+ }
+ },
+ "content-disposition": {
+ "version": "0.5.2",
+ "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz",
+ "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ="
+ },
+ "content-type": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.2.tgz",
+ "integrity": "sha1-t9ETrueo3Se9IRM8TcJSnfFyHu0="
+ },
+ "cookie": {
+ "version": "0.3.1",
+ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz",
+ "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s="
+ },
+ "cookie-signature": {
+ "version": "1.0.6",
+ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
+ "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
+ },
+ "core-util-is": {
+ "version": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+ "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
+ },
+ "cross-env": {
+ "version": "3.2.4",
+ "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-3.2.4.tgz",
+ "integrity": "sha1-ngWF8neGTtQhznVvgamA/w1piro=",
+ "requires": {
+ "cross-spawn": "5.1.0",
+ "is-windows": "1.0.1"
+ }
+ },
+ "cross-spawn": {
+ "version": "5.1.0",
+ "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz",
+ "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=",
+ "requires": {
+ "lru-cache": "4.1.1",
+ "shebang-command": "1.2.0",
+ "which": "1.2.14"
+ }
+ },
+ "debug": {
+ "version": "https://registry.npmjs.org/debug/-/debug-2.6.1.tgz",
+ "integrity": "sha1-eYVQkLosTjEVzH2HaUkdWPBJE1E=",
+ "requires": {
+ "ms": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz"
+ }
+ },
+ "denque": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/denque/-/denque-1.1.1.tgz",
+ "integrity": "sha1-ECKcK4juwb0V/4LF/eNW5762254="
+ },
+ "depd": {
+ "version": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz",
+ "integrity": "sha1-4b2Cxqq2ztlluXuIsX7T5SjKGMM="
+ },
+ "destroy": {
+ "version": "1.0.4",
+ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
+ "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
+ },
+ "dottie": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.0.tgz",
+ "integrity": "sha1-2hkZgci41xPKARXViYzzl8Lw3dA="
+ },
+ "ee-first": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
+ "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
+ },
+ "encodeurl": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.1.tgz",
+ "integrity": "sha1-eePVhlU0aQn+bw9Fpd5oEDspTSA="
+ },
+ "env-cmd": {
+ "version": "5.1.0",
+ "resolved": "https://registry.npmjs.org/env-cmd/-/env-cmd-5.1.0.tgz",
+ "integrity": "sha1-AjbbOTw/AzAFIE/NCpLuQHI6nJ4=",
+ "requires": {
+ "cross-spawn": "5.1.0"
+ }
+ },
+ "escape-html": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
+ "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg="
+ },
+ "esprima": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.0.0.tgz",
+ "integrity": "sha1-U88kes2ncxPlUcOqLnM0LT+099k="
+ },
+ "etag": {
+ "version": "1.8.0",
+ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.0.tgz",
+ "integrity": "sha1-b2Ma7zNtbEY2K1F2QETOIWvjwFE="
+ },
+ "express": {
+ "version": "4.15.3",
+ "resolved": "https://registry.npmjs.org/express/-/express-4.15.3.tgz",
+ "integrity": "sha1-urZdDwOqgMNYQIly/HAPkWlEtmI=",
+ "requires": {
+ "accepts": "1.3.3",
+ "array-flatten": "1.1.1",
+ "content-disposition": "0.5.2",
+ "content-type": "1.0.2",
+ "cookie": "0.3.1",
+ "cookie-signature": "1.0.6",
+ "debug": "2.6.7",
+ "depd": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz",
+ "encodeurl": "1.0.1",
+ "escape-html": "1.0.3",
+ "etag": "1.8.0",
+ "finalhandler": "1.0.3",
+ "fresh": "0.5.0",
+ "merge-descriptors": "1.0.1",
+ "methods": "1.1.2",
+ "on-finished": "2.3.0",
+ "parseurl": "1.3.1",
+ "path-to-regexp": "0.1.7",
+ "proxy-addr": "1.1.4",
+ "qs": "https://registry.npmjs.org/qs/-/qs-6.4.0.tgz",
+ "range-parser": "1.2.0",
+ "send": "0.15.3",
+ "serve-static": "1.12.3",
+ "setprototypeof": "1.0.3",
+ "statuses": "1.3.1",
+ "type-is": "1.6.15",
+ "utils-merge": "1.0.0",
+ "vary": "1.1.1"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "2.6.7",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.7.tgz",
+ "integrity": "sha1-krrR9tBbu2u6Isyoi80OyJTChh4=",
+ "requires": {
+ "ms": "2.0.0"
+ }
+ },
+ "ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+ }
+ }
+ },
+ "finalhandler": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.0.3.tgz",
+ "integrity": "sha1-70fneVDpmXgOhgIqVg4yF+DQzIk=",
+ "requires": {
+ "debug": "2.6.7",
+ "encodeurl": "1.0.1",
+ "escape-html": "1.0.3",
+ "on-finished": "2.3.0",
+ "parseurl": "1.3.1",
+ "statuses": "1.3.1",
+ "unpipe": "1.0.0"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "2.6.7",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.7.tgz",
+ "integrity": "sha1-krrR9tBbu2u6Isyoi80OyJTChh4=",
+ "requires": {
+ "ms": "2.0.0"
+ }
+ },
+ "ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+ }
+ }
+ },
+ "forwarded": {
+ "version": "0.1.0",
+ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz",
+ "integrity": "sha1-Ge+YdMSuHCl7zweP3mOgm2aoQ2M="
+ },
+ "fresh": {
+ "version": "0.5.0",
+ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.0.tgz",
+ "integrity": "sha1-9HTKXmqSRtb9jglTz6m5yAWvp44="
+ },
+ "generate-function": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz",
+ "integrity": "sha1-aFj+fAlpt9TpCTM3ZHrHn2DfvnQ="
+ },
+ "generic-pool": {
+ "version": "3.1.7",
+ "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.1.7.tgz",
+ "integrity": "sha1-2sIrLHp6BOQXMvfY0tJaMDyI9mI="
+ },
+ "http": {
+ "version": "0.0.0",
+ "resolved": "https://registry.npmjs.org/http/-/http-0.0.0.tgz",
+ "integrity": "sha1-huYybSnF0Dnen6xYSkVon5KfT3I="
+ },
+ "http-errors": {
+ "version": "1.6.1",
+ "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.1.tgz",
+ "integrity": "sha1-X4uO2YrKVFZWv1cplzh/kEpyIlc=",
+ "requires": {
+ "depd": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz",
+ "inherits": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+ "setprototypeof": "1.0.3",
+ "statuses": "1.3.1"
+ }
+ },
+ "iconv-lite": {
+ "version": "0.4.18",
+ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.18.tgz",
+ "integrity": "sha512-sr1ZQph3UwHTR0XftSbK85OvBbxe/abLGzEnPENCQwmHf7sck8Oyu4ob3LgBxWWxRoM+QszeUyl7jbqapu2TqA=="
+ },
+ "inflection": {
+ "version": "1.10.0",
+ "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.10.0.tgz",
+ "integrity": "sha1-W//LEZetPoEFD44X4hZoCH7p6y8="
+ },
+ "inherits": {
+ "version": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+ "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
+ },
+ "ipaddr.js": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.3.0.tgz",
+ "integrity": "sha1-HgOlL9rYOou7KyXL9JmLTP/NPew="
+ },
+ "is-bluebird": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/is-bluebird/-/is-bluebird-1.0.2.tgz",
+ "integrity": "sha1-CWQ5Bg9KpBGr7hkUOoTWpVNG1uI="
+ },
+ "is-windows": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.1.tgz",
+ "integrity": "sha1-MQ23D3QtJZoWo2kgK1GvhCMzENk="
+ },
+ "isexe": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA="
+ },
+ "json-stringify-safe": {
+ "version": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
+ "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
+ },
+ "lodash": {
+ "version": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz",
+ "integrity": "sha1-eCA6TRwyiuHYbcpkYONptX9AVa4="
+ },
+ "long": {
+ "version": "3.2.0",
+ "resolved": "https://registry.npmjs.org/long/-/long-3.2.0.tgz",
+ "integrity": "sha1-2CG3E4yhy1gcFymQ7xTbIAtcR0s="
+ },
+ "lru-cache": {
+ "version": "4.1.1",
+ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.1.tgz",
+ "integrity": "sha512-q4spe4KTfsAS1SUHLO0wz8Qiyf1+vMIAgpRYioFYDMNqKfHQbg+AVDH3i4fvpl71/P1L0dBl+fQi+P37UYf0ew==",
+ "requires": {
+ "pseudomap": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
+ "yallist": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz"
+ }
+ },
+ "media-typer": {
+ "version": "0.3.0",
+ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
+ "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
+ },
+ "merge-descriptors": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
+ "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E="
+ },
+ "methods": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
+ "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4="
+ },
+ "mime": {
+ "version": "1.3.4",
+ "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz",
+ "integrity": "sha1-EV+eO2s9rylZmDyzjxSaLUDrXVM="
+ },
+ "mime-db": {
+ "version": "https://registry.npmjs.org/mime-db/-/mime-db-1.27.0.tgz",
+ "integrity": "sha1-gg9XIpa70g7CXtVeW13oaeVDbrE="
+ },
+ "mime-types": {
+ "version": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.15.tgz",
+ "integrity": "sha1-pOv1BkCUVpI3uM9wBGd20J/JKu0=",
+ "requires": {
+ "mime-db": "https://registry.npmjs.org/mime-db/-/mime-db-1.27.0.tgz"
+ }
+ },
+ "moment": {
+ "version": "2.18.1",
+ "resolved": "https://registry.npmjs.org/moment/-/moment-2.18.1.tgz",
+ "integrity": "sha1-w2GT3Tzhwu7SrbfIAtu8d6gbHA8="
+ },
+ "moment-timezone": {
+ "version": "0.5.13",
+ "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.13.tgz",
+ "integrity": "sha1-mc5cfYJyYusPH3AgRBd/YHRde5A=",
+ "requires": {
+ "moment": "2.18.1"
+ }
+ },
+ "ms": {
+ "version": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz",
+ "integrity": "sha1-riXPJRKziFodldfwN4aNhDESR2U="
+ },
+ "mysql2": {
+ "version": "1.3.6",
+ "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-1.3.6.tgz",
+ "integrity": "sha1-1cFJF94LFYAabBiDWZvPb7J8Gh4=",
+ "requires": {
+ "cardinal": "1.0.0",
+ "denque": "1.1.1",
+ "generate-function": "2.0.0",
+ "iconv-lite": "0.4.18",
+ "long": "3.2.0",
+ "lru-cache": "4.1.1",
+ "named-placeholders": "1.1.1",
+ "object-assign": "4.1.1",
+ "readable-stream": "2.3.2",
+ "safe-buffer": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.0.1.tgz",
+ "seq-queue": "0.0.5",
+ "sqlstring": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.2.0.tgz"
+ },
+ "dependencies": {
+ "isarray": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+ "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+ },
+ "lru-cache": {
+ "version": "4.1.1",
+ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.1.tgz",
+ "integrity": "sha512-q4spe4KTfsAS1SUHLO0wz8Qiyf1+vMIAgpRYioFYDMNqKfHQbg+AVDH3i4fvpl71/P1L0dBl+fQi+P37UYf0ew==",
+ "requires": {
+ "pseudomap": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
+ "yallist": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz"
+ }
+ },
+ "readable-stream": {
+ "version": "2.3.2",
+ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.2.tgz",
+ "integrity": "sha1-WgTfBeT1f+Pw3Gj90R3FyXx+b00=",
+ "requires": {
+ "core-util-is": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+ "inherits": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+ "isarray": "1.0.0",
+ "process-nextick-args": "1.0.7",
+ "safe-buffer": "5.1.1",
+ "string_decoder": "1.0.3",
+ "util-deprecate": "1.0.2"
+ },
+ "dependencies": {
+ "safe-buffer": {
+ "version": "5.1.1",
+ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
+ "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
+ }
+ }
+ },
+ "string_decoder": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz",
+ "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==",
+ "requires": {
+ "safe-buffer": "5.1.1"
+ },
+ "dependencies": {
+ "safe-buffer": {
+ "version": "5.1.1",
+ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
+ "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
+ }
+ }
+ }
+ }
+ },
+ "named-placeholders": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.1.tgz",
+ "integrity": "sha1-O3oNJiA910s6nfTJz7gnsvuQfmQ=",
+ "requires": {
+ "lru-cache": "2.5.0"
+ },
+ "dependencies": {
+ "lru-cache": {
+ "version": "2.5.0",
+ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.5.0.tgz",
+ "integrity": "sha1-2COIrpyWC+y+oMc7uet5tsbOmus="
+ }
+ }
+ },
+ "negotiator": {
+ "version": "0.6.1",
+ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz",
+ "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk="
+ },
+ "node-loggly-bulk": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/node-loggly-bulk/-/node-loggly-bulk-2.0.0.tgz",
+ "integrity": "sha1-5M+UOY15jroSQwUsLybX3ZuSEE0=",
+ "requires": {
+ "json-stringify-safe": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
+ "request": "2.81.0",
+ "timespan": "2.3.0"
+ }
+ },
+ "object-assign": {
+ "version": "4.1.1",
+ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
+ "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
+ },
+ "on-finished": {
+ "version": "2.3.0",
+ "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+ "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=",
+ "requires": {
+ "ee-first": "1.1.1"
+ }
+ },
+ "parseurl": {
+ "version": "1.3.1",
+ "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.1.tgz",
+ "integrity": "sha1-yKuMkiO6NIiKpkopeyiFO+wY2lY="
+ },
+ "path-to-regexp": {
+ "version": "0.1.7",
+ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
+ "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
+ },
+ "process-nextick-args": {
+ "version": "1.0.7",
+ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz",
+ "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M="
+ },
+ "proxy-addr": {
+ "version": "1.1.4",
+ "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.1.4.tgz",
+ "integrity": "sha1-J+VF9pYKRKYn2bREZ+NcG2tM4vM=",
+ "requires": {
+ "forwarded": "0.1.0",
+ "ipaddr.js": "1.3.0"
+ }
+ },
+ "pseudomap": {
+ "version": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
+ "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
+ },
+ "punycode": {
+ "version": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
+ "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
+ },
+ "qs": {
+ "version": "https://registry.npmjs.org/qs/-/qs-6.4.0.tgz",
+ "integrity": "sha1-E+JtKK1rD/qpExLNO/cI7TUecjM="
+ },
+ "range-parser": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz",
+ "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4="
+ },
+ "redeyed": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/redeyed/-/redeyed-1.0.1.tgz",
+ "integrity": "sha1-6WwZO0DAgWsArshCaY5hGF5VSYo=",
+ "requires": {
+ "esprima": "3.0.0"
+ }
+ },
+ "request": {
+ "version": "2.81.0",
+ "resolved": "https://registry.npmjs.org/request/-/request-2.81.0.tgz",
+ "integrity": "sha1-xpKJRqDgbF+Nb4qTM0af/aRimKA=",
+ "requires": {
+ "aws-sign2": "0.6.0",
+ "aws4": "1.6.0",
+ "caseless": "0.12.0",
+ "combined-stream": "1.0.5",
+ "extend": "3.0.1",
+ "forever-agent": "0.6.1",
+ "form-data": "2.1.4",
+ "har-validator": "4.2.1",
+ "hawk": "3.1.3",
+ "http-signature": "1.1.1",
+ "is-typedarray": "1.0.0",
+ "isstream": "0.1.2",
+ "json-stringify-safe": "5.0.1",
+ "mime-types": "2.1.15",
+ "oauth-sign": "0.8.2",
+ "performance-now": "0.2.0",
+ "qs": "6.4.0",
+ "safe-buffer": "5.1.1",
+ "stringstream": "0.0.5",
+ "tough-cookie": "2.3.2",
+ "tunnel-agent": "0.6.0",
+ "uuid": "3.1.0"
+ },
+ "dependencies": {
+ "ajv": {
+ "version": "4.11.8",
+ "resolved": "https://registry.npmjs.org/ajv/-/ajv-4.11.8.tgz",
+ "integrity": "sha1-gv+wKynmYq5TvcIK8VlHcGc5xTY=",
+ "requires": {
+ "co": "4.6.0",
+ "json-stable-stringify": "1.0.1"
+ }
+ },
+ "asn1": {
+ "version": "0.2.3",
+ "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz",
+ "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y="
+ },
+ "assert-plus": {
+ "version": "0.2.0",
+ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz",
+ "integrity": "sha1-104bh+ev/A24qttwIfP+SBAasjQ="
+ },
+ "asynckit": {
+ "version": "0.4.0",
+ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+ "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
+ },
+ "aws-sign2": {
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz",
+ "integrity": "sha1-FDQt0428yU0OW4fXY81jYSwOeU8="
+ },
+ "aws4": {
+ "version": "1.6.0",
+ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz",
+ "integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4="
+ },
+ "bcrypt-pbkdf": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz",
+ "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=",
+ "optional": true,
+ "requires": {
+ "tweetnacl": "0.14.5"
+ }
+ },
+ "boom": {
+ "version": "2.10.1",
+ "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz",
+ "integrity": "sha1-OciRjO/1eZ+D+UkqhI9iWt0Mdm8=",
+ "requires": {
+ "hoek": "2.16.3"
+ }
+ },
+ "caseless": {
+ "version": "0.12.0",
+ "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
+ "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
+ },
+ "co": {
+ "version": "4.6.0",
+ "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
+ "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ="
+ },
+ "combined-stream": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz",
+ "integrity": "sha1-k4NwpXtKUd6ix3wV1cX9+JUWQAk=",
+ "requires": {
+ "delayed-stream": "1.0.0"
+ }
+ },
+ "cryptiles": {
+ "version": "2.0.5",
+ "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz",
+ "integrity": "sha1-O9/s3GCBR8HGcgL6KR59ylnqo7g=",
+ "requires": {
+ "boom": "2.10.1"
+ }
+ },
+ "dashdash": {
+ "version": "1.14.1",
+ "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
+ "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
+ "requires": {
+ "assert-plus": "1.0.0"
+ },
+ "dependencies": {
+ "assert-plus": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
+ "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
+ }
+ }
+ },
+ "delayed-stream": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+ "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
+ },
+ "ecc-jsbn": {
+ "version": "0.1.1",
+ "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz",
+ "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=",
+ "optional": true,
+ "requires": {
+ "jsbn": "0.1.1"
+ }
+ },
+ "extend": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz",
+ "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ="
+ },
+ "extsprintf": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz",
+ "integrity": "sha1-4QgOBljjALBilJkMxw4VAiNf1VA="
+ },
+ "forever-agent": {
+ "version": "0.6.1",
+ "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
+ "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
+ },
+ "form-data": {
+ "version": "2.1.4",
+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.1.4.tgz",
+ "integrity": "sha1-M8GDrPGTJ27KqYFDpp6Uv+4XUNE=",
+ "requires": {
+ "asynckit": "0.4.0",
+ "combined-stream": "1.0.5",
+ "mime-types": "2.1.15"
+ }
+ },
+ "getpass": {
+ "version": "0.1.7",
+ "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
+ "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
+ "requires": {
+ "assert-plus": "1.0.0"
+ },
+ "dependencies": {
+ "assert-plus": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
+ "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
+ }
+ }
+ },
+ "har-schema": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-1.0.5.tgz",
+ "integrity": "sha1-0mMTX0MwfALGAq/I/pWXDAFRNp4="
+ },
+ "har-validator": {
+ "version": "4.2.1",
+ "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-4.2.1.tgz",
+ "integrity": "sha1-M0gdDxu/9gDdID11gSpqX7oALio=",
+ "requires": {
+ "ajv": "4.11.8",
+ "har-schema": "1.0.5"
+ }
+ },
+ "hawk": {
+ "version": "3.1.3",
+ "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz",
+ "integrity": "sha1-B4REvXwWQLD+VA0sm3PVlnjo4cQ=",
+ "requires": {
+ "boom": "2.10.1",
+ "cryptiles": "2.0.5",
+ "hoek": "2.16.3",
+ "sntp": "1.0.9"
+ }
+ },
+ "hoek": {
+ "version": "2.16.3",
+ "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz",
+ "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0="
+ },
+ "http-signature": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz",
+ "integrity": "sha1-33LiZwZs0Kxn+3at+OE0qPvPkb8=",
+ "requires": {
+ "assert-plus": "0.2.0",
+ "jsprim": "1.4.0",
+ "sshpk": "1.13.1"
+ }
+ },
+ "is-typedarray": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
+ "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
+ },
+ "isstream": {
+ "version": "0.1.2",
+ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
+ "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
+ },
+ "jsbn": {
+ "version": "0.1.1",
+ "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
+ "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=",
+ "optional": true
+ },
+ "json-schema": {
+ "version": "0.2.3",
+ "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
+ "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
+ },
+ "json-stable-stringify": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz",
+ "integrity": "sha1-mnWdOcXy/1A/1TAGRu1EX4jE+a8=",
+ "requires": {
+ "jsonify": "0.0.0"
+ }
+ },
+ "json-stringify-safe": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
+ "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
+ },
+ "jsonify": {
+ "version": "0.0.0",
+ "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz",
+ "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM="
+ },
+ "jsprim": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.0.tgz",
+ "integrity": "sha1-o7h+QCmNjDgFUtjMdiigu5WiKRg=",
+ "requires": {
+ "assert-plus": "1.0.0",
+ "extsprintf": "1.0.2",
+ "json-schema": "0.2.3",
+ "verror": "1.3.6"
+ },
+ "dependencies": {
+ "assert-plus": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
+ "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
+ }
+ }
+ },
+ "mime-db": {
+ "version": "1.27.0",
+ "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.27.0.tgz",
+ "integrity": "sha1-gg9XIpa70g7CXtVeW13oaeVDbrE="
+ },
+ "mime-types": {
+ "version": "2.1.15",
+ "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.15.tgz",
+ "integrity": "sha1-pOv1BkCUVpI3uM9wBGd20J/JKu0=",
+ "requires": {
+ "mime-db": "1.27.0"
+ }
+ },
+ "oauth-sign": {
+ "version": "0.8.2",
+ "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz",
+ "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM="
+ },
+ "performance-now": {
+ "version": "0.2.0",
+ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-0.2.0.tgz",
+ "integrity": "sha1-M+8wxcd9TqIcWlOGnZG1bY8lVeU="
+ },
+ "punycode": {
+ "version": "1.4.1",
+ "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
+ "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
+ },
+ "qs": {
+ "version": "6.4.0",
+ "resolved": "https://registry.npmjs.org/qs/-/qs-6.4.0.tgz",
+ "integrity": "sha1-E+JtKK1rD/qpExLNO/cI7TUecjM="
+ },
+ "safe-buffer": {
+ "version": "5.1.1",
+ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
+ "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
+ },
+ "sntp": {
+ "version": "1.0.9",
+ "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz",
+ "integrity": "sha1-ZUEYTMkK7qbG57NeJlkIJEPGYZg=",
+ "requires": {
+ "hoek": "2.16.3"
+ }
+ },
+ "sshpk": {
+ "version": "1.13.1",
+ "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz",
+ "integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=",
+ "requires": {
+ "asn1": "0.2.3",
+ "assert-plus": "1.0.0",
+ "bcrypt-pbkdf": "1.0.1",
+ "dashdash": "1.14.1",
+ "ecc-jsbn": "0.1.1",
+ "getpass": "0.1.7",
+ "jsbn": "0.1.1",
+ "tweetnacl": "0.14.5"
+ },
+ "dependencies": {
+ "assert-plus": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
+ "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
+ }
+ }
+ },
+ "stringstream": {
+ "version": "0.0.5",
+ "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz",
+ "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg="
+ },
+ "tough-cookie": {
+ "version": "2.3.2",
+ "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.2.tgz",
+ "integrity": "sha1-8IH3bkyFcg5sN6X6ztc3FQ2EByo=",
+ "requires": {
+ "punycode": "1.4.1"
+ }
+ },
+ "tunnel-agent": {
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
+ "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
+ "requires": {
+ "safe-buffer": "5.1.1"
+ }
+ },
+ "tweetnacl": {
+ "version": "0.14.5",
+ "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
+ "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
+ "optional": true
+ },
+ "uuid": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz",
+ "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g=="
+ },
+ "verror": {
+ "version": "1.3.6",
+ "resolved": "https://registry.npmjs.org/verror/-/verror-1.3.6.tgz",
+ "integrity": "sha1-z/XfEpRtKX0rqu+qJoniW+AcAFw=",
+ "requires": {
+ "extsprintf": "1.0.2"
+ }
+ }
+ }
+ },
+ "request-promise": {
+ "version": "4.2.1",
+ "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.1.tgz",
+ "integrity": "sha1-fuxWyJMXqCLL/qmbA5zlQ8LhX2c=",
+ "requires": {
+ "bluebird": "3.5.0",
+ "request-promise-core": "1.1.1",
+ "stealthy-require": "1.1.1",
+ "tough-cookie": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.2.tgz"
+ },
+ "dependencies": {
+ "request-promise-core": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz",
+ "integrity": "sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=",
+ "requires": {
+ "lodash": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz"
+ }
+ }
+ }
+ },
+ "retry-as-promised": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-2.2.0.tgz",
+ "integrity": "sha1-sEY9f9PPWy/tZFAKtui4pJxbjmw=",
+ "requires": {
+ "bluebird": "3.5.0",
+ "cross-env": "3.2.4",
+ "debug": "https://registry.npmjs.org/debug/-/debug-2.6.1.tgz"
+ }
+ },
+ "safe-buffer": {
+ "version": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.0.1.tgz",
+ "integrity": "sha1-0mPKVGls2KMGtcplUekt5XkY++c="
+ },
+ "semver": {
+ "version": "5.3.0",
+ "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz",
+ "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8="
+ },
+ "send": {
+ "version": "0.15.3",
+ "resolved": "https://registry.npmjs.org/send/-/send-0.15.3.tgz",
+ "integrity": "sha1-UBP5+ZAj31DRvZiSwZ4979HVMwk=",
+ "requires": {
+ "debug": "2.6.7",
+ "depd": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz",
+ "destroy": "1.0.4",
+ "encodeurl": "1.0.1",
+ "escape-html": "1.0.3",
+ "etag": "1.8.0",
+ "fresh": "0.5.0",
+ "http-errors": "1.6.1",
+ "mime": "1.3.4",
+ "ms": "2.0.0",
+ "on-finished": "2.3.0",
+ "range-parser": "1.2.0",
+ "statuses": "1.3.1"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "2.6.7",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.7.tgz",
+ "integrity": "sha1-krrR9tBbu2u6Isyoi80OyJTChh4=",
+ "requires": {
+ "ms": "2.0.0"
+ }
+ },
+ "ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+ }
+ }
+ },
+ "seq-queue": {
+ "version": "0.0.5",
+ "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz",
+ "integrity": "sha1-1WgS4cAXpuTnw+Ojeh2m143TyT4="
+ },
+ "sequelize": {
+ "version": "4.3.2",
+ "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-4.3.2.tgz",
+ "integrity": "sha1-FHrRnW6lk++YME6MsLimkrS50aM=",
+ "requires": {
+ "bluebird": "3.5.0",
+ "cls-bluebird": "2.0.1",
+ "debug": "https://registry.npmjs.org/debug/-/debug-2.6.1.tgz",
+ "depd": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz",
+ "dottie": "2.0.0",
+ "env-cmd": "5.1.0",
+ "generic-pool": "3.1.7",
+ "inflection": "1.10.0",
+ "lodash": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz",
+ "moment": "2.18.1",
+ "moment-timezone": "0.5.13",
+ "retry-as-promised": "2.2.0",
+ "semver": "5.3.0",
+ "terraformer-wkt-parser": "1.1.2",
+ "toposort-class": "1.0.1",
+ "uuid": "https://registry.npmjs.org/uuid/-/uuid-3.0.1.tgz",
+ "validator": "6.3.0",
+ "wkx": "0.4.1"
+ }
+ },
+ "serve-static": {
+ "version": "1.12.3",
+ "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.12.3.tgz",
+ "integrity": "sha1-n0uhni8wMMVH+K+ZEHg47DjVseI=",
+ "requires": {
+ "encodeurl": "1.0.1",
+ "escape-html": "1.0.3",
+ "parseurl": "1.3.1",
+ "send": "0.15.3"
+ }
+ },
+ "setprototypeof": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz",
+ "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ="
+ },
+ "shebang-command": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz",
+ "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=",
+ "requires": {
+ "shebang-regex": "1.0.0"
+ }
+ },
+ "shebang-regex": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz",
+ "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM="
+ },
+ "shimmer": {
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.1.0.tgz",
+ "integrity": "sha1-l9c3cTf/u6tCVSLkKf4KqJpIizU="
+ },
+ "sqlstring": {
+ "version": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.2.0.tgz",
+ "integrity": "sha1-wxNcTqirzX5+50GklmqJHYak8ZE="
+ },
+ "statuses": {
+ "version": "1.3.1",
+ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz",
+ "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4="
+ },
+ "stealthy-require": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
+ "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
+ },
+ "terraformer": {
+ "version": "1.0.8",
+ "resolved": "https://registry.npmjs.org/terraformer/-/terraformer-1.0.8.tgz",
+ "integrity": "sha1-UeCtiXRvzyFh3G9lqnDkI3fItZM=",
+ "requires": {
+ "@types/geojson": "1.0.2"
+ }
+ },
+ "terraformer-wkt-parser": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/terraformer-wkt-parser/-/terraformer-wkt-parser-1.1.2.tgz",
+ "integrity": "sha1-M2oMj8gglKWv+DKI9prt7NNpvww=",
+ "requires": {
+ "terraformer": "1.0.8"
+ }
+ },
+ "timespan": {
+ "version": "2.3.0",
+ "resolved": "https://registry.npmjs.org/timespan/-/timespan-2.3.0.tgz",
+ "integrity": "sha1-SQLOBAvRPYRcj1myfp1ZutbzmSk="
+ },
+ "toposort-class": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz",
+ "integrity": "sha1-f/0feMi+KMO6Rc1OGj9e4ZO9mYg="
+ },
+ "tough-cookie": {
+ "version": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.2.tgz",
+ "integrity": "sha1-8IH3bkyFcg5sN6X6ztc3FQ2EByo=",
+ "requires": {
+ "punycode": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz"
+ }
+ },
+ "type-is": {
+ "version": "1.6.15",
+ "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.15.tgz",
+ "integrity": "sha1-yrEPtJCeRByChC6v4a1kbIGARBA=",
+ "requires": {
+ "media-typer": "0.3.0",
+ "mime-types": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.15.tgz"
+ }
+ },
+ "unpipe": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
+ "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
+ },
+ "util-deprecate": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
+ "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
+ },
+ "utils-merge": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz",
+ "integrity": "sha1-ApT7kiu5N1FTVBxPcJYjHyh8ivg="
+ },
+ "uuid": {
+ "version": "https://registry.npmjs.org/uuid/-/uuid-3.0.1.tgz",
+ "integrity": "sha1-ZUS7ot/ajBzxfmKaOjBeK7H+5sE="
+ },
+ "validator": {
+ "version": "6.3.0",
+ "resolved": "https://registry.npmjs.org/validator/-/validator-6.3.0.tgz",
+ "integrity": "sha1-R84j7Y1Ord+p1LjvAHG2zxB418g="
+ },
+ "vary": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.1.tgz",
+ "integrity": "sha1-Z1Neu2lMHVIldFeYRmUyP1h+jTc="
+ },
+ "webstomp-client": {
+ "version": "1.0.6",
+ "resolved": "https://registry.npmjs.org/webstomp-client/-/webstomp-client-1.0.6.tgz",
+ "integrity": "sha1-n+FifdSII82l9K3p00ZyrDPIN+A="
+ },
+ "which": {
+ "version": "1.2.14",
+ "resolved": "https://registry.npmjs.org/which/-/which-1.2.14.tgz",
+ "integrity": "sha1-mofEN48D6CfOyvGs31bHNsAcFOU=",
+ "requires": {
+ "isexe": "2.0.0"
+ }
+ },
+ "winston": {
+ "version": "2.3.1",
+ "resolved": "https://registry.npmjs.org/winston/-/winston-2.3.1.tgz",
+ "integrity": "sha1-C0hCDZeMAYBM8CMLZIhhWYIloRk=",
+ "requires": {
+ "async": "1.0.0",
+ "colors": "1.0.3",
+ "cycle": "1.0.3",
+ "eyes": "0.1.8",
+ "isstream": "0.1.2",
+ "stack-trace": "0.0.10"
+ },
+ "dependencies": {
+ "async": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz",
+ "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k="
+ },
+ "colors": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
+ "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs="
+ },
+ "cycle": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz",
+ "integrity": "sha1-IegLK+hYD5i0aPN5QwZisEbDStI="
+ },
+ "eyes": {
+ "version": "0.1.8",
+ "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz",
+ "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A="
+ },
+ "isstream": {
+ "version": "0.1.2",
+ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
+ "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
+ },
+ "stack-trace": {
+ "version": "0.0.10",
+ "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz",
+ "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA="
+ }
+ }
+ },
+ "winston-loggly-bulk": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/winston-loggly-bulk/-/winston-loggly-bulk-2.0.0.tgz",
+ "integrity": "sha1-olOCeBihS7b+CwbMHCLY0+19EoA=",
+ "requires": {
+ "node-loggly-bulk": "2.0.0",
+ "winston": "2.3.1"
+ }
+ },
+ "wkx": {
+ "version": "0.4.1",
+ "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.1.tgz",
+ "integrity": "sha1-L8FxtenLVcYlb+9L3h8hvkE77+4=",
+ "requires": {
+ "@types/node": "6.0.82"
+ }
+ },
+ "yallist": {
+ "version": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
+ "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI="
+ }
+ }
+}
diff --git a/package.json b/package.json
index f75cd0c..31235cd 100644
--- a/package.json
+++ b/package.json
@@ -6,16 +6,15 @@
"dependencies": {
"amqplib": "^0.5.1",
"bluebird": "^3.5.0",
- "express": "^4.15.2",
+ "express": "^4.15.3",
"http": "^0.0.0",
- "mysql": "^2.13.0",
+ "mysql2": "^1.3.6",
"request": "^2.81.0",
- "request-promise": "^4.2.0",
- "sequelize": "^3.30.4",
- "sleep-promise": "^2.0.0",
+ "request-promise": "^4.2.1",
+ "sequelize": "^4.3.2",
"webstomp-client": "^1.0.6",
"winston": "^2.3.1",
- "winston-loggly-bulk": "^1.4.2"
+ "winston-loggly-bulk": "^2.0.0"
},
"devDependencies": {},
"scripts": {
diff --git a/server.js b/server.js
index 3c0cee1..3e0ce17 100644
--- a/server.js
+++ b/server.js
@@ -8,36 +8,16 @@ const amqp = require("amqplib"),
Seq = require("sequelize"),
request = require("request-promise"),
express = require("express"),
- http = require("http"),
- sleep = require("sleep-promise"),
- api = require("../orm/api");
+ http = require("http");
const PORT = parseInt(process.env.PORT) || 8880,
- KEY_TYPE = process.env.KEY_TYPE || "crunch", // backwards compat
DATABASE_URI = process.env.DATABASE_URI,
DATABASE_BRAWL_URI = process.env.DATABASE_BRAWL_URI,
DATABASE_TOURNAMENT_URI = process.env.DATABASE_TOURNAMENT_URI,
RABBITMQ_URI = process.env.RABBITMQ_URI || "amqp://localhost",
LOGGLY_TOKEN = process.env.LOGGLY_TOKEN,
BRAWL = process.env.BRAWL != "false",
- TOURNAMENT = process.env.TOURNAMENT != "false",
- REGIONS = (process.env.REGIONS || "na,eu,sg,sa,ea").split(","),
- TOURNAMENT_REGIONS = (process.env.TOURNAMENT_REGIONS || "tournament-na," +
- "tournament-eu,tournament-sg,tournament-sa,tournament-ea").split(","),
- GRABSTART = process.env.GRABSTART || "2017-02-14T00:00:00Z",
- BRAWL_RETENTION_DAYS = parseInt(process.env.BRAWL_RETENTION_DAYS) || 3,
- PLAYER_PROCESS_QUEUE = process.env.PLAYER_PROCESS_QUEUE || "process",
- PLAYER_TOURNAMENT_PROCESS_QUEUE = process.env.PLAYER_TOURNAMENT_PROCESS_QUEUE || "process_tournament",
- GRAB_QUEUE = process.env.GRAB_QUEUE || "grab",
- GRAB_BRAWL_QUEUE = process.env.GRAB_BRAWL_QUEUE || "grab_brawl",
- GRAB_TOURNAMENT_QUEUE = process.env.GRAB_TOURNAMENT_QUEUE || "grab_tournament",
- CRUNCH_QUEUE = process.env.CRUNCH_QUEUE || "crunch",
- CRUNCH_TOURNAMENT_QUEUE = process.env.CRUNCH_TOURNAMENT_QUEUE || "crunch_tournament",
- ANALYZE_QUEUE = process.env.ANALYZE_QUEUE || "analyze",
- ANALYZE_TOURNAMENT_QUEUE = process.env.ANALYZE_TOURNAMENT_QUEUE || "analyze_tournament",
- SAMPLE_QUEUE = process.env.SAMPLE_QUEUE || "sample",
- SAMPLE_TOURNAMENT_QUEUE = process.env.SAMPLE_TOURNAMENT_QUEUE || "sample_tournament",
- SHOVEL_SIZE = parseInt(process.env.SHOVEL_SIZE) || 1000;
+ TOURNAMENT = process.env.TOURNAMENT != "false";
const app = express(),
server = http.Server(app),
@@ -49,7 +29,17 @@ const app = express(),
})
]
});
+
+global.logger = logger;
+
+const Crunch = require("./service_crunch.js"),
+ Analyze = require("./service_analyze.js"),
+ Grab = require("./service_grab.js"),
+ Telemetry = require("./service_telemetry.js");
+
let rabbit, ch,
+ cruncher = new Crunch(), analyzer = new Analyze(),
+ grabber = new Grab(), telemeter = new Telemetry(),
seq, seqBrawl, seqTournament,
model, modelBrawl, modelTournament;
@@ -62,669 +52,41 @@ if (LOGGLY_TOKEN)
json: true
});
-// connect to broker, retrying forever
-(async () => {
- while (true) {
- try {
- seq = new Seq(DATABASE_URI, { logging: () => {} });
- if (BRAWL)
- seqBrawl = new Seq(DATABASE_BRAWL_URI, { logging: () => {} });
- if (TOURNAMENT)
- seqTournament = new Seq(DATABASE_TOURNAMENT_URI, { logging: () => {} });
- rabbit = await amqp.connect(RABBITMQ_URI);
- ch = await rabbit.createChannel();
- await ch.assertQueue(GRAB_QUEUE, {durable: true});
- await ch.assertQueue(GRAB_BRAWL_QUEUE, {durable: true});
- await ch.assertQueue(GRAB_TOURNAMENT_QUEUE, {durable: true});
- await ch.assertQueue(PLAYER_PROCESS_QUEUE, {durable: true});
- await ch.assertQueue(PLAYER_TOURNAMENT_PROCESS_QUEUE, {durable: true});
- await ch.assertQueue(CRUNCH_QUEUE, {durable: true});
- await ch.assertQueue(CRUNCH_TOURNAMENT_QUEUE, {durable: true});
- await ch.assertQueue(SAMPLE_QUEUE, {durable: true});
- await ch.assertQueue(SAMPLE_TOURNAMENT_QUEUE, {durable: true});
- break;
- } catch (err) {
- logger.error("Error connecting", err);
- await sleep(5000);
- }
- }
- model = require("../orm/model")(seq, Seq);
- if (BRAWL)
- modelBrawl = require("../orm/model")(seqBrawl, Seq);
- if (TOURNAMENT)
- modelTournament = require("../orm/model")(seqTournament, Seq);
-})();
-
-server.listen(PORT);
-app.use(express.static("assets"));
-
-// convenience mapping
-// `category`: "regular", "brawl", "tournament"
-// TODO refactor this
-function gameModesForCategory(category) {
- switch (category) {
- case "regular": return "casual,ranked";
- case "brawl": return "casual_aral,blitz_pvp_ranked";
- case "tournament": return "private,private_party_draft_match";
- default: logger.error("unsupported game mode category",
- { category: category });
- }
-}
-
-// convenience function to sort into the right grab queue
-function grabQueueForCategory(category) {
- switch (category) {
- case "regular": return GRAB_QUEUE;
- case "brawl": return GRAB_BRAWL_QUEUE;
- case "tournament": return GRAB_TOURNAMENT_QUEUE;
- default: logger.error("unsupported game mode category",
- { category: category });
- }
-}
-
-function sampleQueueForCategory(category) {
- switch (category) {
- case "regular": return SAMPLE_QUEUE;
- case "tournament": return SAMPLE_TOURNAMENT_QUEUE;
- default: logger.error("unsupported game mode category",
- { category: category });
- }
-}
-
-function processQueueForCategory(category) {
- switch (category) {
- case "regular": return PLAYER_PROCESS_QUEUE;
- case "tournament": return PLAYER_TOURNAMENT_PROCESS_QUEUE;
- default: logger.error("unsupported game mode category",
- { category: category });
- }
-}
-
-function crunchQueueForCategory(category) {
- switch (category) {
- case "regular": return CRUNCH_QUEUE;
- case "tournament": return CRUNCH_TOURNAMENT_QUEUE;
- default: logger.error("unsupported game mode category",
- { category: category });
- }
-}
-
-function analyzeQueueForCategory(category) {
- switch (category) {
- case "regular": return ANALYZE_QUEUE;
- case "tournament": return ANALYZE_TOURNAMENT_QUEUE;
- default: logger.error("unsupported game mode category",
- { category: category });
- }
-}
-
-// API requires maximum time interval of 4w
-// return [payload, payload, …] with fixed createdAt
-function splitGrabs(payload, start, end) {
- let part_start = start,
- part_end = start,
- payloads = [];
- // loop forwards, adding 4w until we passed NOW
- while (part_start < end) {
- part_end = new Date(part_end.getTime() + (4 * 60*60*24*7*1000)); // add 4w
- if(part_end > end) part_end = end;
- let pl = payload;
- pl["params"]["filter[createdAt-start]"] = part_start.toISOString();
- pl["params"]["filter[createdAt-end]"] = part_end.toISOString();
- // push a deep clone or the object will have the reference to part_start
- // which leads to all start/end dates being the same (?!)
- payloads.push(JSON.parse(JSON.stringify(pl))); // JavaScript sucks.
- part_start = part_end;
- }
- return payloads;
-}
-
-// request grab jobs for a player's matches
-async function grabPlayer(name, region, last_match_created_date, id, category) {
- const start = new Date(Date.parse(last_match_created_date)),
- end = new Date(); // today
-
- const payload_template = {
- "region": region,
- "params": {
- "filter[playerIds]": id,
- "filter[gameMode]": gameModesForCategory(category),
- "sort": "createdAt"
- }
- };
- await Promise.each(splitGrabs(payload_template, start, end), async (payload) => {
- logger.info("requesting update", {
- name: name,
- region: region
- });
-
- await ch.sendToQueue(grabQueueForCategory(category),
- new Buffer(JSON.stringify(payload)), {
- persistent: true,
- type: "matches",
- headers: {
- donotify: true, // TODO backwards compat
- notify: "player." + name
- }
- });
- });
-}
-// request grab job for multiple players' matches
-async function grabPlayers(names, region, grabstart, category) {
- // TODO MadGlory change, call API in 4w distances
- const payload_template = {
- "region": region,
- "params": {
- "filter[playerNames]": names,
- "filter[gameMode]": gameModesForCategory(category),
- "sort": "createdAt"
- }
- }, now = new Date();
- logger.info("requesting triple player grab", {
- names: names,
- region: region,
- category: category
- });
-
- await Promise.each(splitGrabs(payload_template, grabstart, now), async (payload) => {
- await ch.sendToQueue(grabQueueForCategory(category),
- new Buffer(JSON.stringify(payload)), {
- persistent: true,
- type: "matches",
- headers: {
- donotify: true, // TODO
- notify: "player." + names
- }
- });
- });
-}
-
-// request grab jobs for a region's matches
-async function grabMatches(region, last_match_created_date) {
- last_match_created_date = last_match_created_date
- || new Date(Date.parse(defaultGrabstartForCategory("tournament")));
- const now = new Date();
-
- // add 1s, because createdAt-start <= x <= createdAt-end
- // so without the +1s, we'd always get the last_match_created_date match back
- last_match_created_date.setSeconds(last_match_created_date.getSeconds() + 1);
-
- const payload_template = {
- "region": region,
- "params": {
- "sort": "createdAt",
- "page[limit]": 5 // TODO maximum for `/matches`
- }
- };
- logger.info("requesting region update", { region: region });
-
- await Promise.each(splitGrabs(payload_template, last_match_created_date, now), async (payload) => {
- await ch.sendToQueue(GRAB_TOURNAMENT_QUEUE, new Buffer(JSON.stringify(payload)), {
- persistent: true,
- type: "matches"
- });
- });
-}
-
-// search for a player name in one region
-// request process for lifetime
-// return an array (JSONAPI response)
-async function searchPlayerInRegion(region, name, id, category) {
- logger.info("searching", { name: name, id: id, region: region });
- let options = {};
- if (id == undefined) options["filter[playerNames]"] = name
- else options["filter[playerIds]"] = id
-
- // players.length and page length will be 1 in 99.9999% of all cases
- // - but just in case.
- const players = await Promise.map(await api.requests("players",
- region, options, logger),
- async (player) => {
- logger.info("found", { name: name, id: id, region: region });
- // notify web that data is being loaded
- await ch.publish("amq.topic", "player." + name,
- new Buffer("search_success"));
- // send to processor, so the player is in db
- // no matter whether we find matches or not
- await ch.sendToQueue(processQueueForCategory(category),
- new Buffer(JSON.stringify(player)), {
- persistent: true, type: "player",
- headers: {
- donotify: true, // TODO
- notify: "player." + player.name
- }
- });
- return player;
- }
- );
- if (players.length == 0)
- logger.warn("not found", { name: name, id: id, region: region });
+// connect to broker and db
+let dbs = {};
+seq = new Seq(DATABASE_URI, { logging: false });
+dbs.regular = model = require("../orm/model")(seq, Seq);
+if (BRAWL) seqBrawl = new Seq(DATABASE_BRAWL_URI, { logging: false });
+if (BRAWL) dbs.brawl = modelBrawl = require("../orm/model")(seqBrawl, Seq);
+if (TOURNAMENT) seqTournament = new Seq(DATABASE_TOURNAMENT_URI, { logging: false });
+if (TOURNAMENT) dbs.tournament = modelTournament = require("../orm/model")(seqTournament, Seq);
- return players;
-}
+cruncher.setDatabases(dbs);
+grabber.setDatabases(dbs);
+analyzer.setDatabases(dbs);
+telemeter.setDatabases(dbs);
-// search for player name on all shards
-// grab player(s)
-// send a notification for results and request updates
-async function searchPlayer(name, category) {
- let found = false;
- logger.info("searching", { name: name });
- await Promise.map(regionsForCategory(category), async (region) => {
- const players = await searchPlayerInRegion(region, name, undefined, category);
- if (players.length > 0) found = true;
-
- // request grab jobs
- await Promise.map(players, (p) =>
- grabPlayer(p.name, p.shardId,
- defaultGrabstartForCategory(category), p.id, category));
+amqp.connect(RABBITMQ_URI).then(async (rabbit) => {
+ process.on("SIGINT", () => {
+ rabbit.close();
+ process.exit();
});
- // notify web
- if (!found) {
- logger.info("search failed", { name: name });
- await ch.publish("amq.topic", "player." + name,
- new Buffer("search_fail"));
- }
-}
-
-// return REGIONS based on category
-function regionsForCategory(category) {
- if (category == "regular")
- return REGIONS;
- if (category == "tournament")
- return TOURNAMENT_REGIONS;
- logger.error("unsupported category", { category: category });
-}
-
-// return the fitting db connection based on game mode
-// `category` is an enum (brawl, regular, tournament)
-function databaseForCategory(category) {
- if (category == "brawl")
- return modelBrawl;
- if (category == "tournament")
- return modelTournament;
- if (category == "regular")
- return model;
- logger.error("unsupported category", { category: category });
-}
-
-// return a default `createdAt-start` based on game mode
-function defaultGrabstartForCategory(category) {
- if (category == "brawl") {
- let past = new Date();
- past.setDate(past.getDate() - BRAWL_RETENTION_DAYS);
- return past.toISOString(); // minimum
- }
- if (category == "tournament")
- return "2017-02-12T00:00:00Z"; // all
- if (category == "regular")
- return GRABSTART; // as via env var
-}
-
-// update a player from API based on db record
-// & grab player
-async function updatePlayer(player, category) {
- // set last_update and request an update job
- // if last_update is null, we need that player's full history
- let grabstart;
- if (player.get("last_update") != null && category == "regular")
- // TODO do not fetch everything on category tournament / brawl
- grabstart = player.get("created_at");
- if (grabstart == undefined) grabstart = defaultGrabstartForCategory(category);
- else
- // add 1s, because createdAt-start <= x <= createdAt-end
- // so without the +1s, we'd always get the last_match_created_date match back
- grabstart.setSeconds(grabstart.getSeconds() + 1);
-
- const players = await searchPlayerInRegion(
- player.get("shard_id"), player.get("name"), player.get("api_id"), category);
-
- // will be the same as `player` 99.9% of the time
- // but not when the player changed their name
- // search happened by ID, so we get only 1 player back
- // but player.name != players[0].attributes.name
- // (with a name change)
- await Promise.map(players, async (p) =>
- await grabPlayer(p.name, p.shardId,
- grabstart, p.id, category));
-}
-
-// update a region from API based on db records
-async function updateRegion(region, category) {
- let grabstart;
- const last_match = await databaseForCategory(category).Participant.findOne({
- attributes: ["created_at"],
- where: { shard_id: region },
- order: [ [seq.col("created_at"), "DESC"] ]
- });
- if (last_match == null) grabstart = undefined;
- else grabstart = last_match.get("created_at");
-
- await grabMatches(region, grabstart);
-}
-// update a region's samples since samples_last_update
-async function updateSamples(region) {
- const last_update = await getKey(model,
- `samples_last_update+${region}`,
- (new Date(0)).toISOString());
-
- logger.info("updating samples", { region: region, last_update: last_update });
- await ch.sendToQueue(GRAB_QUEUE, new Buffer(
- JSON.stringify({ region: region,
- params: {
- sort: "createdAt",
- "filter[createdAt-start]": last_update
- }
- })),
- { persistent: true, type: "samples" });
-
- last_update = (new Date()).toISOString();
- await setKey(model, `samples_last_update+${region}`,
- region, last_update);
-}
-
-// upcrunch player's stats
-async function crunchPlayer(category, api_id) {
- const db = databaseForCategory(category),
- last_crunch = await db.PlayerPoint.findOne({
- attributes: ["updated_at"],
- where: { player_api_id: api_id },
- order: [ ["updated_at", "DESC"] ]
- }),
- last_crunch_date = last_crunch == undefined?
- new Date(0) : last_crunch.updated_at;
- // get all participants for this player
- const participations = await db.Participant.findAll({
- attributes: ["api_id"],
- where: {
- player_api_id: api_id,
- created_at: {
- $gt: last_crunch_date
- }
- } });
- // send everything to cruncher
- logger.info("sending participations to cruncher",
- { length: participations.length });
- await Promise.map(participations, async (p) =>
- await ch.sendToQueue(crunchQueueForCategory(category),
- new Buffer(p.api_id),
- { persistent: true, type: "player" }));
- // jobs with the type "player" won't be taken into account for global stats
- // global stats would increase on every player refresh otherwise
-}
-
-// upanalyze player's matches (TrueSkill)
-async function analyzePlayer(category, api_id) {
- const db = databaseForCategory(category),
- participations = await db.Participant.findAll({
- attributes: ["match_api_id"],
- where: {
- player_api_id: api_id,
- trueskill_mu: null // where not analyzed yet
- },
- order: [ ["created_at", "ASC"] ]
- });
- logger.info("sending matches to analyzer",
- { length: participations.length });
- await Promise.each(participations, async (p) =>
- await ch.sendToQueue(analyzeQueueForCategory(category),
- new Buffer(p.match_api_id),
- { persistent: true }));
-}
-
-// reset fame and crunch
-// TODO: incremental crunch possible?
-async function crunchTeam(team_id) {
- await ch.sendToQueue(CRUNCH_QUEUE, new Buffer(team_id),
- { persistent: true, type: "team" });
-}
-
-// crunch (force = recrunch) global stats
-async function crunchGlobal(category, force=false) {
- const db = databaseForCategory(category);
- // get lcpid from keys table
- let last_crunch_participant_id = await getKey(db,
- "global_last_crunch_participant_id", 0);
-
- if (force) {
- // refresh everything
- logger.info("deleting all global points");
- await db.GlobalPoint.destroy({ truncate: true });
- last_crunch_participant_id = await setKey(db,
- "global_last_crunch_participant_id", 0);
- }
-
- // don't load the whole Participant table at once into memory
- let participations;
-
- logger.info("loading all participations into cruncher",
- { last_crunch_participant_id: last_crunch_participant_id });
- do {
- participations = await db.Participant.findAll({
- attributes: ["api_id", "id"],
- where: {
- id: { $gt: last_crunch_participant_id }
- },
- limit: SHOVEL_SIZE,
- order: [ ["id", "ASC"] ]
- });
- await Promise.map(participations, async (p) =>
- await ch.sendToQueue(crunchQueueForCategory(category),
- new Buffer(p.api_id),
- { persistent: true, type: "global" }));
-
- // update lpcid & refetch
- if (participations.length > 0) {
- last_crunch_participant_id = participations[participations.length-1].id;
- await setKey(db, "global_last_crunch_participant_id",
- last_crunch_participant_id);
- }
- logger.info("loading more participations into cruncher", {
- limit: SHOVEL_SIZE,
- size: participations.length,
- last_crunch_participant_id: last_crunch_participant_id
- });
- } while (participations.length == SHOVEL_SIZE);
- logger.info("done loading participations into cruncher");
-}
-
-// upanalyze all matches TODO add force option
-async function analyzeGlobal(category) {
- const db = databaseForCategory(category);
- let offset = 0, matches;
- do {
- matches = await db.Match.findAll({
- attributes: ["api_id"],
- where: {
- trueskill_quality: null
- },
- limit: SHOVEL_SIZE,
- offset: offset,
- order: [ ["created_at", "ASC"] ]
- });
- await Promise.each(matches, async (m) =>
- await ch.sendToQueue(analyzeQueueForCategory(category),
- new Buffer(m.api_id),
- { persistent: true }));
- offset += SHOVEL_SIZE;
- logger.info("loading more matches into analyzer",
- { offset: offset, limit: SHOVEL_SIZE, size: matches.length });
- } while (matches.length == SHOVEL_SIZE);
- logger.info("done loading matches into analyzer");
-}
-
-// return an entry from keys db
-async function getKey(db, key, default_value) {
- const record = await db.Keys.findOrCreate({
- where: {
- type: KEY_TYPE,
- key: key
- },
- defaults: { value: default_value }
- });
- return record[0].value;
-}
-
-// update an entry from keys db
-async function setKey(db, key, value) {
- const record = await db.Keys.findOrCreate({
- where: {
- type: KEY_TYPE,
- key: key
- },
- defaults: { value: value }
- });
- record[0].update({ value: value });
- return value;
-}
-
-/* routes */
-// update a region, used for tournament shard
-app.post("/api/match/:region/update", async (req, res) => {
- const region = req.params.region;
- if (TOURNAMENT_REGIONS.indexOf(region) == -1) {
- logger.error("called with unsupported region", { region: region });
- res.sendStatus(404);
- return;
- }
- updateRegion(region, "tournament"); // fire away
- res.sendStatus(204);
-});
-// force an update
-app.post("/api/player/:name/search/:category*?", async (req, res) => {
- const name = req.params.name, category = req.params.category || "regular";
- searchPlayer(name, category); // do not await, just fire
- res.sendStatus(204); // notifications will follow
-});
-// update a known user
-// flow:
-// * get from db (depending on category, either regular, brawl or tourn)
-// * update lifetime from `/players` via id, catch IGN changes
-// * update from `/matches`
-app.post("/api/player/:name/update/:category*?", async (req, res) => {
- const name = req.params.name,
- category = req.params.category || "regular",
- db = databaseForCategory(category),
- players = await db.Player.findAll({ where: { name: req.params.name } });
- if (players == undefined) {
- logger.error("player not found in db", { name: req.params.name });
- res.sendStatus(404);
- return;
- }
- logger.info("player in db, updating", { name: name, category: category });
- players.forEach((player) => updatePlayer(player, category)); // fire away
- res.sendStatus(204);
-});
-// crunch a known user TODO force mode
-app.post("/api/player/:name/crunch/:category*?", async (req, res) => {
- const category = req.params.category || "regular",
- db = databaseForCategory(category),
- players = await db.Player.findAll({ where: { name: req.params.name } });
- if (players == undefined) {
- logger.error("player not found in db, won't recrunch",
- { name: req.params.name });
- res.sendStatus(404);
- return;
- }
- logger.info("player in db, recrunching", { name: req.params.name });
- players.forEach((player) => crunchPlayer(category, player.api_id)); // fire away
- res.sendStatus(204);
-});
-// analyze a known user (calculate mmr)
-app.post("/api/player/:name/rank/:category*?", async (req, res) => {
- const category = req.params.category || "regular",
- db = databaseForCategory(category),
- players = await db.Player.findAll({ where: { name: req.params.name } });
- if (players == undefined) {
- logger.error("player not found in db, won't analyze",
- { name: req.params.name });
- res.sendStatus(404);
- return;
- }
- logger.info("player in db, analyzing", { name: req.params.name });
- players.forEach((player) => analyzePlayer(category, player.api_id)); // fire away
- res.sendStatus(204);
-});
-// crunch a known team
-app.post("/api/team/:id/crunch", async (req, res) => {
- if (await model.Team.findOne({ where: { id: req.params.id } }) == undefined) {
- logger.error("team not found in db, won't recrunch",
- { name: req.params.id });
- res.sendStatus(404);
- return;
- }
- logger.info("team in db, recrunching", { name: req.params.id });
- crunchTeam(req.params.id); // fire away
- res.sendStatus(204);
-});
-// grab multiple users by IGN
-app.post("/api/players/:region/grab/:category*?", async (req, res) => {
- let grabstart = new Date();
- grabstart.setTime(grabstart.getTime() - req.query.minutesAgo * 60000);
- grabPlayers(req.query.names, req.params.region, grabstart,
- req.params.category || "regular");
- res.sendStatus(204);
-});
-// update a random user
-app.post("/api/player", async (req, res) => {
- let player = await model.Player.findOne({ order: [ Seq.fn("RAND") ] });
- updatePlayer(player, "regular"); // do not await
- res.json(player);
-});
-// download sample sets
-app.post("/api/samples/:region", async (req, res) => {
- await updateSamples(req.params.region);
- res.sendStatus(204);
-});
-app.post("/api/samples", async (req, res) => {
- await Promise.map(REGIONS, (region) =>
- updateSamples(region));
- res.sendStatus(204);
-});
-// download Telemetry
-app.post("/api/match/:match/telemetry/:category?", async (req, res) => {
- logger.info("requesting download for Telemetry", { api_id: req.params.match });
- const category = req.params.category || "regular",
- db = databaseForCategory(category),
- asset = await db.Asset.findOne({ where: {
- match_api_id: req.params.match
- } });
- if (asset == undefined) {
- logger.error("could not find any assets for match",
- { api_id: req.params.match });
- res.sendStatus(404);
- return;
- }
- await ch.sendToQueue(sampleQueueForCategory(category), new Buffer(JSON.stringify(asset.url)), {
- persistent: true, type: "telemetry",
- headers: { match_api_id: req.params.match }
- });
- res.sendStatus(204);
-});
-// crunch all global stats
-app.post("/api/crunch/:category*?", async (req, res) => {
- const category = req.params.category || "regular";
- crunchGlobal(category, false); // fire away
- res.sendStatus(204);
-});
-// analyze *all* matches
-app.post("/api/rank/:category*?", async (req, res) => {
- const category = req.params.category || "regular";
- analyzeGlobal(category); // fire away
- res.sendStatus(204);
-});
-// force updating all stats
-app.post("/api/recrunch/:category*?", async (req, res) => {
- const category = req.params.category || "regular";
- crunchGlobal(category, true); // fire away
- res.sendStatus(204);
+ ch = await rabbit.createChannel();
+ await cruncher.register(ch, app);
+ await grabber.register(ch, app);
+ await analyzer.register(ch, app);
+ await telemeter.register(ch, app);
});
+server.listen(PORT);
+app.use(express.static("assets"));
+
/* internal monitoring */
-app.get("/", (req, res) => {
- res.sendFile(__dirname + "/index.html");
-});
+app.get("/", (req, res) => res.sendFile(__dirname + "/index.html"));
process.on("unhandledRejection", err => {
logger.error("Uncaught Promise Error:", err.stack);
+ // bridge never exits!
+ //process.exit();
});
diff --git a/service_analyze.js b/service_analyze.js
new file mode 100644
index 0000000..8e6ace8
--- /dev/null
+++ b/service_analyze.js
@@ -0,0 +1,83 @@
+#!/usr/bin/env node
+/* jshint esnext: true */
+"use strict";
+
+const Promise = require("bluebird"),
+ Service = require("./service_skeleton.js");
+
+const logger = global.logger,
+ ANALYZE_QUEUE = process.env.ANALYZE_QUEUE || "analyze",
+ ANALYZE_TOURNAMENT_QUEUE = process.env.ANALYZE_TOURNAMENT_QUEUE || "analyze_tournament",
+ SHOVEL_SIZE = parseInt(process.env.SHOVEL_SIZE) || 1000;
+
+module.exports = class Analyzer extends Service {
+ constructor() {
+ super();
+
+ this.setTargets({
+ "regular": ANALYZE_QUEUE,
+ "tournament": ANALYZE_TOURNAMENT_QUEUE
+ });
+
+ this.setRoutes({
+ // calculate TrueSkill for everyone
+ "/api/rank/:category*?": async (req, res) => {
+ this.analyzeGlobal(req.params.category || "regular");
+ res.sendStatus(204);
+ },
+ // calculate TrueSkill for one player
+ "/api/player/:name/rank/:category*?": async (req, res) => {
+ const category = req.params.category || "regular",
+ db = this.getDatabase(category),
+ players = await db.Player.findAll({ where: { name: req.params.name } });
+ if (players == undefined) {
+ logger.error("player not found in db, won't analyze",
+ { name: req.params.name });
+ res.sendStatus(404);
+ return;
+ }
+ logger.info("player in db, analyzing", { name: req.params.name });
+ players.forEach((player) => this.analyzePlayer(category, player.api_id));
+ res.sendStatus(204);
+ }
+ });
+ }
+
+ async analyzeGlobal(category) {
+ const db = this.getDatabase(category);
+ let offset = 0, matches;
+ do {
+ matches = await db.Match.findAll({
+ attributes: ["api_id"],
+ where: { trueskill_quality: null },
+ limit: SHOVEL_SIZE,
+ offset: offset,
+ order: [ ["created_at", "ASC"] ]
+ });
+ await Promise.each(matches, async (m) =>
+ await this.forward(this.getTarget(category), m.api_id,
+ { persistent: true }));
+ offset += SHOVEL_SIZE;
+ logger.info("loading more matches into analyzer",
+ { offset: offset, limit: SHOVEL_SIZE, size: matches.length });
+ } while (matches.length == SHOVEL_SIZE);
+ logger.info("done loading matches into analyzer");
+ }
+
+ async analyzePlayer(category, api_id) {
+ const db = this.getDatabase(category),
+ participations = await db.Participant.findAll({
+ attributes: ["match_api_id"],
+ where: {
+ player_api_id: api_id,
+ trueskill_mu: null // where not analyzed yet
+ },
+ order: [ ["created_at", "ASC"] ]
+ });
+ logger.info("sending matches to analyzer",
+ { length: participations.length });
+ await Promise.each(participations, async (p) =>
+ await this.forward(this.getTarget(category),
+ p.match_api_id, { persistent: true }));
+ }
+}
diff --git a/service_crunch.js b/service_crunch.js
new file mode 100644
index 0000000..bc8d06b
--- /dev/null
+++ b/service_crunch.js
@@ -0,0 +1,133 @@
+#!/usr/bin/env node
+/* jshint esnext: true */
+"use strict";
+
+const Promise = require("bluebird"),
+ Service = require("./service_skeleton.js");
+
+const logger = global.logger,
+ CRUNCH_QUEUE = process.env.CRUNCH_QUEUE || "crunch",
+ CRUNCH_TOURNAMENT_QUEUE = process.env.CRUNCH_TOURNAMENT_QUEUE || "crunch_tournament",
+ SHOVEL_SIZE = parseInt(process.env.SHOVEL_SIZE) || 1000;
+
+module.exports = class Cruncher extends Service {
+ constructor() {
+ super();
+
+ this.setTargets({
+ "regular": CRUNCH_QUEUE,
+ "tournament": CRUNCH_TOURNAMENT_QUEUE
+ });
+
+ this.setRoutes({
+ // crunch global meta
+ "/api/crunch/:category*?": async (req, res) => {
+ this.crunchGlobal(req.params.category || "regular");
+ res.sendStatus(204);
+ },
+ // crunch a player
+ "/api/player/:name/crunch/:category*?": async (req, res) => {
+ const category = req.params.category || "regular",
+ db = this.getDatabase(req.params.category || "regular"),
+ players = await db.Player.findAll({ where: { name: req.params.name } });
+ if (players == undefined) {
+ logger.error("player not found in db, won't crunch",
+ { name: req.params.name });
+ res.sendStatus(404);
+ return;
+ }
+ logger.info("player in db, crunching", { name: req.params.name });
+ players.forEach((player) =>
+ this.crunchPlayer(category, player.api_id)); // fire away
+ res.sendStatus(204);
+ },
+ // crunch a team
+ "/api/team/:id/crunch": async (req, res) => {
+ const db = this.getDatabase("regular"),
+ team = await db.Team.findOne({ where: { id: req.params.id } });
+ if (team == undefined) {
+ logger.error("team not found in db, won't crunch",
+ { name: req.params.id });
+ res.sendStatus(404);
+ return;
+ }
+ logger.info("team in db, crunching", { name: team.id });
+ crunchTeam(team.id); // fire away
+ res.sendStatus(204);
+ }
+ });
+ }
+
+ // upcrunch player's stats
+ async crunchPlayer(category, api_id) {
+ const db = this.getDatabase(category),
+ where = { player_api_id: api_id },
+ last_crunch = await db.PlayerPoint.findOne({
+ attributes: ["updated_at"],
+ where,
+ order: [ ["updated_at", "DESC"] ]
+ });
+ if (last_crunch) where.created_at = { $gt: last_crunch.updated_at };
+
+ // get all participants for this player
+ const participations = await db.Participant.findAll({
+ attributes: ["api_id"],
+ where
+ });
+ // send everything to cruncher
+ logger.info("sending participations to cruncher",
+ { length: participations.length });
+ await Promise.map(participations, async (p) =>
+ await this.forward(this.getQueue(category),
+ p.api_id, { persistent: true, type: "player" }));
+ // jobs with the type "player" won't be taken into account for global stats
+ // global stats would increase on every player refresh otherwise
+ }
+
+ // reset fame and crunch
+ // TODO: incremental crunch possible?
+ async crunchTeam(team_id) {
+ await this.forward(CRUNCH_QUEUE, team_id,
+ { persistent: true, type: "team" });
+ }
+
+ // crunch global stats
+ async crunchGlobal(category) {
+ const db = this.getDatabase(category);
+ // get lcpid from keys table
+ let last_crunch_participant_id = await this.getKey(db,
+ "global_last_crunch_participant_id", 0);
+
+ // don't load the whole Participant table at once into memory
+ let participations;
+
+ logger.info("loading all participations into cruncher",
+ { last_crunch_participant_id: last_crunch_participant_id });
+ do {
+ participations = await db.Participant.findAll({
+ attributes: ["api_id", "id"],
+ where: {
+ id: { $gt: last_crunch_participant_id }
+ },
+ limit: SHOVEL_SIZE,
+ order: [ ["id", "ASC"] ]
+ });
+ await Promise.map(participations, async (p) =>
+ await this.forward(this.getQueue(category), p.api_id,
+ { persistent: true, type: "global" }));
+
+ // update lpcid & refetch
+ if (participations.length > 0) {
+ last_crunch_participant_id = participations[participations.length-1].id;
+ await this.setKey(db, "global_last_crunch_participant_id",
+ last_crunch_participant_id);
+ }
+ logger.info("loading more participations into cruncher", {
+ limit: SHOVEL_SIZE,
+ size: participations.length,
+ last_crunch_participant_id: last_crunch_participant_id
+ });
+ } while (participations.length == SHOVEL_SIZE);
+ logger.info("done loading participations into cruncher");
+ }
+}
diff --git a/service_grab.js b/service_grab.js
new file mode 100644
index 0000000..96770ce
--- /dev/null
+++ b/service_grab.js
@@ -0,0 +1,295 @@
+#!/usr/bin/env node
+/* jshint esnext: true */
+"use strict";
+
+const api = require("../orm/api"),
+ Promise = require("bluebird"),
+ Seq = require("sequelize"),
+ Service = require("./service_skeleton.js");
+
+const GRABSTART = process.env.GRABSTART || "2017-02-14T00:00:00Z",
+ BRAWL_RETENTION_DAYS = parseInt(process.env.BRAWL_RETENTION_DAYS) || 3,
+ PLAYER_PROCESS_QUEUE = process.env.PLAYER_PROCESS_QUEUE || "process",
+ PLAYER_TOURNAMENT_PROCESS_QUEUE = process.env.PLAYER_TOURNAMENT_PROCESS_QUEUE || "process_tournament",
+ GRAB_QUEUE = process.env.GRAB_QUEUE || "grab",
+ GRAB_BRAWL_QUEUE = process.env.GRAB_BRAWL_QUEUE || "grab_brawl",
+ GRAB_TOURNAMENT_QUEUE = process.env.GRAB_TOURNAMENT_QUEUE || "grab_tournament",
+ REGIONS = (process.env.REGIONS || "na,eu,sg,sa,ea").split(","),
+ TOURNAMENT_REGIONS = (process.env.TOURNAMENT_REGIONS || "tournament-na," +
+ "tournament-eu,tournament-sg,tournament-sa,tournament-ea").split(",");
+
+module.exports = class Analyzer extends Service {
+ constructor() {
+ super();
+
+ this.setTargets({
+ "regular": GRAB_QUEUE,
+ "brawl": GRAB_BRAWL_QUEUE,
+ "tournament": GRAB_TOURNAMENT_QUEUE,
+ "regular_player": PLAYER_PROCESS_QUEUE,
+ "tournament_player": PLAYER_TOURNAMENT_PROCESS_QUEUE
+ });
+
+ this.setRoutes({
+ // update a whole region (tournament only)
+ "/api/match/:region/update": async (req, res) => {
+ const region = req.params.region;
+ if (TOURNAMENT_REGIONS.indexOf(region) == -1) {
+ logger.error("called with unsupported region", { region: region });
+ res.sendStatus(404);
+ return;
+ }
+ this.updateRegion(region, "tournament"); // fire away
+ res.sendStatus(204);
+ },
+ // grab multiple users by IGN
+ "/api/players/:region/grab/:category*?": async (req, res) => {
+ let grabstart = new Date();
+ grabstart.setTime(grabstart.getTime() - req.query.minutesAgo * 60000);
+ this.grabPlayers(req.query.names, req.params.region, grabstart,
+ req.params.category || "regular");
+ res.sendStatus(204);
+ },
+
+ // find a player / force a player update
+ "/api/player/:name/search/:category*?": async (req, res) => {
+ this.searchPlayer(req.params.name, req.params.category || "regular");
+ res.sendStatus(204); // notifications will follow
+ },
+ // update a known user
+ // flow:
+ // * get from db (depending on category, either regular, brawl or tourn)
+ // * update lifetime from `/players` via id, catch IGN changes
+ // * update from `/matches`
+ "/api/player/:name/update/:category*?": async (req, res) => {
+ const name = req.params.name,
+ category = req.params.category || "regular",
+ db = this.getDatabase(category),
+ players = await db.Player.findAll({ where: { name: req.params.name } });
+ if (players == undefined) {
+ logger.error("player not found in db", { name: req.params.name });
+ res.sendStatus(404);
+ return;
+ }
+ logger.info("player in db, updating", { name: name, category: category });
+ players.forEach((player) => this.updatePlayer(player, category));
+ res.sendStatus(204);
+ },
+ // update a random user
+ "/api/player": async (req, res) => {
+ // TODO this becomes slow
+ let player = await this.getDatabase("regular").Player.findOne({ order: [ Seq.fn("RAND") ] });
+ this.updatePlayer(player, "regular");
+ res.json(player);
+ }
+ });
+ }
+
+ // convenience
+ getGameModes(category) {
+ switch (category) {
+ case "regular": return "casual,ranked";
+ case "brawl": return "casual_aral,blitz_pvp_ranked";
+ case "tournament": return "private,private_party_draft_match";
+ }
+ logger.error("unsupported game mode category", { category: category });
+ }
+ getRegions(category) {
+ switch (category) {
+ case "brawl":
+ case "regular": return REGIONS;
+ case "tournament": return TOURNAMENT_REGIONS;
+ }
+ logger.error("unsupported region category", { category: category });
+ }
+ // return a default `createdAt-start` based on game mode
+ getGrabstart(category) {
+ switch (category) {
+ case "regular": return GRABSTART;
+ case "brawl":
+ let past = new Date();
+ past.setDate(past.getDate() - BRAWL_RETENTION_DAYS);
+ return past.toISOString(); // minimum
+ case "tournament": return "2017-02-12T00:00:00Z"; // all
+ }
+ logger.error("unsupported grabstart category", { category: category });
+ }
+
+ // API requires maximum time interval of 4w
+ // return [payload, payload, …] with fixed createdAt
+ getSplitGrabs(payload, start, end) {
+ let part_start = start, part_end = start,
+ payloads = [];
+ // loop forwards, adding 4w until we passed NOW
+ while (part_start < end) {
+ part_end = new Date(part_end.getTime() + (4 * 60*60*24*7*1000)); // add 4w
+ if(part_end > end) part_end = end;
+ let pl = payload;
+ pl["params"]["filter[createdAt-start]"] = part_start.toISOString();
+ pl["params"]["filter[createdAt-end]"] = part_end.toISOString();
+ // push a deep clone or the object will have the reference to part_start
+ // which leads to all start/end dates being the same (?!)
+ payloads.push(JSON.parse(JSON.stringify(pl))); // JavaScript sucks.
+ part_start = part_end;
+ }
+ return payloads;
+ }
+
+ // request grab jobs for a player's matches
+ async grabPlayer(name, region, last_match_created_date, id, category) {
+ const start = new Date(Date.parse(last_match_created_date)),
+ end = new Date(); // today
+
+ const payload_template = {
+ "region": region,
+ "params": {
+ "filter[playerIds]": id,
+ "filter[gameMode]": this.getGameModes(category),
+ "sort": "createdAt"
+ }
+ };
+ await Promise.each(this.getSplitGrabs(payload_template, start, end), async (payload) => {
+ logger.info("requesting update", { name: name, region: region });
+
+ await this.forward(this.getTarget(category), JSON.stringify(payload), {
+ persistent: true,
+ headers: { notify: "player." + name }
+ });
+ });
+ }
+
+ // request grab job for multiple players' matches
+ async grabPlayers(names, region, grabstart, category) {
+ const payload_template = {
+ "region": region,
+ "params": {
+ "filter[playerNames]": names,
+ "filter[gameMode]": this.getGameModes(category),
+ "sort": "createdAt"
+ }
+ }, now = new Date();
+ logger.info("requesting triple player grab", {
+ names: names, region: region, category: category
+ });
+
+ await Promise.each(this.getSplitGrabs(payload_template, grabstart, now), async (payload) => {
+ await this.forward(this.getTarget(category), JSON.stringify(payload), {
+ persistent: true,
+ headers: { notify: "player." + names }
+ });
+ });
+ }
+
+ // request grab jobs for a region's matches
+ async grabMatches(region, lmcd) {
+ if (lmcd == undefined) lmcd = new Date(Date.parse(this.getGrabstart("tournament")));
+ const now = new Date();
+
+ // add 1s, because createdAt-start <= x <= createdAt-end
+ // so without the +1s, we'd always get the last_match_created_date match back
+ last_match_created_date.setSeconds(last_match_created_date.getSeconds() + 1);
+
+ const payload_template = {
+ "region": region,
+ "params": { "sort": "createdAt" }
+ };
+ logger.info("requesting region update", { region: region });
+
+ await Promise.each(this.getSplitGrabs(payload_template, last_match_created_date, now), async (payload) => {
+ await this.forward(GRAB_TOURNAMENT_QUEUE,
+ JSON.stringify(payload), { persistent: true });
+ });
+ }
+
+ // search for a player name in one region
+ // request process for lifetime
+ // return an array (JSONAPI response)
+ async searchPlayerInRegion(region, name, id, category) {
+ logger.info("searching", { name: name, id: id, region: region });
+ let options = {};
+ if (id == undefined) options["filter[playerNames]"] = name
+ else options["filter[playerIds]"] = id
+
+ // players.length and page length will be 1 in 99.9999% of all cases
+ // - but just in case.
+ const players = await Promise.map(await api.requests("players", region, options, logger), async (player) => {
+ logger.info("found", { name: name, id: id, region: region });
+ await this.notify("player." + name, "search_success");
+ // send to processor, so the player is in db
+ // no matter whether we find matches or not
+ await this.forward(this.getTarget(category + "_player"), JSON.stringify(player), {
+ persistent: true, type: "player",
+ headers: { notify: "player." + player.name }
+ });
+ return player;
+ });
+ if (players.length == 0)
+ logger.warn("not found", { name: name, id: id, region: region });
+
+ return players;
+ }
+
+ // search for player name on all shards
+ // grab player(s)
+ // send a notification for results and request updates
+ async searchPlayer(name, category) {
+ let found = false;
+ logger.info("searching", { name: name });
+ await Promise.map(this.getRegions(category), async (region) => {
+ const players = await this.searchPlayerInRegion(region, name, undefined, category);
+ if (players.length > 0) found = true;
+
+ // request grab jobs
+ await Promise.map(players, async (p) =>
+ await this.grabPlayer(p.name, p.shardId,
+ this.getGrabstart(category), p.id, category));
+ });
+ // notify web
+ if (!found) {
+ logger.info("search failed", { name: name });
+ await this.notify("player." + name, "search_fail");
+ }
+ }
+
+ // update a player from API based on db record
+ // & grab player
+ async updatePlayer(player, category) {
+ // set last_update and request an update job
+ // if last_update is null, we need that player's full history
+ let grabstart;
+ if (player.get("last_update") != null && category == "regular")
+ // TODO do not fetch everything on category tournament / brawl
+ grabstart = player.get("created_at");
+ if (grabstart == undefined) grabstart = this.getGrabstart(category);
+ else
+ // add 1s, because createdAt-start <= x <= createdAt-end
+ // so without the +1s, we'd always get the last_match_created_date match back
+ grabstart.setSeconds(grabstart.getSeconds() + 1);
+
+ const players = await this.searchPlayerInRegion(
+ player.shard_id, player.name, player.api_id, category);
+
+ // will be the same as `player` 99.9% of the time
+ // but not when the player changed their name
+ // search happened by ID, so we get only 1 player back
+ // but player.name != players[0].attributes.name
+ // (with a name change)
+ await Promise.map(players, async (p) =>
+ await this.grabPlayer(p.name, p.shardId, grabstart, p.id, category));
+ }
+
+ // update a region from API based on db records
+ async updateRegion(region, category) {
+ let grabstart;
+ const last_match = await this.getDatabase(category).Participant.findOne({
+ attributes: ["created_at"],
+ where: { shard_id: region },
+ order: [ [seq.col("created_at"), "DESC"] ]
+ });
+ if (last_match == null) grabstart = undefined;
+ else grabstart = last_match.get("created_at");
+
+ await this.grabMatches(region, grabstart);
+ }
+
+}
diff --git a/service_skeleton.js b/service_skeleton.js
new file mode 100644
index 0000000..1dc9a3b
--- /dev/null
+++ b/service_skeleton.js
@@ -0,0 +1,73 @@
+#!/usr/bin/env node
+/* jshint esnext: true */
+"use strict";
+
+const Promise = require("bluebird");
+
+const logger = global.logger,
+ KEY_TYPE = process.env.KEY_TYPE || "crunch"; // backwards compat
+
+module.exports = class Service {
+ setRoutes(routes) { // route str -> func
+ this.routes = new Map(Object.entries(routes));
+ }
+
+ setDatabases(dbs, defaultDb) { // mode str -> Sequelize
+ this.dbs = new Map(Object.entries(dbs));
+ }
+ getDatabase(category) {
+ if (!this.dbs.has(category))
+ logger.error("unsupported database category");
+ return this.dbs.get(category);
+ }
+
+ async register(rmqChannel, expressApp) { // assert queues, initialize routes
+ this.notify = async (topic, msg) =>
+ await rmqChannel.publish("amq.topic", topic, new Buffer(msg));
+ this.forward = async (queue, payload, options) =>
+ await rmqChannel.sendToQueue(queue, new Buffer(payload), options);
+
+ await Promise.all(this.targets, (queue) =>
+ rmqChannel.assertQueue(queue, { durable: true }));
+
+ for(let [route, func] of this.routes) {
+ expressApp.post(route, func);
+ }
+ }
+
+ setTargets(queues) { // set channels messages can be forwarded to, str -> str
+ this.targets = new Map(Object.entries(queues));
+ }
+ getTarget(category) { // return channel for category
+ if (!this.targets.has(category))
+ logger.error("unsupported queue category", category);
+ return this.targets.get(category);
+ }
+
+ // return an entry from keys db
+ async getKey(category, key, default_value) {
+ const db = this.getDatabase(category),
+ record = await db.Keys.findOrCreate({
+ where: {
+ type: KEY_TYPE,
+ key: key
+ },
+ defaults: { value: default_value }
+ });
+ return record[0].value;
+ }
+
+ // update an entry from keys db
+ async setKey(category, key, value) {
+ const db = this.getDatabase(category),
+ record = await db.Keys.findOrCreate({
+ where: {
+ type: KEY_TYPE,
+ key: key
+ },
+ defaults: { value: value }
+ });
+ record[0].update({ value: value });
+ return value;
+ }
+}
diff --git a/service_telemetry.js b/service_telemetry.js
new file mode 100644
index 0000000..285de4e
--- /dev/null
+++ b/service_telemetry.js
@@ -0,0 +1,43 @@
+#!/usr/bin/env node
+/* jshint esnext: true */
+"use strict";
+
+const Promise = require("bluebird"),
+ Service = require("./service_skeleton.js");
+
+const logger = global.logger,
+ SAMPLE_QUEUE = process.env.SAMPLE_QUEUE || "sample",
+ SAMPLE_TOURNAMENT_QUEUE = process.env.SAMPLE_TOURNAMENT_QUEUE || "sample_tournament";
+
+module.exports = class Analyzer extends Service {
+ constructor() {
+ super();
+
+ this.setTargets({
+ "regular": SAMPLE_QUEUE,
+ "tournament": SAMPLE_TOURNAMENT_QUEUE
+ });
+
+ this.setRoutes({
+ "/api/match/:match/telemetry/:category?": async (req, res) => {
+ logger.info("requesting download for Telemetry", { api_id: req.params.match });
+ const category = req.params.category || "regular",
+ db = this.getDatabase(category),
+ asset = await db.Asset.findOne({ where: {
+ match_api_id: req.params.match
+ } });
+ if (asset == undefined) {
+ logger.error("could not find any assets for match",
+ { api_id: req.params.match });
+ res.sendStatus(404);
+ return;
+ }
+ await this.forward(this.getTarget(category), asset.url, {
+ persistent: true,
+ headers: { match_api_id: req.params.match }
+ });
+ res.sendStatus(204);
+ }
+ });
+ }
+}
generated by cgit v1.3.1 (git 2.54.0) at 2026-08-01 23:32:30 +0000