diff options
| -rw-r--r-- | package-lock.json | 566 | ||||
| -rw-r--r-- | package.json | 8 | ||||
| -rw-r--r-- | worker.js | 86 |
3 files changed, 604 insertions, 56 deletions
diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..e0ca984 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,566 @@ +{ + "name": "apigrabber", + "version": "2.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "amqplib": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.5.1.tgz", + "integrity": "sha1-fMz+ur5WwumE6noiQ/fO/m+/xs8=", + "requires": { + "bitsyntax": "0.0.4", + "bluebird": "3.5.0", + "buffer-more-ints": "0.0.2", + "readable-stream": "1.1.14" + }, + "dependencies": { + "bitsyntax": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/bitsyntax/-/bitsyntax-0.0.4.tgz", + "integrity": "sha1-6xDMb4K4xJDj6FaY8H6D1G4MuoI=", + "requires": { + "buffer-more-ints": "0.0.2" + } + }, + "buffer-more-ints": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-0.0.2.tgz", + "integrity": "sha1-JrOIXRD6E9t/wBquOquHAZngEkw=" + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "0.0.1", + "string_decoder": "0.10.31" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, + "bluebird": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.0.tgz", + "integrity": "sha1-eRQg1/VR7qKJdFOop3ZT+WYG1nw=" + }, + "json-stringify-safe": { + "version": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "lodash": { + "version": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz", + "integrity": "sha1-eCA6TRwyiuHYbcpkYONptX9AVa4=" + }, + "node-loggly-bulk": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/node-loggly-bulk/-/node-loggly-bulk-2.0.0.tgz", + "integrity": "sha1-5M+UOY15jroSQwUsLybX3ZuSEE0=", + "requires": { + "json-stringify-safe": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "request": "2.81.0", + "timespan": "2.3.0" + } + }, + "punycode": { + "version": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + }, + "request": { + "version": "2.81.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.81.0.tgz", + "integrity": "sha1-xpKJRqDgbF+Nb4qTM0af/aRimKA=", + "requires": { + "aws-sign2": "0.6.0", + "aws4": "1.6.0", + "caseless": "0.12.0", + "combined-stream": "1.0.5", + "extend": "3.0.1", + "forever-agent": "0.6.1", + "form-data": "2.1.4", + "har-validator": "4.2.1", + "hawk": "3.1.3", + "http-signature": "1.1.1", + "is-typedarray": "1.0.0", + "isstream": "0.1.2", + "json-stringify-safe": "5.0.1", + "mime-types": "2.1.15", + "oauth-sign": "0.8.2", + "performance-now": "0.2.0", + "qs": "6.4.0", + "safe-buffer": "5.1.1", + "stringstream": "0.0.5", + "tough-cookie": "2.3.2", + "tunnel-agent": "0.6.0", + "uuid": "3.1.0" + }, + "dependencies": { + "ajv": { + "version": "4.11.8", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-4.11.8.tgz", + "integrity": "sha1-gv+wKynmYq5TvcIK8VlHcGc5xTY=", + "requires": { + "co": "4.6.0", + "json-stable-stringify": "1.0.1" + } + }, + "asn1": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", + "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=" + }, + "assert-plus": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz", + "integrity": "sha1-104bh+ev/A24qttwIfP+SBAasjQ=" + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "aws-sign2": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz", + "integrity": "sha1-FDQt0428yU0OW4fXY81jYSwOeU8=" + }, + "aws4": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz", + "integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4=" + }, + "bcrypt-pbkdf": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", + "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=", + "optional": true, + "requires": { + "tweetnacl": "0.14.5" + } + }, + "boom": { + "version": "2.10.1", + "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz", + "integrity": "sha1-OciRjO/1eZ+D+UkqhI9iWt0Mdm8=", + "requires": { + "hoek": "2.16.3" + } + }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" + }, + "combined-stream": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", + "integrity": "sha1-k4NwpXtKUd6ix3wV1cX9+JUWQAk=", + "requires": { + "delayed-stream": "1.0.0" + } + }, + "cryptiles": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz", + "integrity": "sha1-O9/s3GCBR8HGcgL6KR59ylnqo7g=", + "requires": { + "boom": "2.10.1" + } + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "1.0.0" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + } + } + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "ecc-jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", + "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=", + "optional": true, + "requires": { + "jsbn": "0.1.1" + } + }, + "extend": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", + "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=" + }, + "extsprintf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz", + "integrity": "sha1-4QgOBljjALBilJkMxw4VAiNf1VA=" + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.1.4.tgz", + "integrity": "sha1-M8GDrPGTJ27KqYFDpp6Uv+4XUNE=", + "requires": { + "asynckit": "0.4.0", + "combined-stream": "1.0.5", + "mime-types": "2.1.15" + } + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "1.0.0" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + } + } + }, + "har-schema": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-1.0.5.tgz", + "integrity": "sha1-0mMTX0MwfALGAq/I/pWXDAFRNp4=" + }, + "har-validator": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-4.2.1.tgz", + "integrity": "sha1-M0gdDxu/9gDdID11gSpqX7oALio=", + "requires": { + "ajv": "4.11.8", + "har-schema": "1.0.5" + } + }, + "hawk": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz", + "integrity": "sha1-B4REvXwWQLD+VA0sm3PVlnjo4cQ=", + "requires": { + "boom": "2.10.1", + "cryptiles": "2.0.5", + "hoek": "2.16.3", + "sntp": "1.0.9" + } + }, + "hoek": { + "version": "2.16.3", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz", + "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=" + }, + "http-signature": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz", + "integrity": "sha1-33LiZwZs0Kxn+3at+OE0qPvPkb8=", + "requires": { + "assert-plus": "0.2.0", + "jsprim": "1.4.0", + "sshpk": "1.13.1" + } + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", + "optional": true + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, + "json-stable-stringify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz", + "integrity": "sha1-mnWdOcXy/1A/1TAGRu1EX4jE+a8=", + "requires": { + "jsonify": "0.0.0" + } + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "jsonify": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz", + "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=" + }, + "jsprim": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.0.tgz", + "integrity": "sha1-o7h+QCmNjDgFUtjMdiigu5WiKRg=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.0.2", + "json-schema": "0.2.3", + "verror": "1.3.6" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + } + } + }, + "mime-db": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.27.0.tgz", + "integrity": "sha1-gg9XIpa70g7CXtVeW13oaeVDbrE=" + }, + "mime-types": { + "version": "2.1.15", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.15.tgz", + "integrity": "sha1-pOv1BkCUVpI3uM9wBGd20J/JKu0=", + "requires": { + "mime-db": "1.27.0" + } + }, + "oauth-sign": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", + "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=" + }, + "performance-now": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-0.2.0.tgz", + "integrity": "sha1-M+8wxcd9TqIcWlOGnZG1bY8lVeU=" + }, + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + }, + "qs": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.4.0.tgz", + "integrity": "sha1-E+JtKK1rD/qpExLNO/cI7TUecjM=" + }, + "safe-buffer": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", + "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" + }, + "sntp": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz", + "integrity": "sha1-ZUEYTMkK7qbG57NeJlkIJEPGYZg=", + "requires": { + "hoek": "2.16.3" + } + }, + "sshpk": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz", + "integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=", + "requires": { + "asn1": "0.2.3", + "assert-plus": "1.0.0", + "bcrypt-pbkdf": "1.0.1", + "dashdash": "1.14.1", + "ecc-jsbn": "0.1.1", + "getpass": "0.1.7", + "jsbn": "0.1.1", + "tweetnacl": "0.14.5" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + } + } + }, + "stringstream": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", + "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=" + }, + "tough-cookie": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.2.tgz", + "integrity": "sha1-8IH3bkyFcg5sN6X6ztc3FQ2EByo=", + "requires": { + "punycode": "1.4.1" + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "5.1.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", + "optional": true + }, + "uuid": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", + "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==" + }, + "verror": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.3.6.tgz", + "integrity": "sha1-z/XfEpRtKX0rqu+qJoniW+AcAFw=", + "requires": { + "extsprintf": "1.0.2" + } + } + } + }, + "request-promise": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.1.tgz", + "integrity": "sha1-fuxWyJMXqCLL/qmbA5zlQ8LhX2c=", + "requires": { + "bluebird": "3.5.0", + "request-promise-core": "1.1.1", + "stealthy-require": "1.1.1", + "tough-cookie": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.2.tgz" + }, + "dependencies": { + "request-promise-core": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz", + "integrity": "sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=", + "requires": { + "lodash": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz" + } + } + } + }, + "stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" + }, + "timespan": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/timespan/-/timespan-2.3.0.tgz", + "integrity": "sha1-SQLOBAvRPYRcj1myfp1ZutbzmSk=" + }, + "tough-cookie": { + "version": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.2.tgz", + "integrity": "sha1-8IH3bkyFcg5sN6X6ztc3FQ2EByo=", + "requires": { + "punycode": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz" + } + }, + "winston": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/winston/-/winston-2.3.1.tgz", + "integrity": "sha1-C0hCDZeMAYBM8CMLZIhhWYIloRk=", + "requires": { + "async": "1.0.0", + "colors": "1.0.3", + "cycle": "1.0.3", + "eyes": "0.1.8", + "isstream": "0.1.2", + "stack-trace": "0.0.10" + }, + "dependencies": { + "async": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", + "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=" + }, + "colors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=" + }, + "cycle": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", + "integrity": "sha1-IegLK+hYD5i0aPN5QwZisEbDStI=" + }, + "eyes": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", + "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=" + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" + } + } + }, + "winston-loggly-bulk": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/winston-loggly-bulk/-/winston-loggly-bulk-2.0.0.tgz", + "integrity": "sha1-olOCeBihS7b+CwbMHCLY0+19EoA=", + "requires": { + "node-loggly-bulk": "2.0.0", + "winston": "2.3.1" + } + } + } +} diff --git a/package.json b/package.json index e69daca..3e7bd2d 100644 --- a/package.json +++ b/package.json @@ -4,16 +4,12 @@ "description": "", "main": "worker.js", "dependencies": { - "adm-zip": "^0.4.7", "amqplib": "^0.5.1", "bluebird": "^3.5.0", "request": "^2.81.0", - "request-promise": "^4.2.0", - "sleep-promise": "^2.0.0", - "snakecase-keys": "^1.1.0", - "superagent-jsonapify": "^1.4.4", + "request-promise": "^4.2.1", "winston": "^2.3.1", - "winston-loggly-bulk": "^1.4.2" + "winston-loggly-bulk": "^2.0.0" }, "devDependencies": {}, "scripts": { @@ -7,16 +7,13 @@ const amqp = require("amqplib"), winston = require("winston"), loggly = require("winston-loggly-bulk"), request = require("request-promise"), - sleep = require("sleep-promise"), api = require("../orm/api"); const MADGLORY_TOKEN = process.env.MADGLORY_TOKEN, QUEUE = process.env.QUEUE || "grab", PROCESS_QUEUE = process.env.PROCESS_QUEUE || "process", - SAMPLE_QUEUE = process.env.SAMPLE_QUEUE || "sample", RABBITMQ_URI = process.env.RABBITMQ_URI || "amqp://localhost", - LOGGLY_TOKEN = process.env.LOGGLY_TOKEN, - GRABBERS = parseInt(process.env.GRABBERS) || 4; + LOGGLY_TOKEN = process.env.LOGGLY_TOKEN; if (MADGLORY_TOKEN == undefined) throw "Need an API token"; const logger = new (winston.Logger)({ @@ -37,59 +34,47 @@ if (LOGGLY_TOKEN) json: true }); -(async () => { - let rabbit, ch; +amqp.connect(RABBITMQ_URI).then(async (rabbit) => { + process.once("SIGINT", rabbit.close.bind(rabbit)); - while (true) { - try { - rabbit = await amqp.connect(RABBITMQ_URI); - ch = await rabbit.createChannel(); - await ch.assertQueue(QUEUE, {durable: true}); - await ch.assertQueue(PROCESS_QUEUE, {durable: true}); - break; - } catch (err) { - logger.error("error connecting", err); - await sleep(5000); - } - } + const ch = await rabbit.createChannel(); + await ch.assertQueue(QUEUE, { durable: true }); + await ch.assertQueue(QUEUE + "_failed", { durable: true }); + await ch.assertQueue(PROCESS_QUEUE, { durable: true }); + await ch.prefetch(1); // 1 worker = 1 grab job - await ch.prefetch(GRABBERS); // main queue ch.consume(QUEUE, async (msg) => { - let payload = JSON.parse(msg.content.toString()), - notify = msg.properties.headers.notify; // where to send progress report - if (msg.properties.type == "matches") - await getAPI(payload, "matches", notify); - if (msg.properties.type == "samples") - await getAPI(payload, "samples"); - - logger.info("done", payload); - ch.ack(msg); + try { + let payload = JSON.parse(msg.content.toString()), + notify = msg.properties.headers.notify; // where to send progress report + await getAPI(payload, notify); + logger.info("done", payload); + await ch.ack(msg); + } catch (err) { + // log, move to error queue and NACK on *any* error + logger.error(err); + await ch.sendToQueue(QUEUE + "_failed", msg.content, { + persistent: true, + headers: msg.properties.headers + }); + await msg.nack(false, false); + } }, { noAck: false }); // loop over API data objects - async function getAPI(payload, where, notify="global") { - const data = await Promise.each(await api.requests(where, + async function getAPI(payload, notify) { + const data = await Promise.each(await api.requests("matches", payload.region, payload.params, logger), async (data, idx, len) => { - if (where == "matches") { // send match structure - await ch.sendToQueue(PROCESS_QUEUE, - new Buffer(JSON.stringify(data)), { - persistent: true, type: "match", - headers: { - notify: notify, - donotify: idx == len -1 // TODO remove this header, backwards compat for web - } - }); - // forward "notify" for the last match on the last page - if (notify) await ch.publish("amq.topic", notify, - new Buffer("match_pending")); - } - if (where == "samples") - // forward to sampler - await ch.sendToQueue(SAMPLE_QUEUE, - new Buffer(JSON.stringify(data.attributes.URL)), - { persistent: true, type: "sample" }); + await ch.sendToQueue(PROCESS_QUEUE, + new Buffer(JSON.stringify(data)), { + persistent: true, type: "match", + headers: { notify } + }); + // forward "notify" for the last match on the last page + if (notify) await ch.publish("amq.topic", notify, + new Buffer("match_pending")); return data; } ); @@ -97,8 +82,9 @@ if (LOGGLY_TOKEN) await ch.publish("amq.topic", notify, new Buffer("matches_none")); } -})(); +}); -process.on("unhandledRejection", err => { +process.on("unhandledRejection", (err) => { logger.error("Uncaught Promise Error:", err.stack); + process.exit(1); // fail hard and die }); |