1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
|
#!/usr/bin/node
/* jshint esnext:true */
"use strict";
const winston = require("winston"),
loggly = require("winston-loggly-bulk"),
Seq = require("sequelize"),
elasticsearch = require("elasticsearch");
const DATABASE_URI = process.env.DATABASE_URI,
ELASTIC_URI = process.env.ELASTIC_URI || "localhost:9200",
LOGGLY_TOKEN = process.env.LOGGLY_TOKEN,
BATCHSIZE = process.env.BATCHSIZE || 50,
MAXCONNS = parseInt(process.env.MAXCONNS) || 20;
const logger = new (winston.Logger)({
transports: [
new (winston.transports.Console)({
timestamp: true,
colorize: true
})
]
});
// loggly integration
if (LOGGLY_TOKEN)
logger.add(winston.transports.Loggly, {
inputToken: LOGGLY_TOKEN,
subdomain: "kvahuja",
tags: ["backend", "reaper"],
json: true
});
// connect to rabbit & db
const seq = new Seq(DATABASE_URI, {
logging: false,
max: MAXCONNS
}),
model = require("../orm/model")(seq, Seq),
elastic = new elasticsearch.Client({ host: ELASTIC_URI, log: "info" });
// assumes `id` exists
// Sequelize model, index type, index key, key to parent id, hook, custom condition
async function load(table, type, includes, filter) {
// load from biggest id to 0
const last_id_r = await model.Keys.findOrCreate({
where: { type: "reaper_last_id_fetched", key: type },
defaults: { value: 0 }
});
while (true) {
const last_id = last_id_r[0].value,
condition = Object.assign({}, filter,
{ id: { $gt: last_id } });
logger.info("loading", { type, last_id });
let data = await table.findAll({
where: condition,
order: [ [seq.col("id"), "DESC"] ],
include: includes,
limit: BATCHSIZE,
raw: true
});
if (data.length == 0) break; // exhausted
await last_id_r[0].update({ value: data[data.length-1].id });
await elastic.bulk({
body: [].concat(... data.map((d) => [
{ index: {
_index: type,
_type: type,
_id: d.api_id || d.id
} },
d
]) )
});
}
logger.info("done.", { type });
}
(async function() {
await Promise.all([
load(model.Participant, "participant", [
model.ParticipantStats,
model.Roster,
model.Match,
model.Region, model.Hero, model.Series, model.GameMode, model.Role
]),
load(model.ParticipantPhases, "participant_phases", [ {
model: model.Participant,
include: [
model.Region, model.Hero, model.Series, model.GameMode, model.Role
]
} ]),
//load(model.Match, "match", [ model.Asset ]),
load(model.PlayerPoint, "player_point", [ {
model: model.Player,
include: [ model.Region ]
},
model.Series, model.Hero, model.GameMode, model.Role
]),
load(model.Player, "player", [ model.Region ]),
]);
})();
process.on("unhandledRejection", (err) => {
logger.error(err);
//process.exit();
});
|
