From f4771f98d5986819948ed6828a8900e5df989c7f Mon Sep 17 00:00:00 2001
From: schneefux <schneefux+commit@schneefux.xyz>
Date: Thu, 27 Jul 2017 22:01:22 +0200
Subject: first version using parent&child relationships

---
 app.js | 125 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 125 insertions(+)
 create mode 100644 app.js

(limited to 'app.js')

diff --git a/app.js b/app.js
new file mode 100644
index 0000000..bd29a39
--- /dev/null
+++ b/app.js
@@ -0,0 +1,125 @@
+#!/usr/bin/node
+/* jshint esnext:true */
+"use strict";
+
+const winston = require("winston"),
+    loggly = require("winston-loggly-bulk"),
+    Seq = require("sequelize"),
+    elasticsearch = require("elasticsearch");
+
+const DATABASE_URI = process.env.DATABASE_URI,
+    ELASTIC_URI = process.env.ELASTIC_URI || "localhost:9200",
+    LOGGLY_TOKEN = process.env.LOGGLY_TOKEN,
+    MAXCONNS = parseInt(process.env.MAXCONNS) || 20;
+
+const logger = new (winston.Logger)({
+    transports: [
+        new (winston.transports.Console)({
+            timestamp: true,
+            colorize: true
+        })
+    ]
+});
+
+// loggly integration
+if (LOGGLY_TOKEN)
+    logger.add(winston.transports.Loggly, {
+        inputToken: LOGGLY_TOKEN,
+        subdomain: "kvahuja",
+        tags: ["backend", "reaper"],
+        json: true
+    });
+
+
+// connect to rabbit & db
+const seq = new Seq(DATABASE_URI, {
+        logging: false,
+        max: MAXCONNS
+    }),
+    model = require("../orm/model")(seq, Seq),
+    elastic = new elasticsearch.Client({ host: ELASTIC_URI, log: "info" });
+
+// assumes `id` exists
+// Sequelize model, index type, index key, key to parent id, hook, custom condition
+async function load(table, type, parent_key, merger, filter) {
+    // load from biggest id to 0
+    const last_id_r = await model.Keys.findOrCreate({
+            where: { type: "reaper_last_id_fetched", key: type },
+            defaults: { value: 2147483647 }
+        });
+    while (true) {
+        const last_id = last_id_r[0].value,
+            condition = Object.assign({}, filter,
+                { id: { $lt: last_id } });
+
+        logger.info("loading", { type, last_id });
+
+        let data = await table.findOne({
+            where: condition,
+            order: [ [seq.col("id"), "DESC"] ],
+            raw: true
+        });
+        if (data == undefined) break;  // exhausted
+        last_id_r[0].value = data.id;
+        await last_id_r[0].save();
+
+        // using a custom function, merge with other models as needed
+        if (merger != undefined)
+            // important! don't merge the other way round, merged ids should not overwrite
+            data = Object.assign(await merger(data), data);
+
+        let id = data.api_id != undefined? data.api_id:data.id,
+            // pp doesn't have api id
+            parent = parent_key != undefined? data[parent_key]:undefined;
+        await elastic.create({
+            index: "semc-vainglory",
+            type,
+            id,
+            parent,
+            body: data
+        });
+    }
+
+    logger.info("done.", { type });
+}
+
+async function createParentChild(parent_type, child_type) {
+    await elastic.indices.putMapping({
+        index: "semc-vainglory",
+        type: child_type,
+        body: {
+            "_parent": { "type": parent_type }
+        }
+    });
+}
+
+(async function() {
+    try {
+        await elastic.indices.create({ index: "semc-vainglory" });
+        await Promise.all([
+            createParentChild("participant", "participant_phases"),
+            createParentChild("roster", "participant"),
+            createParentChild("match", "roster"),
+            createParentChild("player", "player_point")
+        ]);
+    } catch (e) { }  // already exist
+
+    await Promise.all([
+        load(model.Match, "match", undefined, async (m) =>
+            await model.Asset.findOne({
+                where: { match_api_id: m.api_id },
+                raw: true
+            }) || { }
+        ),
+        load(model.Roster, "roster", "match_api_id"),
+        load(model.Participant, "participant", "roster_api_id", async (p) =>
+            await model.ParticipantStats.findOne({
+                where: { participant_api_id: p.api_id },
+                raw: true
+            })
+        ),
+        load(model.ParticipantPhases, "participant_phases", "participant_api_id"),
+        load(model.Player, "player"),
+        load(model.PlayerPoint, "player_point", "player_api_id")
+    ]);
+})();
-- 
cgit v1.3.1