From ce1f18227b3b8e3d087e2b920151d63d0061fe33 Mon Sep 17 00:00:00 2001 From: schneefux Date: Sun, 26 Feb 2017 12:20:05 +0100 Subject: pass object type to process job payload --- api.py | 17 +++++++++++------ insert.sql | 8 ++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/api.py b/api.py index 0e68b44..117f55f 100644 --- a/api.py +++ b/api.py @@ -60,13 +60,18 @@ class Worker(object): async for data in api.matches(region=payload["region"], params=payload["params"]): logging.debug("%s: inserting into database", jobid) - ids = await con.fetch(self._insertquery, json.dumps(data)) - logging.info("%s: inserted %s objects", jobid, len(ids)) - object_ids = [i["id"] for i in ids] - for object_id in object_ids: + objects = await con.fetch(self._insertquery, json.dumps(data)) + logging.info("%s: inserted %s", jobid, + {t: len([s for s in objects if s["type"] == t]) + for t in set([e["type"] for e in objects])} + ) + for obj in objects: await self._queue.request(jobtype="process", - priority=priority, - payload={"id": object_id}) + priority=priority, + payload={ + "id": obj["id"], + "type": obj["type"] + }) async def _work(self): """Fetch a job and run it.""" diff --git a/insert.sql b/insert.sql index be4dcae..abba48e 100644 --- a/insert.sql +++ b/insert.sql @@ -68,15 +68,15 @@ insert_matches AS ( INSERT INTO match SELECT * FROM linked_matches ON CONFLICT(id) DO NOTHING - RETURNING id -- TODO conflict shouldn't happen in prod + RETURNING type, id -- TODO conflict shouldn't happen in prod ), insert_players AS ( INSERT INTO player SELECT * FROM linked_players ON CONFLICT(id) DO UPDATE SET attributes=EXCLUDED.attributes WHERE (player.attributes->'stats'->>'played')::int < (EXCLUDED.attributes->'stats'->>'played')::int - RETURNING id + RETURNING type, id ) - SELECT * FROM insert_matches + SELECT type, id FROM insert_matches UNION - SELECT * FROM insert_players + SELECT type, id FROM insert_players -- cgit v1.3.1