From 522613735f395e555cf4bd4713ee910e4367f731 Mon Sep 17 00:00:00 2001 From: schneefux Date: Tue, 14 Feb 2017 18:59:21 +0100 Subject: workers sleep and don't die when they have no jobs --- api.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'api.py') diff --git a/api.py b/api.py index 299e0f3..39788a3 100644 --- a/api.py +++ b/api.py @@ -148,7 +148,7 @@ class Apigrabber(object): async with con.transaction(isolation="serializable"): # select us our job - jobdate, delta_minutes = await con.fetchrow(""" + row_res = await con.fetchrow(""" SELECT start_date, -- new job's end date LEAST(EXTRACT(EPOCH FROM (start_date-previous_end))/60, $2) -- gap in minutes or default if smaller @@ -163,6 +163,12 @@ class Apigrabber(object): WHERE start_date-previous_end>INTERVAL '0' -- get gaps ORDER BY start_date DESC LIMIT 1 """, region, default_diff) + if row_res == None: + logging.warn("%s: no jobs available. idling.", region) + await asyncio.sleep(60) # a minute TODO make this smarter + break + + jobdate, delta_minutes = row_res delta = datetime.timedelta(minutes=delta_minutes) # store our job as pending jobid = await con.fetchval(""" @@ -211,6 +217,7 @@ class Apigrabber(object): # TODO: respawn a worker if it dies because of connection issues # TODO: insert API version (force update if changed) # TODO: create database indices (id & shardId & type) + # TODO: make workers switch regions flexibly to meet demand? for region in self.regions: await self.request_update(region) -- cgit v1.3.1