diff options
Diffstat (limited to 'api.py')
| -rw-r--r-- | api.py | 9 |
1 files changed, 8 insertions, 1 deletions
@@ -148,7 +148,7 @@ class Apigrabber(object): async with con.transaction(isolation="serializable"): # select us our job - jobdate, delta_minutes = await con.fetchrow(""" + row_res = await con.fetchrow(""" SELECT start_date, -- new job's end date LEAST(EXTRACT(EPOCH FROM (start_date-previous_end))/60, $2) -- gap in minutes or default if smaller @@ -163,6 +163,12 @@ class Apigrabber(object): WHERE start_date-previous_end>INTERVAL '0' -- get gaps ORDER BY start_date DESC LIMIT 1 """, region, default_diff) + if row_res == None: + logging.warn("%s: no jobs available. idling.", region) + await asyncio.sleep(60) # a minute TODO make this smarter + break + + jobdate, delta_minutes = row_res delta = datetime.timedelta(minutes=delta_minutes) # store our job as pending jobid = await con.fetchval(""" @@ -211,6 +217,7 @@ class Apigrabber(object): # TODO: respawn a worker if it dies because of connection issues # TODO: insert API version (force update if changed) # TODO: create database indices (id & shardId & type) + # TODO: make workers switch regions flexibly to meet demand? for region in self.regions: await self.request_update(region) |