From 2bdc6cf44f6c0c62862c4ac5da3a02ac0f7250d6 Mon Sep 17 00:00:00 2001 From: schneefux Date: Wed, 8 Feb 2017 17:16:45 +0100 Subject: use logging module --- api.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'api.py') diff --git a/api.py b/api.py index 2ea8792..08b25fb 100644 --- a/api.py +++ b/api.py @@ -2,6 +2,7 @@ import asyncio import os +import logging import database import crawler @@ -10,7 +11,6 @@ import crawler db = database.Database() -# TODO use logging module instead of print async def crawl_region(region): """Gets some matches from a region and inserts them until the DB is up to date.""" @@ -30,16 +30,16 @@ async def crawl_region(region): except: last_match_update = "2017-02-05T01:01:01Z" - print(region + " fetching matches after " + last_match_update) + logging.info("%s: fetching matches since %s", region, last_match_update) # wait for http requests matches = await api.matches_since(last_match_update, region=region, params={"page[limit]": 50}) if len(matches) > 0: - print(region + " got new data items: " + str(len(matches))) + logging.debug("%s: %s objects", region, len(matches)) else: - print(region + " got no new matches.") + logging.debug("%s: no objects, stopping", region) return # insert asynchronously in the background await db.upsert(matches, True) @@ -49,7 +49,7 @@ async def crawl_forever(): """Gets the latest matches from all regions every 5 minutes.""" # repeat forever while True: - print("getting recent matches") + logging.info("pulling recent matches") # TODO: insert API version (force update if changed) # TODO: create database indices @@ -65,6 +65,7 @@ async def crawl_forever(): await asyncio.sleep(300) +logging.basicConfig(level=logging.DEBUG) loop = asyncio.get_event_loop() loop.run_until_complete(db.connect( host=os.environ["POSTGRESQL_HOST"], -- cgit v1.3.1 From 3513b689ca2cc625b5be591f64701016b8d61d04 Mon Sep 17 00:00:00 2001 From: schneefux Date: Wed, 8 Feb 2017 17:59:31 +0100 Subject: refactor api.py --- api.py | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) (limited to 'api.py') diff --git a/api.py b/api.py index 08b25fb..86c7e35 100644 --- a/api.py +++ b/api.py @@ -12,8 +12,8 @@ db = database.Database() async def crawl_region(region): - """Gets some matches from a region and inserts them - until the DB is up to date.""" + """Get matches from a region and insert them + until the DB is up to date. Repeat after five minutes.""" api = crawler.Crawler() # fetch until exhausted @@ -30,7 +30,8 @@ async def crawl_region(region): except: last_match_update = "2017-02-05T01:01:01Z" - logging.info("%s: fetching matches since %s", region, last_match_update) + logging.info("%s: fetching matches since %s", + region, last_match_update) # wait for http requests matches = await api.matches_since(last_match_update, @@ -40,29 +41,23 @@ async def crawl_region(region): logging.debug("%s: %s objects", region, len(matches)) else: logging.debug("%s: no objects, stopping", region) - return - # insert asynchronously in the background + break + # wait for db inserts await db.upsert(matches, True) + logging.debug("%s: going to sleep", region) + await asyncio.sleep(300) + asyncio.ensure_future(crawl_region(region)) # restart self -async def crawl_forever(): - """Gets the latest matches from all regions every 5 minutes.""" - # repeat forever - while True: - logging.info("pulling recent matches") - - # TODO: insert API version (force update if changed) - # TODO: create database indices - # get or put when the last crawl was executed - # crawl and upsert - tasks = [] - for region in ["na", "eu"]: - # fire workers - tasks.append(asyncio.ensure_future(crawl_region(region))) +async def start_crawlers(): + """Start the tasks that pull the data.""" + # TODO: insert API version (force update if changed) + # TODO: create database indices - await asyncio.gather(*tasks) # wait until all have completed - await asyncio.sleep(300) + for region in ["na", "eu"]: + # fire workers + asyncio.ensure_future(crawl_region(region)) logging.basicConfig(level=logging.DEBUG) @@ -75,5 +70,6 @@ loop.run_until_complete(db.connect( database=os.environ["POSTGRESQL_DB"] )) loop.run_until_complete( - crawl_forever() + start_crawlers() ) +loop.run_forever() -- cgit v1.3.1 From daa2c59cd8b56d4f5bdec85d7421cf8f7827a908 Mon Sep 17 00:00:00 2001 From: schneefux Date: Wed, 8 Feb 2017 18:01:26 +0100 Subject: add all regions --- api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'api.py') diff --git a/api.py b/api.py index 86c7e35..9e5a8d7 100644 --- a/api.py +++ b/api.py @@ -28,7 +28,7 @@ async def crawl_region(region): """) )[0]["created"] except: - last_match_update = "2017-02-05T01:01:01Z" + last_match_update = "2017-02-07T01:01:01Z" # TODO logging.info("%s: fetching matches since %s", region, last_match_update) @@ -55,7 +55,7 @@ async def start_crawlers(): # TODO: insert API version (force update if changed) # TODO: create database indices - for region in ["na", "eu"]: + for region in ["na", "eu", "sg", "ea", "sa", "cn"]: # fire workers asyncio.ensure_future(crawl_region(region)) -- cgit v1.3.1 From 64c9a5de92934a1f779a747123f433bb2c7e1bcf Mon Sep 17 00:00:00 2001 From: schneefux Date: Wed, 8 Feb 2017 18:56:30 +0100 Subject: tolerate connection errors --- api.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'api.py') diff --git a/api.py b/api.py index 9e5a8d7..7843f12 100644 --- a/api.py +++ b/api.py @@ -34,9 +34,14 @@ async def crawl_region(region): region, last_match_update) # wait for http requests - matches = await api.matches_since(last_match_update, - region=region, - params={"page[limit]": 50}) + try: + matches = await api.matches_since(last_match_update, + region=region, + params={"page[limit]": 50}) + except: + logging.error("%s: connection error, retrying", region) + await asyncio.sleep(5) + if len(matches) > 0: logging.debug("%s: %s objects", region, len(matches)) else: -- cgit v1.3.1