diff options
| author | schneefux <schneefux+commit@schneefux.xyz> | 2017-02-07 22:58:46 +0100 |
|---|---|---|
| committer | schneefux <schneefux+commit@schneefux.xyz> | 2017-02-07 22:58:46 +0100 |
| commit | 2de1dd8766382938ded91f887640d5eb87316432 (patch) | |
| tree | 7877bff44437ee08c35bff9de8c4f9db351ad3b7 | |
| parent | dc005cf7e1a32c18de955aa73d73bad2e4766020 (diff) | |
| download | apigrabber-2de1dd8766382938ded91f887640d5eb87316432.tar.gz apigrabber-2de1dd8766382938ded91f887640d5eb87316432.zip | |
fix api grabber: run forever
| -rw-r--r-- | api.py | 26 |
1 files changed, 14 insertions, 12 deletions
@@ -16,6 +16,7 @@ async def crawl_region(region): until the DB is up to date.""" api = crawler.Crawler() + # fetch until exhausted while True: try: last_match_update = (await db.select( @@ -46,21 +47,22 @@ async def crawl_region(region): async def crawl_forever(): """Gets the latest matches from all regions every 5 minutes.""" - print("getting recent matches") + # repeat forever + while True: + print("getting recent matches") - # TODO: insert API version (force update if changed) - # TODO: create database indices - # get or put when the last crawl was executed + # TODO: insert API version (force update if changed) + # TODO: create database indices + # get or put when the last crawl was executed - # crawl and upsert - tasks = [] - for region in ["na", "eu"]: - # fire workers - tasks.append(asyncio.ensure_future(crawl_region(region))) + # crawl and upsert + tasks = [] + for region in ["na", "eu"]: + # fire workers + tasks.append(asyncio.ensure_future(crawl_region(region))) - await asyncio.gather(*tasks) # wait until all have completed - await asyncio.sleep(300) - asyncio.ensure_future(crawl_forever()) # repeat. + await asyncio.gather(*tasks) # wait until all have completed + await asyncio.sleep(300) loop = asyncio.get_event_loop() |
