summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorschneefux <schneefux+commit@schneefux.xyz>2017-02-07 22:58:46 +0100
committerschneefux <schneefux+commit@schneefux.xyz>2017-02-07 22:58:46 +0100
commit2de1dd8766382938ded91f887640d5eb87316432 (patch)
tree7877bff44437ee08c35bff9de8c4f9db351ad3b7
parentdc005cf7e1a32c18de955aa73d73bad2e4766020 (diff)
downloadapigrabber-2de1dd8766382938ded91f887640d5eb87316432.tar.gz
apigrabber-2de1dd8766382938ded91f887640d5eb87316432.zip
fix api grabber: run forever
-rw-r--r--api.py26
1 files changed, 14 insertions, 12 deletions
diff --git a/api.py b/api.py
index 77cd37c..2ea8792 100644
--- a/api.py
+++ b/api.py
@@ -16,6 +16,7 @@ async def crawl_region(region):
until the DB is up to date."""
api = crawler.Crawler()
+ # fetch until exhausted
while True:
try:
last_match_update = (await db.select(
@@ -46,21 +47,22 @@ async def crawl_region(region):
async def crawl_forever():
"""Gets the latest matches from all regions every 5 minutes."""
- print("getting recent matches")
+ # repeat forever
+ while True:
+ print("getting recent matches")
- # TODO: insert API version (force update if changed)
- # TODO: create database indices
- # get or put when the last crawl was executed
+ # TODO: insert API version (force update if changed)
+ # TODO: create database indices
+ # get or put when the last crawl was executed
- # crawl and upsert
- tasks = []
- for region in ["na", "eu"]:
- # fire workers
- tasks.append(asyncio.ensure_future(crawl_region(region)))
+ # crawl and upsert
+ tasks = []
+ for region in ["na", "eu"]:
+ # fire workers
+ tasks.append(asyncio.ensure_future(crawl_region(region)))
- await asyncio.gather(*tasks) # wait until all have completed
- await asyncio.sleep(300)
- asyncio.ensure_future(crawl_forever()) # repeat.
+ await asyncio.gather(*tasks) # wait until all have completed
+ await asyncio.sleep(300)
loop = asyncio.get_event_loop()