From 77cb877e849c59de646fa296f192c153b7b6848e Mon Sep 17 00:00:00 2001 From: schneefux Date: Sat, 28 Jan 2017 23:48:40 +0100 Subject: api: get matches from eu+na --- api.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) (limited to 'api.py') diff --git a/api.py b/api.py index 1120c09..5af7ee8 100644 --- a/api.py +++ b/api.py @@ -25,17 +25,27 @@ async def recrawl(): # TODO: insert API version (force update if changed) # TODO: create database indices # get or put when the last crawl was executed - last_match_update = (await db.select( - "SELECT data->'attributes'->>'createdAt' AS created FROM match ORDER BY data->'attributes'->>'createdAt' DESC LIMIT 1") - )[0]["created"] # crawl and upsert - matches = await api.matches_since(last_match_update) - if len(matches) > 0: - print("got a lot new data items: " + str(len(matches))) - else: - print("got no new matches.") - await db.upsert(matches, True) + for region in ["na", "eu"]: + try: + last_match_update = (await db.select( + """ + SELECT data->'attributes'->>'createdAt' AS created + FROM match + WHERE data->'attributes'->>'shardId'='""" + region + """' + ORDER BY data->'attributes'->>'createdAt' DESC LIMIT 1 + """) + )[0]["created"] + except IndexError: + last_match_update = "2017-01-01T01:01:01Z" + + matches = await api.matches_since(last_match_update, region=region) + if len(matches) > 0: + print(region + " got a lot new data items: " + str(len(matches))) + else: + print(region + " got no new matches.") + await db.upsert(matches, True) asyncio.ensure_future(recrawl_soon()) -- cgit v1.3.1