diff options
| -rw-r--r-- | api.py | 11 | ||||
| -rw-r--r-- | crawler.py | 28 |
2 files changed, 26 insertions, 13 deletions
@@ -28,7 +28,9 @@ def iso2date(d): class Apigrabber(object): - def __init__(self, regions, first_fetch=None, last_fetch=None): + def __init__(self, regions, apitoken, + first_fetch=None, last_fetch=None): + self._apitoken = apitoken self.update_live = False # TODO refactor this if first_fetch is None: @@ -138,7 +140,7 @@ class Apigrabber(object): async def crawl_timeframe(self, region, jobid, jobstart, jobend): """Crawl a time frame forwards from `date` in `region`.""" async with self._pool.acquire() as con: - api = crawler.Crawler() + api = crawler.Crawler(self._apitoken) async with con.transaction(): params = { "filter[createdAt-start]": date2iso(jobstart), @@ -254,7 +256,7 @@ class Apigrabber(object): for region in self.regions: await self.request_update(region) - for _ in range(5): + for _ in range(1): # supports scaling :] asyncio.ensure_future(self.crawl_region(region)) @@ -267,6 +269,7 @@ class Apigrabber(object): async def startup(): apigrabber = Apigrabber( + apitoken=os.environ["VAINSOCIAL_APITOKEN"], regions=os.environ["VAINSOCIAL_REGIONS"].split(","), first_fetch=os.environ.get("VAINSOCIAL_STARTDATE"), last_fetch=os.environ.get("VAINSOCIAL_ENDDATE") @@ -285,4 +288,4 @@ logging.basicConfig(level=logging.DEBUG) loop = asyncio.get_event_loop() loop.run_until_complete(startup()) loop.run_forever() -
\ No newline at end of file + @@ -4,15 +4,14 @@ import asyncio import logging import aiohttp -TOKEN = "aaa.bbb.ccc" APIURL = "https://api.dc01.gamelockerapp.com/" class Crawler(object): - def __init__(self): + def __init__(self, token): """Sets constants.""" self._apiurl = APIURL - self._token = TOKEN + self._token = token self._pagelimit = 50 async def _req(self, session, path, params=None): @@ -31,14 +30,21 @@ class Crawler(object): "Authorization": "Bearer " + self._token, "X-TITLE-ID": "semc-vainglory", "Accept": "application/vnd.api+json", - "Content-Encoding": "gzip" + "Accept-Encoding": "gzip" } try: - async with session.get(self._apiurl + path, headers=headers, - params=params) as response: - assert response.status == 200 - return await response.json() - except (aiohttp.errors.ClientResponseError, RuntimeError): + while True: + async with session.get(self._apiurl + path, headers=headers, + params=params) as response: + if response.status == 429: + logging.warning("hit by rate limit, retrying") + await asyncio.sleep(10) + continue + assert response.status == 200 + return await response.json() + except (aiohttp.errors.ClientResponseError, + RuntimeError, + aiohttp.errors.ContentEncodingError): logging.warning("error connecting to API, retrying") return await self._req(session, path, params) @@ -82,6 +88,10 @@ class Crawler(object): data += res["data"] + res["included"] + if len(res["data"]) < 50: + # asked for 50, got less -> exhausted + break + if not forever: break # stop after one iteration |