From e1b0c99706fc7c0d1b95f4e047b9ac37539a59cb Mon Sep 17 00:00:00 2001 From: schneefux Date: Sat, 25 Feb 2017 13:44:19 +0100 Subject: respect rate limits --- crawler.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) (limited to 'crawler.py') diff --git a/crawler.py b/crawler.py index b68ec22..b188a86 100644 --- a/crawler.py +++ b/crawler.py @@ -33,11 +33,18 @@ class Crawler(object): "Content-Encoding": "gzip" } try: - async with session.get(self._apiurl + path, headers=headers, - params=params) as response: - assert response.status == 200 - return await response.json() - except (aiohttp.errors.ClientResponseError, RuntimeError): + while True: + async with session.get(self._apiurl + path, headers=headers, + params=params) as response: + if response.status == 429: + logging.warning("hit by rate limit, retrying") + await asyncio.sleep(10) + continue + assert response.status == 200 + return await response.json() + except (aiohttp.errors.ClientResponseError, + RuntimeError, + aiohttp.errors.ContentEncodingError): logging.warning("error connecting to API, retrying") return await self._req(session, path, params) @@ -81,6 +88,10 @@ class Crawler(object): data += res["data"] + res["included"] + if len(res["data"]) < 50: + # asked for 50, got less -> exhausted + break + if not forever: break # stop after one iteration -- cgit v1.3.1