From 90ee943705e6ed373b778929c0bab94c6e34f674 Mon Sep 17 00:00:00 2001 From: schneefux Date: Sun, 26 Feb 2017 18:35:32 +0100 Subject: report job fails in queue --- crawler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'crawler.py') diff --git a/crawler.py b/crawler.py index def9f82..44396ae 100644 --- a/crawler.py +++ b/crawler.py @@ -7,6 +7,9 @@ import aiohttp APIURL = "https://api.dc01.gamelockerapp.com/" +class ApiError(Exception): + pass + class Crawler(object): def __init__(self, token): """Sets constants.""" @@ -65,7 +68,7 @@ class Crawler(object): if "errors" in res: logging.warn("API returned error: '%s'", res["errors"]) - break + raise ApiError(res["errors"]) yield res -- cgit v1.3.1 From b4cbe522fae1e379466b64ac8e3a06c3e45ce26a Mon Sep 17 00:00:00 2001 From: schneefux Date: Sun, 26 Feb 2017 19:49:19 +0100 Subject: catch more API bugs --- crawler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'crawler.py') diff --git a/crawler.py b/crawler.py index 44396ae..5db8489 100644 --- a/crawler.py +++ b/crawler.py @@ -43,7 +43,10 @@ class Crawler(object): logging.warning("rate limited, retrying") else: return await response.json() - except (aiohttp.errors.ContentEncodingError): + except (aiohttp.errors.ContentEncodingError, + aiohttp.errors.ServerDisconnectedError, + aiohttp.errors.ClientResponseError, + aiohttp.errors.ClientOSError): # API bug? pass await asyncio.sleep(10) -- cgit v1.3.1