diff options
| -rw-r--r-- | api.py | 8 | ||||
| -rw-r--r-- | crawler.py | 10 | ||||
| m--------- | joblib | 0 |
3 files changed, 14 insertions, 4 deletions
@@ -69,8 +69,12 @@ class Worker(object): if jobid is None: raise LookupError("no jobs available") logging.debug("%s: starting job", jobid) - await self._execute_job(jobid, payload, priority) - await self._queue.finish(jobid) + try: + await self._execute_job(jobid, payload, priority) + await self._queue.finish(jobid) + except crawler.ApiError as error: + logging.warning("%s: failed", jobid) + await self._queue.fail(jobid, error.args[0]) logging.debug("%s: finished job", jobid) async def run(self): @@ -7,6 +7,9 @@ import aiohttp APIURL = "https://api.dc01.gamelockerapp.com/" +class ApiError(Exception): + pass + class Crawler(object): def __init__(self, token): """Sets constants.""" @@ -40,7 +43,10 @@ class Crawler(object): logging.warning("rate limited, retrying") else: return await response.json() - except (aiohttp.errors.ContentEncodingError): + except (aiohttp.errors.ContentEncodingError, + aiohttp.errors.ServerDisconnectedError, + aiohttp.errors.ClientResponseError, + aiohttp.errors.ClientOSError): # API bug? pass await asyncio.sleep(10) @@ -65,7 +71,7 @@ class Crawler(object): if "errors" in res: logging.warn("API returned error: '%s'", res["errors"]) - break + raise ApiError(res["errors"]) yield res diff --git a/joblib b/joblib -Subproject f7b858a53e0b3233e04186d18d309f8403ea99d +Subproject 31d0588a35c8df17a5d3adcfc25af2b72c896cc |