Merge branch 'errors'

3 files changed, 14 insertions, 4 deletions

diff --git a/api.py b/api.py
index 2dfa964..1da9e8d 100644
--- a/api.py
+++ b/api.py
@@ -69,8 +69,12 @@ class Worker(object):
         if jobid is None:
             raise LookupError("no jobs available")
         logging.debug("%s: starting job", jobid)
-        await self._execute_job(jobid, payload, priority)
-        await self._queue.finish(jobid)
+        try:
+            await self._execute_job(jobid, payload, priority)
+            await self._queue.finish(jobid)
+        except crawler.ApiError as error:
+            logging.warning("%s: failed", jobid)
+            await self._queue.fail(jobid, error.args[0])
         logging.debug("%s: finished job", jobid)
 
     async def run(self):
diff --git a/crawler.py b/crawler.py
index 7e9c862..f31c99a 100644
--- a/crawler.py
+++ b/crawler.py
@@ -7,6 +7,9 @@ import aiohttp
 APIURL = "https://api.dc01.gamelockerapp.com/"
 
 
+class ApiError(Exception):
+    pass
+
 class Crawler(object):
     def __init__(self, token):
         """Sets constants."""
@@ -40,7 +43,10 @@ class Crawler(object):
                         logging.warning("rate limited, retrying")
                     else:
                         return await response.json()
-            except (aiohttp.errors.ContentEncodingError):
+            except (aiohttp.errors.ContentEncodingError,
+                    aiohttp.errors.ServerDisconnectedError,
+                    aiohttp.errors.ClientResponseError,
+                    aiohttp.errors.ClientOSError):
                 # API bug?
                 pass
             await asyncio.sleep(10)
@@ -65,7 +71,7 @@ class Crawler(object):
                 if "errors" in res:
                     logging.warn("API returned error: '%s'",
                                  res["errors"])
-                    break
+                    raise ApiError(res["errors"])
 
                 yield res
 
diff --git a/joblib b/joblib
-Subproject f7b858a53e0b3233e04186d18d309f8403ea99d
+Subproject 31d0588a35c8df17a5d3adcfc25af2b72c896cc