From 90ee943705e6ed373b778929c0bab94c6e34f674 Mon Sep 17 00:00:00 2001
From: schneefux <schneefux+commit@schneefux.xyz>
Date: Sun, 26 Feb 2017 18:35:32 +0100
Subject: report job fails in queue

---
 api.py     | 8 ++++++--
 crawler.py | 5 ++++-
 joblib     | 2 +-
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/api.py b/api.py
index 2dfa964..1da9e8d 100644
--- a/api.py
+++ b/api.py
@@ -69,8 +69,12 @@ class Worker(object):
         if jobid is None:
             raise LookupError("no jobs available")
         logging.debug("%s: starting job", jobid)
-        await self._execute_job(jobid, payload, priority)
-        await self._queue.finish(jobid)
+        try:
+            await self._execute_job(jobid, payload, priority)
+            await self._queue.finish(jobid)
+        except crawler.ApiError as error:
+            logging.warning("%s: failed", jobid)
+            await self._queue.fail(jobid, error.args[0])
         logging.debug("%s: finished job", jobid)
 
     async def run(self):
diff --git a/crawler.py b/crawler.py
index def9f82..44396ae 100644
--- a/crawler.py
+++ b/crawler.py
@@ -7,6 +7,9 @@ import aiohttp
 APIURL = "https://api.dc01.gamelockerapp.com/"
 
 
+class ApiError(Exception):
+    pass
+
 class Crawler(object):
     def __init__(self, token):
         """Sets constants."""
@@ -65,7 +68,7 @@ class Crawler(object):
                 if "errors" in res:
                     logging.warn("API returned error: '%s'",
                                  res["errors"])
-                    break
+                    raise ApiError(res["errors"])
 
                 yield res
 
diff --git a/joblib b/joblib
index f7b858a..31d0588 160000
--- a/joblib
+++ b/joblib
@@ -1 +1 @@
-Subproject commit f7b858a53e0b3233e04186d18d309f8403ea99d1
+Subproject commit 31d0588a35c8df17a5d3adcfc25af2b72c896cc7
-- 
cgit v1.3.1


From b4cbe522fae1e379466b64ac8e3a06c3e45ce26a Mon Sep 17 00:00:00 2001
From: schneefux <schneefux+commit@schneefux.xyz>
Date: Sun, 26 Feb 2017 19:49:19 +0100
Subject: catch more API bugs

---
 crawler.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/crawler.py b/crawler.py
index 44396ae..5db8489 100644
--- a/crawler.py
+++ b/crawler.py
@@ -43,7 +43,10 @@ class Crawler(object):
                         logging.warning("rate limited, retrying")
                     else:
                         return await response.json()
-            except (aiohttp.errors.ContentEncodingError):
+            except (aiohttp.errors.ContentEncodingError,
+                    aiohttp.errors.ServerDisconnectedError,
+                    aiohttp.errors.ClientResponseError,
+                    aiohttp.errors.ClientOSError):
                 # API bug?
                 pass
             await asyncio.sleep(10)
-- 
cgit v1.3.1