From 1c550fdc51d16ec15c39edb346c28d5f07a2a425 Mon Sep 17 00:00:00 2001
From: schneefux <schneefux+commit@schneefux.xyz>
Date: Sat, 25 Feb 2017 13:43:39 +0100
Subject: add VAINSOCIAL_APIKEY env variable

---
 api.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'api.py')

diff --git a/api.py b/api.py
index 1cfa918..89573ce 100644
--- a/api.py
+++ b/api.py
@@ -28,7 +28,9 @@ def iso2date(d):
 
 
 class Apigrabber(object):
-    def __init__(self, regions, first_fetch=None, last_fetch=None):
+    def __init__(self, regions, apitoken,
+                 first_fetch=None, last_fetch=None):
+        self._apitoken = apitoken
         self.update_live = False
         # TODO refactor this
         if first_fetch is None:
@@ -138,7 +140,7 @@ class Apigrabber(object):
     async def crawl_timeframe(self, region, jobid, jobstart, jobend):
         """Crawl a time frame forwards from `date` in `region`."""
         async with self._pool.acquire() as con:
-            api = crawler.Crawler()
+            api = crawler.Crawler(self._apitoken)
             async with con.transaction():
                 params = {
                     "filter[createdAt-start]": date2iso(jobstart),
@@ -267,6 +269,7 @@ class Apigrabber(object):
 
 async def startup():
     apigrabber = Apigrabber(
+        apitoken=os.environ["VAINSOCIAL_APITOKEN"],
         regions=os.environ["VAINSOCIAL_REGIONS"].split(","),
         first_fetch=os.environ.get("VAINSOCIAL_STARTDATE"),
         last_fetch=os.environ.get("VAINSOCIAL_ENDDATE")
-- 
cgit v1.3.1


From e1b0c99706fc7c0d1b95f4e047b9ac37539a59cb Mon Sep 17 00:00:00 2001
From: schneefux <schneefux+commit@schneefux.xyz>
Date: Sat, 25 Feb 2017 13:44:19 +0100
Subject: respect rate limits

---
 api.py     |  4 ++--
 crawler.py | 21 ++++++++++++++++-----
 2 files changed, 18 insertions(+), 7 deletions(-)

(limited to 'api.py')

diff --git a/api.py b/api.py
index 89573ce..45eb5ba 100644
--- a/api.py
+++ b/api.py
@@ -256,7 +256,7 @@ class Apigrabber(object):
 
         for region in self.regions:
             await self.request_update(region)
-            for _ in range(5):
+            for _ in range(1):
                 # supports scaling :]
                 asyncio.ensure_future(self.crawl_region(region))
 
@@ -288,4 +288,4 @@ logging.basicConfig(level=logging.DEBUG)
 loop = asyncio.get_event_loop()
 loop.run_until_complete(startup())
 loop.run_forever()
- 
\ No newline at end of file
+ 
diff --git a/crawler.py b/crawler.py
index b68ec22..b188a86 100644
--- a/crawler.py
+++ b/crawler.py
@@ -33,11 +33,18 @@ class Crawler(object):
             "Content-Encoding": "gzip"
         }
         try:
-            async with session.get(self._apiurl + path, headers=headers,
-                                   params=params) as response:
-                assert response.status == 200
-                return await response.json()
-        except (aiohttp.errors.ClientResponseError, RuntimeError):
+            while True:
+                async with session.get(self._apiurl + path, headers=headers,
+                                       params=params) as response:
+                    if response.status == 429:
+                        logging.warning("hit by rate limit, retrying")
+                        await asyncio.sleep(10)
+                        continue
+                    assert response.status == 200
+                    return await response.json()
+        except (aiohttp.errors.ClientResponseError,
+                RuntimeError,
+                aiohttp.errors.ContentEncodingError):
             logging.warning("error connecting to API, retrying")
             return await self._req(session, path, params)
 
@@ -81,6 +88,10 @@ class Crawler(object):
 
                 data += res["data"] + res["included"]
 
+                if len(res["data"]) < 50:
+                    # asked for 50, got less -> exhausted
+                    break
+
                 if not forever:
                     break  # stop after one iteration
 
-- 
cgit v1.3.1