From 068035760ca3abd22b67d0b53bf73f2e83209b53 Mon Sep 17 00:00:00 2001
From: schneefux <schneefux+commit@schneefux.xyz>
Date: Mon, 23 Jan 2017 12:58:12 +0100
Subject: api: send requests synchronously

---
 crawler.py | 38 +++++++++++++++-----------------------
 1 file changed, 15 insertions(+), 23 deletions(-)

(limited to 'crawler.py')

diff --git a/crawler.py b/crawler.py
index 7ec99b0..96eb8ce 100644
--- a/crawler.py
+++ b/crawler.py
@@ -1,7 +1,6 @@
 #!/usr/bin/python
 
 import asyncio
-import datetime
 import aiohttp
 
 TOKEN = "aaa.bbb.ccc"
@@ -13,7 +12,6 @@ class Crawler(object):
         """Sets constants."""
         self._apiurl = APIURL
         self._token = TOKEN
-        self._lastquery = datetime.datetime(1, 1, 1)
         self._pagelimit = 50
 
     async def _req(self, session, path, params=None):
@@ -36,8 +34,7 @@ class Crawler(object):
         }
         async with session.get(self._apiurl + path, headers=headers,
                                params=params) as response:
-            if response.status != 200:
-                return None
+            assert response.status == 200
             return await response.json()
 
     async def matches(self, region="na", params=None):
@@ -55,35 +52,30 @@ class Crawler(object):
             params = dict()
         params["page[limit]"] = self._pagelimit
         params["page[offset]"] = 0
-        batchsize = 100  # FIXME We don't know how long we can paginate
 
-        tasks = []
         data = []
-        # fire a lot of http requests
         async with aiohttp.ClientSession() as session:
-            for _ in range(0, batchsize):
+            while True:
                 params["page[offset]"] += params["page[limit]"]
-                task = asyncio.ensure_future(
-                    self._req(
-                        session,
-                        "shards/" + region + "/matches",
-                        params))
-                tasks.append(task)
+                try:
+                    res = await self._req(session,
+                                          "shards/" + region + "/matches",
+                                          params)
+                except AssertionError:
+                    break
 
-            results = await asyncio.gather(*tasks)
-            for res in results:
-                if res is None:
-                    continue
                 data += res["data"] + res["included"]
 
         return data
 
-    async def matches_new(self, region="na"):
-        """Queries the API for new matches since the last query.
+    async def matches_since(self, date, region="na"):
+        """Queries the API for new matches since the given date.
 
         :param region: see `matches`
         :type region: str
+        :param date: Start date in ISO8601 format.
+        :type date: str
+        :return: Processed API response
+        :rtype: list of dict
         """
-        lastdate = self._lastquery.replace(microsecond=0).isoformat() + "Z"
-        self._lastquery = datetime.datetime.now()
-        return self.matches(region, {"filter[createdAt-start]": lastdate})
+        return await self.matches(region, {"filter[createdAt-start]": date})
-- 
cgit v1.3.1