From c121a662e16c5ff17bb8a0ed64875dd8a1a8f2b0 Mon Sep 17 00:00:00 2001
From: schneefux <schneefux+commit@schneefux.xyz>
Date: Sun, 22 Jan 2017 17:39:26 +0100
Subject: api: rewrite to use async

---
 crawler.py | 53 +++++++++++++++++++++++++++++++++++------------------
 1 file changed, 35 insertions(+), 18 deletions(-)

(limited to 'crawler.py')

diff --git a/crawler.py b/crawler.py
index c9e528d..7ec99b0 100644
--- a/crawler.py
+++ b/crawler.py
@@ -1,11 +1,13 @@
 #!/usr/bin/python
 
-import requests
+import asyncio
 import datetime
+import aiohttp
 
 TOKEN = "aaa.bbb.ccc"
 APIURL = "https://api.dc01.gamelockerapp.com/"
 
+
 class Crawler(object):
     def __init__(self):
         """Sets constants."""
@@ -14,9 +16,11 @@ class Crawler(object):
         self._lastquery = datetime.datetime(1, 1, 1)
         self._pagelimit = 50
 
-    def _req(self, path, params=None):
+    async def _req(self, session, path, params=None):
         """Sends an API request and returns the response dict.
 
+        :param session: aiohttp client session.
+        :type session: :class:`aiohttp.ClientSession`
         :param path: URL path.
         :type path: str
         :param params: (optional) Request parameters.
@@ -30,12 +34,13 @@ class Crawler(object):
             "Accept": "application/vnd.api+json",
             "Content-Encoding": "gzip"
         }
-        http = requests.get(self._apiurl + path, headers=headers,
-                            params=params)
-        http.raise_for_status()
-        return http.json()
+        async with session.get(self._apiurl + path, headers=headers,
+                               params=params) as response:
+            if response.status != 200:
+                return None
+            return await response.json()
 
-    def matches(self, region="na", params=None):
+    async def matches(self, region="na", params=None):
         """Queries the API for matches and their related data.
 
         :param region: (optional) Region where the matches were played.
@@ -48,20 +53,32 @@ class Crawler(object):
         """
         if params is None:
             params = dict()
-        resp = []
         params["page[limit]"] = self._pagelimit
         params["page[offset]"] = 0
-        while True:
-            # go one page forward until 404
-            try:
-                json = self._req("shards/" + region + "/matches", params)
-            except requests.exceptions.HTTPError:
-                break
-            resp += json["data"] + json["included"]
-            params["page[offset]"] += params["page[limit]"]
-        return resp
+        batchsize = 100  # FIXME We don't know how long we can paginate
+
+        tasks = []
+        data = []
+        # fire a lot of http requests
+        async with aiohttp.ClientSession() as session:
+            for _ in range(0, batchsize):
+                params["page[offset]"] += params["page[limit]"]
+                task = asyncio.ensure_future(
+                    self._req(
+                        session,
+                        "shards/" + region + "/matches",
+                        params))
+                tasks.append(task)
+
+            results = await asyncio.gather(*tasks)
+            for res in results:
+                if res is None:
+                    continue
+                data += res["data"] + res["included"]
+
+        return data
 
-    def matches_new(self, region="na"):
+    async def matches_new(self, region="na"):
         """Queries the API for new matches since the last query.
 
         :param region: see `matches`
-- 
cgit v1.3.1