crawl regions peux a peux

2 files changed, 45 insertions, 21 deletions

diff --git a/api.py b/api.py
index dff8ebf..9ba1300 100644
--- a/api.py
+++ b/api.py
@@ -17,21 +17,17 @@ route = aiohttp_route_decorator.RouteCollector()
 db = database.Database()
 
 
-async def recrawl():
-    """Gets the latest matches and inserts them into the database."""
-    print("getting recent matches")
+# TODO use logging module instead of print
+async def crawl_region(region):
+    """Gets some matches from a region and inserts them
+       until the DB is up to date."""
     api = crawler.Crawler()
 
-    # TODO: insert API version (force update if changed)
-    # TODO: create database indices
-    # get or put when the last crawl was executed
-
-    # crawl and upsert
-    for region in ["na", "eu"]:
+    while True:
         try:
             last_match_update = (await db.select(
                 """
-                SELECT data->'attributes'->>'createdAt' AS created 
+                SELECT data->'attributes'->>'createdAt' AS created
                 FROM match
                 WHERE data->'attributes'->>'shardId'='""" + region + """'
                 ORDER BY data->'attributes'->>'createdAt' DESC LIMIT 1
@@ -40,19 +36,35 @@ async def recrawl():
         except:
             last_match_update = "2017-02-05T01:01:01Z"
 
-        matches = await api.matches_since(last_match_update, region=region)
+        print(region + " fetching matches after " + last_match_update)
+
+        # wait for http requests
+        matches = await api.matches_since(last_match_update,
+                                          region=region,
+                                          params={"page[limit]": 50})
         if len(matches) > 0:
-            print(region + " got a lot new data items: " + str(len(matches)))
+            print(region + " got new data items: " + str(len(matches)))
         else:
             print(region + " got no new matches.")
+            return
+        # insert asynchronously in the background
         await db.upsert(matches, True)
 
-    asyncio.ensure_future(recrawl_soon())
 
-async def recrawl_soon():
-    """Calls `recrawl` after 60 seconds."""
-    print("crawler sleeping, Zzzzzz…")
-    await asyncio.sleep(60)
+async def recrawl():
+    """Gets the latest matches from all regions every 5 minutes."""
+    print("getting recent matches")
+
+    # TODO: insert API version (force update if changed)
+    # TODO: create database indices
+    # get or put when the last crawl was executed
+
+    # crawl and upsert
+    for region in ["na", "eu"]:
+        # fire workers
+        asyncio.ensure_future(crawl_region(region))
+
+    await asyncio.sleep(300)
     asyncio.ensure_future(recrawl())
 
 
diff --git a/crawler.py b/crawler.py
index 111f752..dac2303 100644
--- a/crawler.py
+++ b/crawler.py
@@ -55,10 +55,14 @@ class Crawler(object):
         :return: Processed API response
         :rtype: list of dict
         """
+        forever = False  # do not fetch until exhausted
         if params is None:
             params = dict()
-        params["page[limit]"] = self._pagelimit
-        params["page[offset]"] = 0
+        if "page[limit]" not in params:
+            forever = True  # no limit specified, fetch all we can
+            params["page[limit]"] = self._pagelimit
+        if "page[offset]" not in params:
+            params["page[offset]"] = 0
 
         data = []
         async with aiohttp.ClientSession() as session:
@@ -74,16 +78,24 @@ class Crawler(object):
 
                 data += res["data"] + res["included"]
 
+                if not forever:
+                    break  # stop after one iteration
+
         return data
 
-    async def matches_since(self, date, region="na"):
+    async def matches_since(self, date, region="na", params=None):
         """Queries the API for new matches since the given date.
 
         :param region: see `matches`
         :type region: str
         :param date: Start date in ISO8601 format.
         :type date: str
+        :param params: (optional) Additional filters.
+        :type params: dict
         :return: Processed API response
         :rtype: list of dict
         """
-        return await self.matches(region, {"filter[createdAt-start]": date})
+        if params is None:
+            params = dict()
+        params["filter[createdAt-start]"] = date
+        return await self.matches(region, params)