From ae779cb6f084e92eaa77277632610548dd6bae9a Mon Sep 17 00:00:00 2001 From: schneefux Date: Mon, 6 Feb 2017 16:16:02 +0100 Subject: crawl regions peux a peux --- crawler.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'crawler.py') diff --git a/crawler.py b/crawler.py index 111f752..dac2303 100644 --- a/crawler.py +++ b/crawler.py @@ -55,10 +55,14 @@ class Crawler(object): :return: Processed API response :rtype: list of dict """ + forever = False # do not fetch until exhausted if params is None: params = dict() - params["page[limit]"] = self._pagelimit - params["page[offset]"] = 0 + if "page[limit]" not in params: + forever = True # no limit specified, fetch all we can + params["page[limit]"] = self._pagelimit + if "page[offset]" not in params: + params["page[offset]"] = 0 data = [] async with aiohttp.ClientSession() as session: @@ -74,16 +78,24 @@ class Crawler(object): data += res["data"] + res["included"] + if not forever: + break # stop after one iteration + return data - async def matches_since(self, date, region="na"): + async def matches_since(self, date, region="na", params=None): """Queries the API for new matches since the given date. :param region: see `matches` :type region: str :param date: Start date in ISO8601 format. :type date: str + :param params: (optional) Additional filters. + :type params: dict :return: Processed API response :rtype: list of dict """ - return await self.matches(region, {"filter[createdAt-start]": date}) + if params is None: + params = dict() + params["filter[createdAt-start]"] = date + return await self.matches(region, params) -- cgit v1.3.1