diff options
| -rw-r--r-- | api.py | 12 | ||||
| -rw-r--r-- | crawler.py | 3 | ||||
| -rw-r--r-- | database.py | 28 | ||||
| -rw-r--r-- | queries/recent-matches.sql | 2 |
4 files changed, 7 insertions, 38 deletions
@@ -25,19 +25,17 @@ async def recrawl(): # TODO: insert API version (force update if changed) # TODO: create database indices # get or put when the last crawl was executed - try: - last_match_update = await db.meta("last_match_update") - except KeyError: - last_match_update = datetime.datetime(1, 1, 1).isoformat() - await db.meta("last_match_update", last_match_update) - nowiso = datetime.datetime.now().isoformat() + last_match_update = (await db.select( + "SELECT data->'attributes'->>'createdAt' AS created FROM match ORDER BY data->'attributes'->>'createdAt' DESC LIMIT 1") + )[0]["created"] # crawl and upsert matches = await api.matches_since(last_match_update) if len(matches) > 0: print("got a lot new data items: " + str(len(matches))) + else: + print("got no new matches.") await db.upsert(matches, True) - await db.meta("last_match_update", nowiso) asyncio.ensure_future(recrawl_soon()) @@ -65,6 +65,7 @@ class Crawler(object): while True: params["page[offset]"] += params["page[limit]"] try: + print("asking for more matches…") res = await self._req(session, "shards/" + region + "/matches", params) @@ -85,6 +86,4 @@ class Crawler(object): :return: Processed API response :rtype: list of dict """ - date = ".".join(date.split(".")[:-1]) # remove microseconds - date = date + "Z" return await self.matches(region, {"filter[createdAt-start]": date}) diff --git a/database.py b/database.py index a54f7b5..daab3a3 100644 --- a/database.py +++ b/database.py @@ -75,34 +75,6 @@ class Database(object): tasks.append(task) await asyncio.gather(*tasks) - async def meta(self, key, value=None): - """Sets or gets data into a dictionary-like database object. - - :param key: ID of the value. - :type key: str - :param value: (optional) Value to set. - :type value: object - :return: If `value` is None, the value the database entry. - :rtype: object - """ - async with self._pool.acquire() as conn: - async with conn.transaction(): - await conn.execute("CREATE TABLE IF NOT EXISTS meta " + - "(key TEXT PRIMARY KEY, value jsonb)") - if value is not None: - value = json.dumps(value) - await conn.execute("INSERT INTO meta(key, value) " + - "VALUES ($1, $2) " + - "ON CONFLICT (key) DO " + - "UPDATE SET value=$2", - key, value) - else: - result = await conn.fetch("SELECT value FROM meta " + - "WHERE key='" + key + "'") - if len(result) == 0: - raise KeyError - return json.loads(result[0]["value"]) - async def execute(self, query, args): """Runs an SQL statement. diff --git a/queries/recent-matches.sql b/queries/recent-matches.sql index 9295813..1bde01b 100644 --- a/queries/recent-matches.sql +++ b/queries/recent-matches.sql @@ -27,4 +27,4 @@ SELECT ) FROM match ORDER BY match.data->'attributes'->>'createdAt' DESC -)) AS data;
\ No newline at end of file +)) AS data; |