From 9535cab8d794a01134b8dccfbf54ebf3eac65037 Mon Sep 17 00:00:00 2001
From: schneefux <schneefux+commit@schneefux.xyz>
Date: Sat, 28 Jan 2017 16:41:01 +0100
Subject: api: get new matches more reliably, remove now unused `db.meta`

---
 api.py                     | 12 +++++-------
 crawler.py                 |  3 +--
 database.py                | 28 ----------------------------
 queries/recent-matches.sql |  2 +-
 4 files changed, 7 insertions(+), 38 deletions(-)

diff --git a/api.py b/api.py
index 46c8f72..1120c09 100644
--- a/api.py
+++ b/api.py
@@ -25,19 +25,17 @@ async def recrawl():
     # TODO: insert API version (force update if changed)
     # TODO: create database indices
     # get or put when the last crawl was executed
-    try:
-        last_match_update = await db.meta("last_match_update")
-    except KeyError:
-        last_match_update = datetime.datetime(1, 1, 1).isoformat()
-        await db.meta("last_match_update", last_match_update)
-    nowiso = datetime.datetime.now().isoformat()
+    last_match_update = (await db.select(
+        "SELECT data->'attributes'->>'createdAt' AS created FROM match ORDER BY data->'attributes'->>'createdAt' DESC LIMIT 1")
+    )[0]["created"]
 
     # crawl and upsert
     matches = await api.matches_since(last_match_update)
     if len(matches) > 0:
         print("got a lot new data items: " + str(len(matches)))
+    else:
+        print("got no new matches.")
     await db.upsert(matches, True)
-    await db.meta("last_match_update", nowiso)
 
     asyncio.ensure_future(recrawl_soon())
 
diff --git a/crawler.py b/crawler.py
index 68b994f..111f752 100644
--- a/crawler.py
+++ b/crawler.py
@@ -65,6 +65,7 @@ class Crawler(object):
             while True:
                 params["page[offset]"] += params["page[limit]"]
                 try:
+                    print("asking for more matches…")
                     res = await self._req(session,
                                           "shards/" + region + "/matches",
                                           params)
@@ -85,6 +86,4 @@ class Crawler(object):
         :return: Processed API response
         :rtype: list of dict
         """
-        date = ".".join(date.split(".")[:-1])  # remove microseconds
-        date = date + "Z"
         return await self.matches(region, {"filter[createdAt-start]": date})
diff --git a/database.py b/database.py
index a54f7b5..daab3a3 100644
--- a/database.py
+++ b/database.py
@@ -75,34 +75,6 @@ class Database(object):
             tasks.append(task)
         await asyncio.gather(*tasks)
 
-    async def meta(self, key, value=None):
-        """Sets or gets data into a dictionary-like database object.
-
-        :param key: ID of the value.
-        :type key: str
-        :param value: (optional) Value to set.
-        :type value: object
-        :return: If `value` is None, the value the database entry.
-        :rtype: object
-        """
-        async with self._pool.acquire() as conn:
-            async with conn.transaction():
-                await conn.execute("CREATE TABLE IF NOT EXISTS meta " +
-                                   "(key TEXT PRIMARY KEY, value jsonb)")
-                if value is not None:
-                    value = json.dumps(value)
-                    await conn.execute("INSERT INTO meta(key, value) " +
-                                       "VALUES ($1, $2) " +
-                                       "ON CONFLICT (key) DO " +
-                                       "UPDATE SET value=$2",
-                                       key, value)
-                else:
-                    result = await conn.fetch("SELECT value FROM meta " +
-                                              "WHERE key='" + key + "'")
-                    if len(result) == 0:
-                        raise KeyError
-                    return json.loads(result[0]["value"])
-
     async def execute(self, query, args):
         """Runs an SQL statement.
 
diff --git a/queries/recent-matches.sql b/queries/recent-matches.sql
index 9295813..1bde01b 100644
--- a/queries/recent-matches.sql
+++ b/queries/recent-matches.sql
@@ -27,4 +27,4 @@ SELECT
 )
 FROM match
 ORDER BY match.data->'attributes'->>'createdAt' DESC
-)) AS data;
\ No newline at end of file
+)) AS data;
-- 
cgit v1.3.1