diff options
| -rw-r--r-- | .gitignore | 91 | ||||
| -rw-r--r-- | __init__.py | 0 | ||||
| -rw-r--r-- | api.py | 10 | ||||
| -rw-r--r-- | crawler.py | 72 | ||||
| -rw-r--r-- | database.py | 83 |
5 files changed, 256 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9a05e2d --- /dev/null +++ b/.gitignore @@ -0,0 +1,91 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +.venv/ +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/python + +import database +import crawler + +db = database.Database("dbname=vgstats user=vgstats") +api = crawler.Crawler() +#db.upsert("na", api.matches(params={"filter[playerNames]": "fallingcomets"}), True) +db.upsert("na", api.matches(), True) +print(db.select("SELECT data->'attributes'->>'name' as name FROM player WHERE CAST(data->'attributes'->'stats'->>'winStreak' AS integer) > 10")) diff --git a/crawler.py b/crawler.py new file mode 100644 index 0000000..c9e528d --- /dev/null +++ b/crawler.py @@ -0,0 +1,72 @@ +#!/usr/bin/python + +import requests +import datetime + +TOKEN = "aaa.bbb.ccc" +APIURL = "https://api.dc01.gamelockerapp.com/" + +class Crawler(object): + def __init__(self): + """Sets constants.""" + self._apiurl = APIURL + self._token = TOKEN + self._lastquery = datetime.datetime(1, 1, 1) + self._pagelimit = 50 + + def _req(self, path, params=None): + """Sends an API request and returns the response dict. + + :param path: URL path. + :type path: str + :param params: (optional) Request parameters. + :type params: dict + :return: API response. + :rtype: dict + """ + headers = { + "Authorization": "Bearer " + self._token, + "X-TITLE-ID": "semc-vainglory", + "Accept": "application/vnd.api+json", + "Content-Encoding": "gzip" + } + http = requests.get(self._apiurl + path, headers=headers, + params=params) + http.raise_for_status() + return http.json() + + def matches(self, region="na", params=None): + """Queries the API for matches and their related data. + + :param region: (optional) Region where the matches were played. + Defaults to "na" (North America). + :type region: str + :param params: (optional) Additional filters. + :type params: dict + :return: Processed API response + :rtype: list of dict + """ + if params is None: + params = dict() + resp = [] + params["page[limit]"] = self._pagelimit + params["page[offset]"] = 0 + while True: + # go one page forward until 404 + try: + json = self._req("shards/" + region + "/matches", params) + except requests.exceptions.HTTPError: + break + resp += json["data"] + json["included"] + params["page[offset]"] += params["page[limit]"] + return resp + + def matches_new(self, region="na"): + """Queries the API for new matches since the last query. + + :param region: see `matches` + :type region: str + """ + lastdate = self._lastquery.replace(microsecond=0).isoformat() + "Z" + self._lastquery = datetime.datetime.now() + return self.matches(region, {"filter[createdAt-start]": lastdate}) diff --git a/database.py b/database.py new file mode 100644 index 0000000..396701a --- /dev/null +++ b/database.py @@ -0,0 +1,83 @@ +#!/usr/bin/python + +import psycopg2 +import psycopg2.extras + +class Database(object): + """Database wrapper class""" + def __init__(self, connstring): + """Connects to the database. + + :param connstring: Connection string containing user and database. + :type connstring: str + """ + self._connection = psycopg2.connect(connstring) + self._c = self._connection.cursor() + + def upsert_type(self, shard, json, objtype, many=False): + """Upserts an object of given `objtype` into the corresponding database. + + :param shard: Region in which an object's id is unique. + :type shard: str + :param json: Object to upsert. + :type json: dict + :param objtype: Object type and table name. + :type objtype: str + :param many: (optional) Whether `json` is an array + of objects of the same objtype. + :type many: bool + """ + if not many: + json = [json] + + arr = [{ + "id": shard + j["id"], + "data": psycopg2.extras.Json(j) + } for j in json] + + self._c.execute("CREATE TABLE IF NOT EXISTS " + objtype + + " (id TEXT PRIMARY KEY, data json)") + + self._c.executemany("INSERT INTO " + objtype + " " + + "VALUES (%(id)s, %(data)s) " + + "ON CONFLICT (id) DO " + + "UPDATE SET data=%(data)s", + arr) + self._connection.commit() + + def upsert(self, shard, json, many=False): + """Upserts an object into the corresponding database. + + :param shard: Region in which an object's id is unique. + :type shard: str + :param json: Object to upsert. + :type json: dict + :param many: (optional) Whether `json` is an array + of objects. + :type many: bool + """ + if not many: + json = [json] + objectmap = dict() + + # figure out the type of each object and sort into map + for j in json: + objtype = j["type"] + if objtype not in objectmap: + objectmap[objtype] = [] + objectmap[objtype].append(j) + + # execute bulk upsert for each type + for objtype, objects in objectmap.items(): + self.upsert_type(shard, objects, objtype, True) + + def select(self, query): + """Returns the result of an SQL query. + + :param query: SQL query to execute. + :type query: str + :return: List of results. + :rtype: list of dict + """ + self._c.execute(query) + return self._c.fetchall() |