1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
|
#!/usr/bin/python
import asyncio
import datetime
import aiohttp
TOKEN = "aaa.bbb.ccc"
APIURL = "https://api.dc01.gamelockerapp.com/"
class Crawler(object):
def __init__(self):
"""Sets constants."""
self._apiurl = APIURL
self._token = TOKEN
self._lastquery = datetime.datetime(1, 1, 1)
self._pagelimit = 50
async def _req(self, session, path, params=None):
"""Sends an API request and returns the response dict.
:param session: aiohttp client session.
:type session: :class:`aiohttp.ClientSession`
:param path: URL path.
:type path: str
:param params: (optional) Request parameters.
:type params: dict
:return: API response.
:rtype: dict
"""
headers = {
"Authorization": "Bearer " + self._token,
"X-TITLE-ID": "semc-vainglory",
"Accept": "application/vnd.api+json",
"Content-Encoding": "gzip"
}
async with session.get(self._apiurl + path, headers=headers,
params=params) as response:
if response.status != 200:
return None
return await response.json()
async def matches(self, region="na", params=None):
"""Queries the API for matches and their related data.
:param region: (optional) Region where the matches were played.
Defaults to "na" (North America).
:type region: str
:param params: (optional) Additional filters.
:type params: dict
:return: Processed API response
:rtype: list of dict
"""
if params is None:
params = dict()
params["page[limit]"] = self._pagelimit
params["page[offset]"] = 0
batchsize = 100 # FIXME We don't know how long we can paginate
tasks = []
data = []
# fire a lot of http requests
async with aiohttp.ClientSession() as session:
for _ in range(0, batchsize):
params["page[offset]"] += params["page[limit]"]
task = asyncio.ensure_future(
self._req(
session,
"shards/" + region + "/matches",
params))
tasks.append(task)
results = await asyncio.gather(*tasks)
for res in results:
if res is None:
continue
data += res["data"] + res["included"]
return data
async def matches_new(self, region="na"):
"""Queries the API for new matches since the last query.
:param region: see `matches`
:type region: str
"""
lastdate = self._lastquery.replace(microsecond=0).isoformat() + "Z"
self._lastquery = datetime.datetime.now()
return self.matches(region, {"filter[createdAt-start]": lastdate})
|
