1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
#!/usr/bin/python
import asyncio
import logging
import aiohttp
APIURL = "https://api.dc01.gamelockerapp.com/"
class Crawler(object):
def __init__(self, token):
"""Sets constants."""
self._apiurl = APIURL
self._token = token
self._pagelimit = 50
async def _req(self, session, path, params=None):
"""Sends an API request and returns the response dict.
:param session: aiohttp client session.
:type session: :class:`aiohttp.ClientSession`
:param path: URL path.
:type path: str
:param params: (optional) Request parameters.
:type params: dict
:return: API response.
:rtype: dict
"""
headers = {
"Authorization": "Bearer " + self._token,
"X-TITLE-ID": "semc-vainglory",
"Accept": "application/vnd.api+json",
"Content-Encoding": "gzip"
}
try:
while True:
async with session.get(self._apiurl + path, headers=headers,
params=params) as response:
if response.status == 429:
logging.warning("hit by rate limit, retrying")
await asyncio.sleep(10)
continue
assert response.status == 200
return await response.json()
except (aiohttp.errors.ClientResponseError,
RuntimeError,
aiohttp.errors.ContentEncodingError):
logging.warning("error connecting to API, retrying")
return await self._req(session, path, params)
async def version(self):
"""Returns the current API version."""
async with aiohttp.ClientSession() as session:
status = await self._req(session, "status")
return status["data"]["attributes"]["version"]
async def matches(self, region="na", params=None):
"""Queries the API for matches and their related data.
:param region: (optional) Region where the matches were played.
Defaults to "na" (North America).
:type region: str
:param params: (optional) Additional filters.
:type params: dict
:return: Processed API response
:rtype: list of dict
"""
forever = False # do not fetch until exhausted
if params is None:
params = dict()
if "page[limit]" not in params:
forever = True # no limit specified, fetch all we can
params["page[limit]"] = self._pagelimit
if "page[offset]" not in params:
params["page[offset]"] = 0
data = []
async with aiohttp.ClientSession() as session:
while True:
params["page[offset]"] += params["page[limit]"]
try:
res = await self._req(session,
"shards/" + region + "/matches",
params)
except AssertionError:
break
data += res["data"] + res["included"]
if len(res["data"]) < 50:
# asked for 50, got less -> exhausted
break
if not forever:
break # stop after one iteration
return data
async def matches_since(self, date, region="na", params=None):
"""Queries the API for new matches since the given date.
:param region: see `matches`
:type region: str
:param date: Start date in ISO8601 format.
:type date: str
:param params: (optional) Additional filters.
:type params: dict
:return: Processed API response
:rtype: list of dict
"""
if params is None:
params = dict()
params["filter[createdAt-start]"] = date
return await self.matches(region, params)
|
