blob: d4c5fab0fafa4ee9e6d6eaa6d03ae8b779ff31c6 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
#!/usr/bin/python
import os
import logging
import crawler
import joblib.joblib
RABBIT = {
"host": os.environ.get("RABBITMQ_HOST"),
"port": os.environ.get("RABBITMQ_PORT"),
"credentials": os.environ.get("RABBITMQ_CREDS")
}
APITOKEN = os.environ["MADGLORY_TOKEN"]
class Apigrabber(joblib.joblib.Worker):
def __init__(self, apitoken):
super().__init__("grab")
self._apitoken = apitoken
def work(self, payload):
"""Finish a job."""
api = crawler.Crawler(self._apitoken)
logging.info("running on %s with parameters '%s'",
payload["region"], payload["params"])
try:
for data in api.matches(region=payload["region"],
params=payload["params"]):
items = data["data"] + data["included"]
for item in items:
self.request("process",
payload={
"id": item["id"],
"type": item["type"],
"data": item
})
except crawler.ApiError as error:
logging.warning("API returned error '%s'", error.args[0])
raise joblib.JobFailed(error.args[0],
False) # not critical
def startup():
worker = Apigrabber(APITOKEN)
worker.connect(**RABBIT)
worker.setup()
worker.run()
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
startup()
|
