diff options
Diffstat (limited to 'api.py')
| -rw-r--r-- | api.py | 28 |
1 files changed, 17 insertions, 11 deletions
@@ -19,7 +19,7 @@ class Worker(object): async def connect(self, **args): """Connect to database.""" - logging.info("connecting to database") + logging.warning("connecting to database") self._queue = joblib.joblib.JobQueue() await self._queue.connect(**args) await self._queue.setup() @@ -42,7 +42,6 @@ class Worker(object): async def _execute_job(self, jobid, payload, priority): """Finish a job.""" api = crawler.Crawler(self._apitoken) - logging.debug("%s: getting matches from API", jobid) # if a player is queried, pass that information to processor if "filter[playerNames]" in payload["params"]: playername = payload["params"]["filter[playerNames]"] @@ -52,9 +51,9 @@ class Worker(object): async with self._pool.acquire() as con: async for data in api.matches(region=payload["region"], params=payload["params"]): - logging.debug("%s: inserting into database", jobid) matchids = await con.fetch(self._insertquery, json.dumps(data)) - logging.info("%s: inserted %s matches", jobid, len(matchids)) + logging.debug("%s: inserted %s matches from API into database", + jobid, len(matchids)) for matchid in matchids: await self._queue.request(jobtype="process", priority=priority, @@ -68,14 +67,13 @@ class Worker(object): jobid, payload, priority = await self._queue.acquire(jobtype="grab") if jobid is None: raise LookupError("no jobs available") - logging.debug("%s: starting job", jobid) try: await self._execute_job(jobid, payload, priority) await self._queue.finish(jobid) except crawler.ApiError as error: - logging.warning("%s: failed", jobid) + logging.warning("%s: failed with %s", jobid, + error.args[0]) await self._queue.fail(jobid, error.args[0]) - logging.debug("%s: finished job", jobid) async def run(self): """Start jobs forever.""" @@ -83,8 +81,7 @@ class Worker(object): try: await self._work() except LookupError: - logging.info("nothing to do, idling") - await asyncio.sleep(10) + await asyncio.sleep(1) async def start(self, number=1): """Start jobs in background.""" @@ -103,9 +100,18 @@ async def startup(): database=os.environ["POSTGRESQL_DB"] ) await worker.setup() - await worker.start() + await worker.start(2) + + +logging.basicConfig( + filename="logs/apigrabber.log", + filemode="a", + level=logging.DEBUG +) +console = logging.StreamHandler() +console.setLevel(logging.WARNING) +logging.getLogger("").addHandler(console) -logging.basicConfig(level=logging.DEBUG) loop = asyncio.get_event_loop() loop.run_until_complete(startup()) loop.run_forever() |