From 3c7b19d77476d774762b4a6c97bff18bcd0b249a Mon Sep 17 00:00:00 2001
From: schneefux <schneefux+commit@schneefux.xyz>
Date: Mon, 27 Mar 2017 20:58:21 +0200
Subject: use 2.0 joblib

---
 worker.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 worker.py

(limited to 'worker.py')

diff --git a/worker.py b/worker.py
new file mode 100644
index 0000000..d4c5fab
--- /dev/null
+++ b/worker.py
@@ -0,0 +1,55 @@
+#!/usr/bin/python
+
+import os
+import logging
+
+import crawler
+import joblib.joblib
+
+
+RABBIT = {
+    "host": os.environ.get("RABBITMQ_HOST"),
+    "port": os.environ.get("RABBITMQ_PORT"),
+    "credentials": os.environ.get("RABBITMQ_CREDS")
+}
+
+APITOKEN = os.environ["MADGLORY_TOKEN"]
+
+
+class Apigrabber(joblib.joblib.Worker):
+    def __init__(self, apitoken):
+        super().__init__("grab")
+        self._apitoken = apitoken
+
+    def work(self, payload):
+        """Finish a job."""
+        api = crawler.Crawler(self._apitoken)
+        logging.info("running on %s with parameters '%s'",
+                     payload["region"], payload["params"])
+        try:
+            for data in api.matches(region=payload["region"],
+                                    params=payload["params"]):
+                items = data["data"] + data["included"]
+                for item in items:
+                    self.request("process",
+                                 payload={
+                                     "id": item["id"],
+                                     "type": item["type"],
+                                     "data": item
+                                 })
+        except crawler.ApiError as error:
+            logging.warning("API returned error '%s'", error.args[0])
+            raise joblib.JobFailed(error.args[0],
+                                   False)  # not critical
+
+
+def startup():
+    worker = Apigrabber(APITOKEN)
+    worker.connect(**RABBIT)
+    worker.setup()
+    worker.run()
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    startup()
-- 
cgit v1.3.1