From 7be4d39c69cbe8d5668cf4eb95859ddf792cdcd9 Mon Sep 17 00:00:00 2001 From: schneefux Date: Sun, 19 Mar 2017 15:41:27 +0100 Subject: kill process when the worker crashes, increase batch limit --- api.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) (limited to 'api.py') diff --git a/api.py b/api.py index f34a34c..23155c0 100644 --- a/api.py +++ b/api.py @@ -224,19 +224,17 @@ class Processor(joblib.worker.Worker): async def startup(): - for _ in range(1): - worker = Processor( - do_preload=os.environ.get("VAINSOCIAL_SPIDER")=="true", - do_analyze=os.environ.get("VAINSOCIAL_ANALYZE")=="true" - ) - await worker.connect( - source_db, dest_db - ) - await worker.setup() - await worker.start(batchlimit=50) + worker = Processor( + do_preload=os.environ.get("VAINSOCIAL_SPIDER")=="true", + do_analyze=os.environ.get("VAINSOCIAL_ANALYZE")=="true" + ) + await worker.connect( + source_db, dest_db + ) + await worker.setup() + await worker.run(batchlimit=1000) logging.basicConfig(level=logging.DEBUG) loop = asyncio.get_event_loop() loop.run_until_complete(startup()) -loop.run_forever() -- cgit v1.3.1