From 819af84db665d5c8feb79af45c2f7a94f1bf85c9 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 13 Jan 2026 14:44:17 +0100 Subject: [PATCH] Misc fixes --- changedetectionio/async_update_worker.py | 15 ++++++++------- changedetectionio/worker_handler.py | 4 +++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/changedetectionio/async_update_worker.py b/changedetectionio/async_update_worker.py index 2765d23b6..ca2987335 100644 --- a/changedetectionio/async_update_worker.py +++ b/changedetectionio/async_update_worker.py @@ -426,14 +426,13 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec datastore.update_watch(uuid=uuid, update_obj={'last_error': f"Worker error: {str(e)}"}) finally: - - try: - await update_handler.fetcher.quit(watch=watch) - except Exception as e: - logger.error(f"Exception while cleaning/quit after calling browser: {e}") - # Always cleanup - this runs whether there was an exception or not if uuid: + try: + if update_handler and hasattr(update_handler, 'fetcher') and update_handler.fetcher: + await update_handler.fetcher.quit(watch=watch) + except Exception as e: + logger.error(f"Exception while cleaning/quit after calling browser: {e}") try: # Mark UUID as no longer being processed by this worker worker_handler.set_uuid_processing(uuid, worker_id=worker_id, processing=False) @@ -472,7 +471,9 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec logger.debug(f"Worker {worker_id} completed watch {uuid} in {time.time()-fetch_start_time:.2f}s") except Exception as cleanup_error: logger.error(f"Worker {worker_id} error during cleanup: {cleanup_error}") - + + del(uuid) + # Brief pause before continuing to avoid tight error loops (only on error) if 'e' in locals(): await asyncio.sleep(1.0) diff --git a/changedetectionio/worker_handler.py b/changedetectionio/worker_handler.py index 859917f13..65b5aca65 100644 --- a/changedetectionio/worker_handler.py +++ b/changedetectionio/worker_handler.py @@ -22,8 +22,10 @@ currently_processing_uuids = {} USE_ASYNC_WORKERS = True # Custom ThreadPoolExecutor for queue operations with named threads +# Scale executor threads with FETCH_WORKERS to avoid bottleneck at high concurrency +_max_executor_workers = max(50, int(os.getenv("FETCH_WORKERS", "10"))) queue_executor = ThreadPoolExecutor( - max_workers=50, # Generous limit for concurrent queue operations + max_workers=_max_executor_workers, thread_name_prefix="QueueGetter-" )