Worker id/available fetcher (#4415)

* Worker ID # should be available to the content fetchers

* Actually init with the worker ID

changedetectionio/content_fetchers/base.py

@@ -90,6 +90,7 @@ class Fetcher():
     screenshot_format = None
     status_code = None
     webdriver_js_execute_code = None
+    worker_id = None
     xpath_data = None
     xpath_element_js = ""
 
@@ -120,6 +121,11 @@ class Fetcher():
         if kwargs and 'lock_viewport_elements' in kwargs:
             self.lock_viewport_elements = kwargs.get('lock_viewport_elements')
 
+        # Which async worker is driving this fetch, subclasses use it to keep per-worker browser
+        # state (profile dirs etc) apart, stays None when we're not called from a worker
+        if kwargs and 'worker_id' in kwargs:
+            self.worker_id = kwargs.get('worker_id')
+
 
     @classmethod
     def get_status_icon_data(cls):

changedetectionio/processors/base.py

@@ -22,11 +22,16 @@ class difference_detection_processor():
     preferred_proxy = None
     screenshot_format = SCREENSHOT_FORMAT_JPEG
     last_raw_content_checksum = None
+    worker_id = None
 
-    def __init__(self, datastore, watch_uuid):
+    def __init__(self, datastore, watch_uuid, worker_id=None):
         self.datastore = datastore
         self.watch_uuid = watch_uuid
 
+        # Which async worker is driving this check, passed down to the fetcher in call_browser()
+        # so it can keep per-worker browser state apart, None when we're not called from a worker
+        self.worker_id = worker_id
+
         # Create a stable snapshot of the watch for processing
         # Why deepcopy?
         # 1. Prevents "dict changed during iteration" errors if watch is modified during processing
@@ -201,7 +206,8 @@ class difference_detection_processor():
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
         self.fetcher = fetcher_obj(proxy_override=proxy_url,
                                    custom_browser_connection_url=custom_browser_connection_url,
-                                   screenshot_format=self.screenshot_format
+                                   screenshot_format=self.screenshot_format,
+                                   worker_id=self.worker_id
                                    )
 
         # Stamp the resolved backend name so downstream consumers (processors, plugins)

changedetectionio/worker.py

@@ -177,7 +177,8 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
                         raise ModuleNotFoundError(error_msg)
 
                     update_handler = processor_module.perform_site_check(datastore=datastore,
-                                                                         watch_uuid=uuid)
+                                                                         watch_uuid=uuid,
+                                                                         worker_id=worker_id)
 
                     # Allow plugins to modify/wrap the update_handler
                     update_handler = apply_update_handler_alter(update_handler, watch, datastore)