WIP

changedetectionio/fetchers/__init__.py

@@ -122,22 +122,29 @@ class Fetcher():
 
 
 def available_fetchers():
-    from . import playwright, requests, webdriver, browserless
+    from . import playwright, html_requests, webdriver
 
     p = []
-    p.append(tuple(['requests', requests.fetcher.fetcher_description]))
-    p.append(tuple(['playwright', playwright.fetcher.fetcher_description]))
-    p.append(tuple(['webdriver', webdriver.fetcher.fetcher_description]))
-    p.append(tuple(['browserless', browserless.fetcher.fetcher_description]))
+    p.append(tuple(['html_requests', html_requests.fetcher.fetcher_description]))
+
+    # Prefer playwright
+    if os.getenv('PLAYWRIGHT_DRIVER_URL', False):
+        p.append(tuple(['html_webdriver', playwright.fetcher.fetcher_description]))
+
+    elif os.getenv('WEBDRIVER_URL'):
+        p.append(tuple(['html_webdriver', webdriver.fetcher.fetcher_description]))
+
 
     return p
 
+html_webdriver = None
+# Decide which is the 'real' HTML webdriver, this is more a system wide config rather than site-specific.
+use_playwright_as_chrome_fetcher = os.getenv('PLAYWRIGHT_DRIVER_URL', False)
+if use_playwright_as_chrome_fetcher:
+    from . import playwright
+    html_webdriver = getattr(playwright, "fetcher")
 
+else:
+    from . import webdriver
+    html_webdriver = getattr(webdriver, "fetcher")
 
-# Decide which is the 'real' HTML webdriver, this is more a system wide config
-# rather than site-specific.
-#use_playwright_as_chrome_fetcher = os.getenv('PLAYWRIGHT_DRIVER_URL', False)
-#if use_playwright_as_chrome_fetcher:
-#    html_webdriver = base_html_playwright
-#else:
-#    html_webdriver = base_html_webdriver

changedetectionio/forms.py

@@ -138,26 +138,27 @@ class ValidateContentFetcherIsReady(object):
 
         # Better would be a radiohandler that keeps a reference to each class
         if field.data is not None and field.data != 'system':
-            prefered_fetcher = importlib.import_module(f'.{field.data}', package='changedetectionio.fetchers')
-            fetcher = prefered_fetcher.fetcher()
-            try:
-                ready = fetcher.is_ready()
+            from . import fetchers
+            if fetchers.html_webdriver is not None:
+                try:
+                    driver = fetchers.html_webdriver()
+                    driver.is_ready()
 
-            except urllib3.exceptions.MaxRetryError as e:
-                driver_url = fetcher.command_executor
-                message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data))
-                message += '<br>' + field.gettext(
-                    'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.')
-                message += '<br>' + field.gettext('Did you follow the instructions in the wiki?')
-                message += '<br><br>' + field.gettext('WebDriver Host: %s' % (driver_url))
-                message += '<br><a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">Go here for more information</a>'
-                message += '<br>'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e)))
+                except urllib3.exceptions.MaxRetryError as e:
+                    driver_url = fetchers.html_webdriver.command_executor
+                    message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data))
+                    message += '<br>' + field.gettext(
+                        'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.')
+                    message += '<br>' + field.gettext('Did you follow the instructions in the wiki?')
+                    message += '<br><br>' + field.gettext('WebDriver Host: %s' % (driver_url))
+                    message += '<br><a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">Go here for more information</a>'
+                    message += '<br>'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e)))
 
-                raise ValidationError(message)
+                    raise ValidationError(message)
 
-            except Exception as e:
-                message = field.gettext('Content fetcher \'%s\' did not respond properly, unable to use it.\n %s')
-                raise ValidationError(message % (field.data, e))
+                except Exception as e:
+                    message = field.gettext('Content fetcher \'%s\' did not respond properly, unable to use it.\n %s')
+                    raise ValidationError(message % (field.data, e))
 
 
 class ValidateNotificationBodyAndTitleWhenURLisSet(object):

changedetectionio/processors/restock_diff.py

@@ -5,6 +5,7 @@ import re
 import urllib3
 from . import difference_detection_processor
 from copy import deepcopy
+from .. import fetchers
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -60,11 +61,12 @@ class perform_site_check(difference_detection_processor):
         if not prefer_backend or prefer_backend == 'system':
             prefer_backend = self.datastore.data['settings']['application']['fetch_backend']
 
-        if hasattr(content_fetcher, prefer_backend):
-            klass = getattr(content_fetcher, prefer_backend)
+        if prefer_backend == 'html_webdriver':
+            preferred_fetcher = fetchers.html_webdriver
         else:
-            # If the klass doesnt exist, just use a default
-            klass = getattr(content_fetcher, "html_requests")
+            from ..fetchers import html_requests
+            preferred_fetcher = html_requests
+
 
         proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
         proxy_url = None
@@ -72,7 +74,7 @@ class perform_site_check(difference_detection_processor):
             proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')
             print("UUID {} Using proxy {}".format(uuid, proxy_url))
 
-        fetcher = klass(proxy_override=proxy_url)
+        fetcher = preferred_fetcher(proxy_override=proxy_url)
 
         # Configurable per-watch or global extra delay before extracting text (for webDriver types)
         system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)

changedetectionio/processors/text_json_diff.py

@@ -102,8 +102,11 @@ class perform_site_check(difference_detection_processor):
         if not prefer_backend or prefer_backend == 'system':
             prefer_backend = self.datastore.data['settings']['application']['fetch_backend']
 
-        import importlib
-        prefered_fetcher = importlib.import_module(f'.{prefer_backend}', package='changedetectionio.fetchers')
+        if prefer_backend == 'html_webdriver':
+            preferred_fetcher = fetchers.html_webdriver
+        else:
+            from ..fetchers import html_requests
+            preferred_fetcher = html_requests
 
 
         proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
@@ -112,7 +115,7 @@ class perform_site_check(difference_detection_processor):
             proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')
             print("UUID {} Using proxy {}".format(uuid, proxy_url))
 
-        fetcher = prefered_fetcher.fetcher(proxy_override=proxy_url)
+        fetcher = preferred_fetcher(proxy_override=proxy_url)
 
         # Configurable per-watch or global extra delay before extracting text (for webDriver types)
         system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)