diff --git a/changedetectionio/fetchers/__init__.py b/changedetectionio/fetchers/__init__.py index 2f0ff1887..bd1694fa4 100644 --- a/changedetectionio/fetchers/__init__.py +++ b/changedetectionio/fetchers/__init__.py @@ -122,22 +122,29 @@ class Fetcher(): def available_fetchers(): - from . import playwright, requests, webdriver, browserless + from . import playwright, html_requests, webdriver p = [] - p.append(tuple(['requests', requests.fetcher.fetcher_description])) - p.append(tuple(['playwright', playwright.fetcher.fetcher_description])) - p.append(tuple(['webdriver', webdriver.fetcher.fetcher_description])) - p.append(tuple(['browserless', browserless.fetcher.fetcher_description])) + p.append(tuple(['html_requests', html_requests.fetcher.fetcher_description])) + + # Prefer playwright + if os.getenv('PLAYWRIGHT_DRIVER_URL', False): + p.append(tuple(['html_webdriver', playwright.fetcher.fetcher_description])) + + elif os.getenv('WEBDRIVER_URL'): + p.append(tuple(['html_webdriver', webdriver.fetcher.fetcher_description])) + return p +html_webdriver = None +# Decide which is the 'real' HTML webdriver, this is more a system wide config rather than site-specific. +use_playwright_as_chrome_fetcher = os.getenv('PLAYWRIGHT_DRIVER_URL', False) +if use_playwright_as_chrome_fetcher: + from . import playwright + html_webdriver = getattr(playwright, "fetcher") +else: + from . import webdriver + html_webdriver = getattr(webdriver, "fetcher") -# Decide which is the 'real' HTML webdriver, this is more a system wide config -# rather than site-specific. -#use_playwright_as_chrome_fetcher = os.getenv('PLAYWRIGHT_DRIVER_URL', False) -#if use_playwright_as_chrome_fetcher: -# html_webdriver = base_html_playwright -#else: -# html_webdriver = base_html_webdriver \ No newline at end of file diff --git a/changedetectionio/fetchers/requests.py b/changedetectionio/fetchers/html_requests.py similarity index 100% rename from changedetectionio/fetchers/requests.py rename to changedetectionio/fetchers/html_requests.py diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 9baf213ea..e4c34b488 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -138,26 +138,27 @@ class ValidateContentFetcherIsReady(object): # Better would be a radiohandler that keeps a reference to each class if field.data is not None and field.data != 'system': - prefered_fetcher = importlib.import_module(f'.{field.data}', package='changedetectionio.fetchers') - fetcher = prefered_fetcher.fetcher() - try: - ready = fetcher.is_ready() + from . import fetchers + if fetchers.html_webdriver is not None: + try: + driver = fetchers.html_webdriver() + driver.is_ready() - except urllib3.exceptions.MaxRetryError as e: - driver_url = fetcher.command_executor - message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data)) - message += '
' + field.gettext( - 'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.') - message += '
' + field.gettext('Did you follow the instructions in the wiki?') - message += '

' + field.gettext('WebDriver Host: %s' % (driver_url)) - message += '
Go here for more information' - message += '
'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e))) + except urllib3.exceptions.MaxRetryError as e: + driver_url = fetchers.html_webdriver.command_executor + message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data)) + message += '
' + field.gettext( + 'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.') + message += '
' + field.gettext('Did you follow the instructions in the wiki?') + message += '

' + field.gettext('WebDriver Host: %s' % (driver_url)) + message += '
Go here for more information' + message += '
'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e))) - raise ValidationError(message) + raise ValidationError(message) - except Exception as e: - message = field.gettext('Content fetcher \'%s\' did not respond properly, unable to use it.\n %s') - raise ValidationError(message % (field.data, e)) + except Exception as e: + message = field.gettext('Content fetcher \'%s\' did not respond properly, unable to use it.\n %s') + raise ValidationError(message % (field.data, e)) class ValidateNotificationBodyAndTitleWhenURLisSet(object): diff --git a/changedetectionio/processors/restock_diff.py b/changedetectionio/processors/restock_diff.py index c096c6015..1137cb71a 100644 --- a/changedetectionio/processors/restock_diff.py +++ b/changedetectionio/processors/restock_diff.py @@ -5,6 +5,7 @@ import re import urllib3 from . import difference_detection_processor from copy import deepcopy +from .. import fetchers urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -60,11 +61,12 @@ class perform_site_check(difference_detection_processor): if not prefer_backend or prefer_backend == 'system': prefer_backend = self.datastore.data['settings']['application']['fetch_backend'] - if hasattr(content_fetcher, prefer_backend): - klass = getattr(content_fetcher, prefer_backend) + if prefer_backend == 'html_webdriver': + preferred_fetcher = fetchers.html_webdriver else: - # If the klass doesnt exist, just use a default - klass = getattr(content_fetcher, "html_requests") + from ..fetchers import html_requests + preferred_fetcher = html_requests + proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid) proxy_url = None @@ -72,7 +74,7 @@ class perform_site_check(difference_detection_processor): proxy_url = self.datastore.proxy_list.get(proxy_id).get('url') print("UUID {} Using proxy {}".format(uuid, proxy_url)) - fetcher = klass(proxy_override=proxy_url) + fetcher = preferred_fetcher(proxy_override=proxy_url) # Configurable per-watch or global extra delay before extracting text (for webDriver types) system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None) diff --git a/changedetectionio/processors/text_json_diff.py b/changedetectionio/processors/text_json_diff.py index 42789ae19..07d8eb339 100644 --- a/changedetectionio/processors/text_json_diff.py +++ b/changedetectionio/processors/text_json_diff.py @@ -102,8 +102,11 @@ class perform_site_check(difference_detection_processor): if not prefer_backend or prefer_backend == 'system': prefer_backend = self.datastore.data['settings']['application']['fetch_backend'] - import importlib - prefered_fetcher = importlib.import_module(f'.{prefer_backend}', package='changedetectionio.fetchers') + if prefer_backend == 'html_webdriver': + preferred_fetcher = fetchers.html_webdriver + else: + from ..fetchers import html_requests + preferred_fetcher = html_requests proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid) @@ -112,7 +115,7 @@ class perform_site_check(difference_detection_processor): proxy_url = self.datastore.proxy_list.get(proxy_id).get('url') print("UUID {} Using proxy {}".format(uuid, proxy_url)) - fetcher = prefered_fetcher.fetcher(proxy_override=proxy_url) + fetcher = preferred_fetcher(proxy_override=proxy_url) # Configurable per-watch or global extra delay before extracting text (for webDriver types) system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)