diff --git a/changedetectionio/blueprint/settings/templates/settings.html b/changedetectionio/blueprint/settings/templates/settings.html index d7b4eb8e5..6e2a96172 100644 --- a/changedetectionio/blueprint/settings/templates/settings.html +++ b/changedetectionio/blueprint/settings/templates/settings.html @@ -47,10 +47,6 @@ {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }} Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later -
- {{ render_field(form.requests.form.workers) }} - Number of concurrent workers to process watches. More workers = faster processing but higher memory usage. -
{{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class="filter_failure_notification_threshold_attempts") }} After this many consecutive times that the CSS/xPath filter is missing, send a notification @@ -139,6 +135,10 @@ {{ render_field(form.application.form.webdriver_delay) }}
+
+ {{ render_field(form.requests.form.workers) }} + Number of concurrent workers to process watches. More workers = faster processing but higher memory usage. +
{{ render_field(form.requests.form.default_ua) }} diff --git a/changedetectionio/content_fetchers/base.py b/changedetectionio/content_fetchers/base.py index 9aa40b400..1abce26de 100644 --- a/changedetectionio/content_fetchers/base.py +++ b/changedetectionio/content_fetchers/base.py @@ -68,7 +68,7 @@ class Fetcher(): return self.error @abstractmethod - def run(self, + async def run(self, url, timeout, request_headers, diff --git a/changedetectionio/content_fetchers/puppeteer.py b/changedetectionio/content_fetchers/puppeteer.py index 22b685690..d62d308d5 100644 --- a/changedetectionio/content_fetchers/puppeteer.py +++ b/changedetectionio/content_fetchers/puppeteer.py @@ -310,15 +310,15 @@ class fetcher(Fetcher): async def main(self, **kwargs): await self.fetch_page(**kwargs) - def run(self, url, timeout, request_headers, request_body, request_method, ignore_status_codes=False, + async def run(self, url, timeout, request_headers, request_body, request_method, ignore_status_codes=False, current_include_filters=None, is_binary=False, empty_pages_are_a_change=False): #@todo make update_worker async which could run any of these content_fetchers within memory and time constraints - max_time = os.getenv('PUPPETEER_MAX_PROCESSING_TIMEOUT_SECONDS', 180) + max_time = int(os.getenv('PUPPETEER_MAX_PROCESSING_TIMEOUT_SECONDS', 180)) - # This will work in 3.10 but not >= 3.11 because 3.11 wants tasks only + # Now we run this properly in async context since we're called from async worker try: - asyncio.run(asyncio.wait_for(self.main( + await asyncio.wait_for(self.main( url=url, timeout=timeout, request_headers=request_headers, @@ -328,7 +328,7 @@ class fetcher(Fetcher): current_include_filters=current_include_filters, is_binary=is_binary, empty_pages_are_a_change=empty_pages_are_a_change - ), timeout=max_time)) + ), timeout=max_time) except asyncio.TimeoutError: raise(BrowserFetchTimedOut(msg=f"Browser connected but was unable to process the page in {max_time} seconds.")) diff --git a/changedetectionio/content_fetchers/webdriver_selenium.py b/changedetectionio/content_fetchers/webdriver_selenium.py index 180d6332e..48897d7aa 100644 --- a/changedetectionio/content_fetchers/webdriver_selenium.py +++ b/changedetectionio/content_fetchers/webdriver_selenium.py @@ -47,7 +47,7 @@ class fetcher(Fetcher): self.proxy_url = k.strip() - def run(self, + async def run(self, url, timeout, request_headers, @@ -58,77 +58,86 @@ class fetcher(Fetcher): is_binary=False, empty_pages_are_a_change=False): - from selenium.webdriver.chrome.options import Options as ChromeOptions - # request_body, request_method unused for now, until some magic in the future happens. + import asyncio + + # Wrap the entire selenium operation in a thread executor + def _run_sync(): + from selenium.webdriver.chrome.options import Options as ChromeOptions + # request_body, request_method unused for now, until some magic in the future happens. - options = ChromeOptions() + options = ChromeOptions() - # Load Chrome options from env - CHROME_OPTIONS = [ - line.strip() - for line in os.getenv("CHROME_OPTIONS", "").strip().splitlines() - if line.strip() - ] + # Load Chrome options from env + CHROME_OPTIONS = [ + line.strip() + for line in os.getenv("CHROME_OPTIONS", "").strip().splitlines() + if line.strip() + ] - for opt in CHROME_OPTIONS: - options.add_argument(opt) + for opt in CHROME_OPTIONS: + options.add_argument(opt) - # 1. proxy_config /Proxy(proxy_config) selenium object is REALLY unreliable - # 2. selenium-wire cant be used because the websocket version conflicts with pypeteer-ng - # 3. selenium only allows ONE runner at a time by default! - # 4. driver must use quit() or it will continue to block/hold the selenium process!! + # 1. proxy_config /Proxy(proxy_config) selenium object is REALLY unreliable + # 2. selenium-wire cant be used because the websocket version conflicts with pypeteer-ng + # 3. selenium only allows ONE runner at a time by default! + # 4. driver must use quit() or it will continue to block/hold the selenium process!! - if self.proxy_url: - options.add_argument(f'--proxy-server={self.proxy_url}') + if self.proxy_url: + options.add_argument(f'--proxy-server={self.proxy_url}') - from selenium.webdriver.remote.remote_connection import RemoteConnection - from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver - driver = None - try: - # Create the RemoteConnection and set timeout (e.g., 30 seconds) - remote_connection = RemoteConnection( - self.browser_connection_url, - ) - remote_connection.set_timeout(30) # seconds + from selenium.webdriver.remote.remote_connection import RemoteConnection + from selenium.webdriver.remote.webdriver import WebDriver as RemoteWebDriver + driver = None + try: + # Create the RemoteConnection and set timeout (e.g., 30 seconds) + remote_connection = RemoteConnection( + self.browser_connection_url, + ) + remote_connection.set_timeout(30) # seconds - # Now create the driver with the RemoteConnection - driver = RemoteWebDriver( - command_executor=remote_connection, - options=options - ) + # Now create the driver with the RemoteConnection + driver = RemoteWebDriver( + command_executor=remote_connection, + options=options + ) - driver.set_page_load_timeout(int(os.getenv("WEBDRIVER_PAGELOAD_TIMEOUT", 45))) - except Exception as e: - if driver: - driver.quit() - raise e + driver.set_page_load_timeout(int(os.getenv("WEBDRIVER_PAGELOAD_TIMEOUT", 45))) + except Exception as e: + if driver: + driver.quit() + raise e - try: - driver.get(url) + try: + driver.get(url) - if not "--window-size" in os.getenv("CHROME_OPTIONS", ""): - driver.set_window_size(1280, 1024) + if not "--window-size" in os.getenv("CHROME_OPTIONS", ""): + driver.set_window_size(1280, 1024) - driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))) - - if self.webdriver_js_execute_code is not None: - driver.execute_script(self.webdriver_js_execute_code) - # Selenium doesn't automatically wait for actions as good as Playwright, so wait again driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))) - # @todo - how to check this? is it possible? - self.status_code = 200 - # @todo somehow we should try to get this working for WebDriver - # raise EmptyReply(url=url, status_code=r.status_code) + if self.webdriver_js_execute_code is not None: + driver.execute_script(self.webdriver_js_execute_code) + # Selenium doesn't automatically wait for actions as good as Playwright, so wait again + driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))) + + # @todo - how to check this? is it possible? + self.status_code = 200 + # @todo somehow we should try to get this working for WebDriver + # raise EmptyReply(url=url, status_code=r.status_code) + + # @todo - dom wait loaded? + import time + time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay) + self.content = driver.page_source + self.headers = {} + self.screenshot = driver.get_screenshot_as_png() + except Exception as e: + driver.quit() + raise e - # @todo - dom wait loaded? - time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay) - self.content = driver.page_source - self.headers = {} - self.screenshot = driver.get_screenshot_as_png() - except Exception as e: driver.quit() - raise e - driver.quit() + # Run the selenium operations in a thread pool to avoid blocking the event loop + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, _run_sync)