From e0d0f4a4294d528762fcf673c38d8eb05d5c39bc Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 26 Mar 2026 22:34:29 +0100 Subject: [PATCH] Move UA browser profiles only --- .../settings/browser_profile/__init__.py | 1 + .../templates/browser_profiles.html | 4 + .../settings/templates/settings.html | 7 - .../content_fetchers/playwright/CDP.py | 1 - .../content_fetchers/puppeteer.py | 1 - .../content_fetchers/webdriver_selenium.py | 1 - changedetectionio/forms.py | 18 +-- changedetectionio/model/App.py | 7 - changedetectionio/model/browser_profile.py | 16 +++ changedetectionio/processors/base.py | 15 ++- .../tests/test_browser_profile_status_icon.py | 124 ++++++++++++++++++ changedetectionio/tests/test_request.py | 99 +++++--------- 12 files changed, 196 insertions(+), 98 deletions(-) diff --git a/changedetectionio/blueprint/settings/browser_profile/__init__.py b/changedetectionio/blueprint/settings/browser_profile/__init__.py index 7faacb36b..028e08b21 100644 --- a/changedetectionio/blueprint/settings/browser_profile/__init__.py +++ b/changedetectionio/blueprint/settings/browser_profile/__init__.py @@ -118,6 +118,7 @@ def construct_blueprint(datastore: ChangeDetectionStore): 'ignore_https_errors': bool(browser_profile_form.ignore_https_errors.data), 'user_agent': browser_profile_form.user_agent.data or None, 'locale': browser_profile_form.locale.data or None, + 'custom_headers': browser_profile_form.custom_headers.data or '', 'is_builtin': False, } diff --git a/changedetectionio/blueprint/settings/browser_profile/templates/browser_profiles.html b/changedetectionio/blueprint/settings/browser_profile/templates/browser_profiles.html index 5c0282f92..d62c4080a 100644 --- a/changedetectionio/blueprint/settings/browser_profile/templates/browser_profiles.html +++ b/changedetectionio/blueprint/settings/browser_profile/templates/browser_profiles.html @@ -108,6 +108,10 @@ {{ render_field(browser_profile_form.locale) }} {{ _('Sets Accept-Language and navigator.language (e.g. en-US, de-DE).') }} +
+ {{ render_field(browser_profile_form.custom_headers) }} + {{ _('Extra HTTP headers for all requests using this profile (one per line, Key: Value). Applied before per-watch headers.') }} +
{% if editing_machine_name %} diff --git a/changedetectionio/blueprint/settings/templates/settings.html b/changedetectionio/blueprint/settings/templates/settings.html index 9f9ed3b65..6839355c3 100644 --- a/changedetectionio/blueprint/settings/templates/settings.html +++ b/changedetectionio/blueprint/settings/templates/settings.html @@ -140,13 +140,6 @@ {{ render_field(form.requests.form.timeout) }} {{ _('For regular plain requests (not chrome based), maximum number of seconds until timeout, 1-999.') }}
-
- {{ render_field(form.requests.form.default_ua) }} - - {{ _('Applied to all requests.') }}

- {{ _('Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it\'s important to consider') }} {{ _('all of the ways that the browser is detected') }}. - -

{{ _('Tip:') }} {{ _('Connect using Bright Data proxies, find out more here.') }} diff --git a/changedetectionio/content_fetchers/playwright/CDP.py b/changedetectionio/content_fetchers/playwright/CDP.py index 0bb4059b4..1a1d234f2 100644 --- a/changedetectionio/content_fetchers/playwright/CDP.py +++ b/changedetectionio/content_fetchers/playwright/CDP.py @@ -12,7 +12,6 @@ from changedetectionio.content_fetchers.playwright import PlaywrightBaseFetcher class fetcher(PlaywrightBaseFetcher): fetcher_description = "Playwright Chrome (CDP/Remote)" requires_connection_url = True - ua_settings_key = 'playwright' # matches DefaultUAInputForm field name def __init__(self, proxy_override=None, custom_browser_connection_url=None, **kwargs): super().__init__(proxy_override=proxy_override, custom_browser_connection_url=custom_browser_connection_url, **kwargs) diff --git a/changedetectionio/content_fetchers/puppeteer.py b/changedetectionio/content_fetchers/puppeteer.py index 135890c3b..1ca54601b 100644 --- a/changedetectionio/content_fetchers/puppeteer.py +++ b/changedetectionio/content_fetchers/puppeteer.py @@ -181,7 +181,6 @@ class fetcher(Fetcher): supports_xpath_element_data = True status_icon = {'filename': 'google-chrome-icon.png', 'alt': 'Using a Chrome browser', 'title': 'Using a Chrome browser'} - ua_settings_key = 'playwright' # shares the 'playwright' UA field in settings UI def disk_cleanup_after_fetch(self): self.delete_browser_steps_screenshots() diff --git a/changedetectionio/content_fetchers/webdriver_selenium.py b/changedetectionio/content_fetchers/webdriver_selenium.py index aa1bfcd8c..354ea8681 100644 --- a/changedetectionio/content_fetchers/webdriver_selenium.py +++ b/changedetectionio/content_fetchers/webdriver_selenium.py @@ -20,7 +20,6 @@ class fetcher(Fetcher): supports_xpath_element_data = True status_icon = {'filename': 'google-chrome-icon.png', 'alt': 'Using a Chrome browser', 'title': 'Using a Chrome browser'} - ua_settings_key = 'selenium' # matches DefaultUAInputForm field name def __init__(self, proxy_override=None, custom_browser_connection_url=None, **kwargs): super().__init__(**kwargs) diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 4696ceede..a366591a7 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -991,13 +991,15 @@ class BrowserProfileForm(Form): [validators.Optional(), validators.Length(max=20)], render_kw={"placeholder": "en-US, de-DE, fr-FR …", "size": 15} ) - -class DefaultUAInputForm(Form): - requests = StringField(_l('Plaintext requests'), validators=[validators.Optional()], render_kw={"placeholder": ""}) - if os.getenv("PLAYWRIGHT_DRIVER_URL") or os.getenv("WEBDRIVER_URL"): - playwright = StringField(_l('Chrome/Playwright requests'), validators=[validators.Optional()], render_kw={"placeholder": ""}) - selenium = StringField(_l('Chrome/Selenium requests'), validators=[validators.Optional()], render_kw={"placeholder": ""}) - puppeteer = StringField(_l('Chrome/Puppeteer requests'), validators=[validators.Optional()], render_kw={"placeholder": ""}) + custom_headers = TextAreaField( + _l('Custom headers'), + [validators.Optional()], + render_kw={ + "placeholder": "Header-Name: value\nAnother-Header: value", + "rows": 4, "cols": 60, + "style": "font-family:monospace;" + } + ) # datastore.data['settings']['requests'].. class globalSettingsRequestForm(Form): @@ -1021,8 +1023,6 @@ class globalSettingsRequestForm(Form): extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5) extra_browsers = FieldList(FormField(SingleExtraBrowser), min_entries=5) - default_ua = FormField(DefaultUAInputForm, label=_l("Default User-Agent overrides")) - def validate_extra_proxies(self, extra_validators=None): for e in self.data['extra_proxies']: if e.get('proxy_name') or e.get('proxy_url'): diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py index 7dd68a63c..ecaeb8c1a 100644 --- a/changedetectionio/model/App.py +++ b/changedetectionio/model/App.py @@ -12,7 +12,6 @@ from changedetectionio.notification import ( # Equal to or greater than this number of FilterNotFoundInResponse exceptions will trigger a filter-not-found notification _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT = 6 -DEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36' @@ -31,12 +30,6 @@ class model(dict): 'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None}, 'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")), # Default 45 seconds 'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "5")), # Number of threads, lower is better for slow connections - 'default_ua': { - 'requests': getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", DEFAULT_SETTINGS_HEADERS_USERAGENT), - 'playwright': None, - 'selenium': None, - 'puppeteer': None, - } }, 'application': { # Custom notification content diff --git a/changedetectionio/model/browser_profile.py b/changedetectionio/model/browser_profile.py index 8ca233170..66dcb021f 100644 --- a/changedetectionio/model/browser_profile.py +++ b/changedetectionio/model/browser_profile.py @@ -39,12 +39,20 @@ references. After that migration no legacy paths are needed here. from __future__ import annotations +import os import re from typing import Optional from loguru import logger from pydantic import BaseModel, field_validator +# Default User-Agent for the built-in plaintext requests profile. +# Overridable via environment variable for deployments that need a custom UA. +_DEFAULT_REQUESTS_UA = os.getenv( + "DEFAULT_SETTINGS_HEADERS_USERAGENT", + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36' +) + # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- @@ -139,6 +147,13 @@ class BrowserProfile(BaseModel): Some sites serve different prices or copy based on locale. """ + custom_headers: str = '' + """ + Extra HTTP headers sent with every request using this profile, in ``Key: Value`` format + (one per line, ``#`` lines are ignored). Applied before per-watch headers so + individual watches can override them. + """ + service_workers: str = 'allow' """ Whether to allow Service Workers in the browser context. @@ -241,6 +256,7 @@ BUILTIN_REQUESTS = BrowserProfile( name='Direct HTTP (requests)', fetch_backend='requests', is_builtin=True, + user_agent=_DEFAULT_REQUESTS_UA, ) BUILTIN_PLAYWRIGHT = BrowserProfile( diff --git a/changedetectionio/processors/base.py b/changedetectionio/processors/base.py index 6847e1f5b..024a94057 100644 --- a/changedetectionio/processors/base.py +++ b/changedetectionio/processors/base.py @@ -186,10 +186,17 @@ class difference_detection_processor(): from changedetectionio.jinja2_custom import render as jinja_render request_headers = CaseInsensitiveDict() - ua = self.datastore.data['settings']['requests'].get('default_ua') - ua_key = getattr(fetcher_obj, 'ua_settings_key', fetcher_class_name) - if ua and ua.get(ua_key): - request_headers.update({'User-Agent': ua.get(ua_key)}) + # Browser profile: UA override (lowest priority — watch headers override this) + if profile.user_agent: + request_headers['User-Agent'] = profile.user_agent + + # Browser profile: custom headers (override profile UA, but watch headers override these) + if profile.custom_headers: + for line in profile.custom_headers.splitlines(): + line = line.strip() + if not line.startswith('#') and ':' in line: + k, v = line.split(':', 1) + request_headers[k.strip()] = v.strip() request_headers.update(self.watch.get('headers', {})) request_headers.update(self.datastore.get_all_base_headers()) diff --git a/changedetectionio/tests/test_browser_profile_status_icon.py b/changedetectionio/tests/test_browser_profile_status_icon.py index 6068e4236..ebcbe188e 100644 --- a/changedetectionio/tests/test_browser_profile_status_icon.py +++ b/changedetectionio/tests/test_browser_profile_status_icon.py @@ -34,6 +34,7 @@ def create_custom_browser_profile(client, name='My Custom Chrome'): 'ignore_https_errors': '', 'user_agent': '', 'locale': '', + 'custom_headers': '', 'original_machine_name': '', }, follow_redirects=True, @@ -43,6 +44,31 @@ def create_custom_browser_profile(client, name='My Custom Chrome'): return BrowserProfile(name=name, fetch_backend='playwright_cdp').get_machine_name() +def create_requests_browser_profile(client, name, user_agent='', custom_headers=''): + """Create a requests-type browser profile with optional UA and custom headers.""" + res = client.post( + url_for('settings.settings_browsers.save'), + data={ + 'name': name, + 'fetch_backend': 'requests', + 'browser_connection_url': '', + 'viewport_width': 1280, + 'viewport_height': 1000, + 'block_images': '', + 'block_fonts': '', + 'ignore_https_errors': '', + 'user_agent': user_agent, + 'locale': '', + 'custom_headers': custom_headers, + 'original_machine_name': '', + }, + follow_redirects=True, + ) + assert b'saved.' in res.data + from changedetectionio.model.browser_profile import BrowserProfile + return BrowserProfile(name=name, fetch_backend='requests').get_machine_name() + + # --------------------------------------------------------------------------- # Unit tests — status_icon attribute on fetcher classes # --------------------------------------------------------------------------- @@ -155,3 +181,101 @@ def test_system_default_browser_inherited_by_watch(client, live_server, measure_ datastore.delete(uuid) set_system_default_profile(client, 'direct_http_requests') client.get(url_for('settings.settings_browsers.delete', machine_name=machine_name), follow_redirects=True) + +# --------------------------------------------------------------------------- +# Integration tests — BrowserProfile UA and custom_headers applied to requests +# --------------------------------------------------------------------------- + +def test_browser_profile_user_agent_applied(client, live_server, measure_memory_usage, datastore_path): + """User-Agent set on a BrowserProfile appears in the fetched request; + a per-watch User-Agent header overrides it.""" + from changedetectionio.tests.util import wait_for_all_checks + + datastore = client.application.config.get('DATASTORE') + test_url = url_for('test_headers', _external=True) + + machine_name = create_requests_browser_profile( + client, name='UA Profile Test', user_agent='profile-ua/2.0' + ) + + uuid = datastore.add_watch(url=test_url, extras={'browser_profile': machine_name}) + client.get(url_for('ui.form_watch_checknow'), follow_redirects=True) + wait_for_all_checks(client) + + res = client.get(url_for('ui.ui_preview.preview_page', uuid='first'), follow_redirects=True) + assert b'profile-ua/2.0' in res.data, "Profile UA should appear in the echoed request headers" + + # Per-watch User-Agent header overrides the profile UA + client.post( + url_for('ui.ui_edit.edit_page', uuid='first'), + data={ + 'url': test_url, + 'tags': '', + 'browser_profile': machine_name, + 'headers': 'User-Agent: watch-ua/3.0', + 'time_between_check_use_default': 'y', + }, + follow_redirects=True, + ) + client.get(url_for('ui.form_watch_checknow'), follow_redirects=True) + wait_for_all_checks(client) + + res = client.get(url_for('ui.ui_preview.preview_page', uuid='first'), follow_redirects=True) + assert b'watch-ua/3.0' in res.data, "Watch-level UA should override profile UA" + assert b'profile-ua/2.0' not in res.data, "Profile UA should be superseded by watch-level header" + + datastore.delete(uuid) + client.get(url_for('settings.settings_browsers.delete', machine_name=machine_name), follow_redirects=True) + + +def test_browser_profile_custom_headers_applied(client, live_server, measure_memory_usage, datastore_path): + """Custom headers set on a BrowserProfile are sent with every request using that profile; + per-watch headers override them when the same header name is used.""" + from changedetectionio.tests.util import wait_for_all_checks + + datastore = client.application.config.get('DATASTORE') + test_url = url_for('test_headers', _external=True) + + machine_name = create_requests_browser_profile( + client, + name='Headers Profile Test', + custom_headers='X-Profile-Header: profile-value\nX-Shared-Header: from-profile', + ) + + uuid = datastore.add_watch(url=test_url, extras={'browser_profile': machine_name}) + client.get(url_for('ui.form_watch_checknow'), follow_redirects=True) + wait_for_all_checks(client) + + res = client.get(url_for('ui.ui_preview.preview_page', uuid='first'), follow_redirects=True) + assert b'X-Profile-Header:profile-value' in res.data, \ + "Profile custom header should appear in the echoed request" + assert b'X-Shared-Header:from-profile' in res.data, \ + "Second profile custom header should appear" + + # Per-watch header for the same key overrides the profile header + client.post( + url_for('ui.ui_edit.edit_page', uuid='first'), + data={ + 'url': test_url, + 'tags': '', + 'browser_profile': machine_name, + 'headers': 'X-Shared-Header: from-watch\nX-Watch-Only: watch-value', + 'time_between_check_use_default': 'y', + }, + follow_redirects=True, + ) + client.get(url_for('ui.form_watch_checknow'), follow_redirects=True) + wait_for_all_checks(client) + + res = client.get(url_for('ui.ui_preview.preview_page', uuid='first'), follow_redirects=True) + assert b'X-Profile-Header:profile-value' in res.data, \ + "Unrelated profile header should still be present" + assert b'X-Shared-Header:from-watch' in res.data, \ + "Watch-level header should override the same-named profile header" + assert b'X-Shared-Header:from-profile' not in res.data, \ + "Profile value for overridden header should be gone" + assert b'X-Watch-Only:watch-value' in res.data, \ + "Watch-only header should appear" + + datastore.delete(uuid) + client.get(url_for('settings.settings_browsers.delete', machine_name=machine_name), follow_redirects=True) diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py index a573ccef3..e499110d3 100644 --- a/changedetectionio/tests/test_request.py +++ b/changedetectionio/tests/test_request.py @@ -249,62 +249,60 @@ def test_method_in_request(client, live_server, measure_memory_usage, datastore_ delete_all_watches(client) -# Re #2408 - user-agent override test, also should handle case-insensitive header deduplication +# Re #2408 - user-agent override via BrowserProfile; per-watch headers override the profile UA def test_ua_global_override(client, live_server, measure_memory_usage, datastore_path): - - if os.getenv('WEBDRIVER_URL'): - print("Selenium doesnt support custom HTTP headers!!") - return - - - ## live_server_setup(live_server) # Setup on conftest per function test_url = url_for('test_headers', _external=True) + datastore = client.application.config.get('DATASTORE') + # Create a requests-type browser profile with a custom UA res = client.post( - url_for("settings.settings_page"), + url_for('settings.settings_browsers.save'), data={ - "application-minutes_between_check": 180, - "requests-default_ua-requests": "html-requests-user-agent" + 'name': 'UA Test Profile', + 'fetch_backend': 'requests', + 'browser_connection_url': '', + 'viewport_width': 1280, + 'viewport_height': 1000, + 'block_images': '', + 'block_fonts': '', + 'ignore_https_errors': '', + 'user_agent': 'profile-ua-test/1.0', + 'locale': '', + 'custom_headers': '', + 'original_machine_name': '', }, - follow_redirects=True + follow_redirects=True, ) - assert b'Settings updated' in res.data + assert b'saved.' in res.data - # Force requests fetcher so default_ua['requests'] applies regardless of system default browser - uuid = client.application.config.get('DATASTORE').add_watch(url=test_url, extras={'browser_profile': 'direct_http_requests'}) + from changedetectionio.model.browser_profile import BrowserProfile + profile_machine_name = BrowserProfile(name='UA Test Profile', fetch_backend='requests').get_machine_name() + + uuid = datastore.add_watch(url=test_url, extras={'browser_profile': profile_machine_name}) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) - wait_for_all_checks(client) - res = client.get( - url_for("ui.ui_preview.preview_page", uuid="first"), - follow_redirects=True - ) - assert b"html-requests-user-agent" in res.data - # default user-agent should have shown by now - # now add a custom one in the headers + res = client.get(url_for("ui.ui_preview.preview_page", uuid="first"), follow_redirects=True) + assert b"profile-ua-test/1.0" in res.data - - # Add some headers to a request + # Per-watch User-Agent header should override the profile UA (case-insensitive) res = client.post( url_for("ui.ui_edit.edit_page", uuid="first"), data={ "url": test_url, - "tags": "testtag", - "browser_profile": 'direct_http_requests', - # Important - also test case-insensitive + "tags": "", + "browser_profile": profile_machine_name, "headers": "User-AGent: agent-from-watch", "time_between_check_use_default": "y"}, follow_redirects=True ) assert b"Updated watch." in res.data wait_for_all_checks(client) - res = client.get( - url_for("ui.ui_preview.preview_page", uuid="first"), - follow_redirects=True - ) + res = client.get(url_for("ui.ui_preview.preview_page", uuid="first"), follow_redirects=True) assert b"agent-from-watch" in res.data - assert b"html-requests-user-agent" not in res.data + assert b"profile-ua-test/1.0" not in res.data + + client.get(url_for('settings.settings_browsers.delete', machine_name=profile_machine_name), follow_redirects=True) delete_all_watches(client) def test_headers_textfile_in_request(client, live_server, measure_memory_usage, datastore_path): @@ -314,40 +312,11 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage, print("Selenium doesnt support custom HTTP headers!!") return - # Add our URL to the import page - webdriver_ua = "Hello fancy webdriver UA 1.0" - requests_ua = "Hello basic requests UA 1.1" - test_url = url_for('test_headers', _external=True) if os.getenv('PLAYWRIGHT_DRIVER_URL'): # Because its no longer calling back to localhost but from the browser container, set in test-only.yml test_url = test_url.replace('localhost', 'cdio') - form_data = { - "application-minutes_between_check": 180, - "requests-default_ua-requests": requests_ua - } - - if os.getenv('PLAYWRIGHT_DRIVER_URL'): - form_data["requests-default_ua-playwright"] = webdriver_ua - - res = client.post( - url_for("settings.settings_page"), - data=form_data, - follow_redirects=True - ) - assert b'Settings updated' in res.data - - res = client.get(url_for("settings.settings_page")) - - # Only when some kind of real browser is setup - if os.getenv('PLAYWRIGHT_DRIVER_URL'): - assert b'requests-default_ua-playwright' in res.data - - # Field should always be there - assert b"requests-default_ua-requests" in res.data - - # Add the test URL twice, we will check uuid = client.application.config.get('DATASTORE').add_watch(url=test_url) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) @@ -410,12 +379,6 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage, assert b"Url-Header-Global:http://example.com/global" in res.data assert b"Url-Header-Watch:http://example.com/watch" in res.data - # Check the custom UA from system settings page made it through - if os.getenv('PLAYWRIGHT_DRIVER_URL'): - assert "User-Agent:".encode('utf-8') + webdriver_ua.encode('utf-8') in res.data - else: - assert "User-Agent:".encode('utf-8') + requests_ua.encode('utf-8') in res.data - # unlink headers.txt on start/stop delete_all_watches(client)