Move UA browser profiles only

changedetectionio/blueprint/settings/browser_profile/__init__.py

@@ -118,6 +118,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             'ignore_https_errors': bool(browser_profile_form.ignore_https_errors.data),
             'user_agent': browser_profile_form.user_agent.data or None,
             'locale': browser_profile_form.locale.data or None,
+            'custom_headers': browser_profile_form.custom_headers.data or '',
             'is_builtin': False,
         }
 

changedetectionio/blueprint/settings/browser_profile/templates/browser_profiles.html

@@ -108,6 +108,10 @@
                         {{ render_field(browser_profile_form.locale) }}
                         <span class="pure-form-message-inline">{{ _('Sets Accept-Language and navigator.language (e.g. en-US, de-DE).') }}</span>
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_field(browser_profile_form.custom_headers) }}
+                        <span class="pure-form-message-inline">{{ _('Extra HTTP headers for all requests using this profile (one per line, Key: Value). Applied before per-watch headers.') }}</span>
+                    </div>
                     <div class="pure-control-group">
                         <button type="submit" class="pure-button pure-button-primary" id="profile-submit-btn">{{ _('Save profile') }}</button>
                         {% if editing_machine_name %}

changedetectionio/blueprint/settings/templates/settings.html

@@ -140,13 +140,6 @@
                     {{ render_field(form.requests.form.timeout) }}
                     <span class="pure-form-message-inline">{{ _('For regular plain requests (not chrome based), maximum number of seconds until timeout, 1-999.') }}</span><br>
                 </div>
-                <div class="pure-control-group inline-radio">
-                    {{ render_field(form.requests.form.default_ua) }}
-                    <span class="pure-form-message-inline">
-                        {{ _('Applied to all requests.') }}<br><br>
-                        {{ _('Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it\'s important to consider') }} <a href="https://changedetection.io/tutorial/what-are-main-types-anti-robot-mechanisms">{{ _('all of the ways that the browser is detected') }}</a>.
-                    </span>
-                </div>
                 <div class="pure-control-group">
                 <br>
                     {{ _('Tip:') }} <a href="{{ url_for('settings.settings_page')}}#proxies">{{ _('Connect using Bright Data proxies, find out more here.') }}</a>

changedetectionio/content_fetchers/playwright/CDP.py

@@ -12,7 +12,6 @@ from changedetectionio.content_fetchers.playwright import PlaywrightBaseFetcher
 class fetcher(PlaywrightBaseFetcher):
     fetcher_description = "Playwright Chrome (CDP/Remote)"
     requires_connection_url = True
-    ua_settings_key = 'playwright'  # matches DefaultUAInputForm field name
 
     def __init__(self, proxy_override=None, custom_browser_connection_url=None, **kwargs):
         super().__init__(proxy_override=proxy_override, custom_browser_connection_url=custom_browser_connection_url, **kwargs)

changedetectionio/content_fetchers/puppeteer.py

@@ -181,7 +181,6 @@ class fetcher(Fetcher):
     supports_xpath_element_data = True
 
     status_icon = {'filename': 'google-chrome-icon.png', 'alt': 'Using a Chrome browser', 'title': 'Using a Chrome browser'}
-    ua_settings_key = 'playwright'  # shares the 'playwright' UA field in settings UI
 
     def disk_cleanup_after_fetch(self):
         self.delete_browser_steps_screenshots()

changedetectionio/content_fetchers/webdriver_selenium.py

@@ -20,7 +20,6 @@ class fetcher(Fetcher):
     supports_xpath_element_data = True
 
     status_icon = {'filename': 'google-chrome-icon.png', 'alt': 'Using a Chrome browser', 'title': 'Using a Chrome browser'}
-    ua_settings_key = 'selenium'  # matches DefaultUAInputForm field name
 
     def __init__(self, proxy_override=None, custom_browser_connection_url=None, **kwargs):
         super().__init__(**kwargs)

changedetectionio/forms.py

@@ -991,13 +991,15 @@ class BrowserProfileForm(Form):
         [validators.Optional(), validators.Length(max=20)],
         render_kw={"placeholder": "en-US, de-DE, fr-FR …", "size": 15}
     )
-
-class DefaultUAInputForm(Form):
-    requests = StringField(_l('Plaintext requests'), validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
-    if os.getenv("PLAYWRIGHT_DRIVER_URL") or os.getenv("WEBDRIVER_URL"):
-        playwright = StringField(_l('Chrome/Playwright requests'), validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
-        selenium = StringField(_l('Chrome/Selenium requests'), validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
-        puppeteer = StringField(_l('Chrome/Puppeteer requests'), validators=[validators.Optional()], render_kw={"placeholder": "<default>"})
+    custom_headers = TextAreaField(
+        _l('Custom headers'),
+        [validators.Optional()],
+        render_kw={
+            "placeholder": "Header-Name: value\nAnother-Header: value",
+            "rows": 4, "cols": 60,
+            "style": "font-family:monospace;"
+        }
+    )
 
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
@@ -1021,8 +1023,6 @@ class globalSettingsRequestForm(Form):
     extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5)
     extra_browsers = FieldList(FormField(SingleExtraBrowser), min_entries=5)
 
-    default_ua = FormField(DefaultUAInputForm, label=_l("Default User-Agent overrides"))
-
     def validate_extra_proxies(self, extra_validators=None):
         for e in self.data['extra_proxies']:
             if e.get('proxy_name') or e.get('proxy_url'):

changedetectionio/model/App.py

@@ -12,7 +12,6 @@ from changedetectionio.notification import (
 
 # Equal to or greater than this number of FilterNotFoundInResponse exceptions will trigger a filter-not-found notification
 _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT = 6
-DEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'
 
 
 
@@ -31,12 +30,6 @@ class model(dict):
                     'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
                     'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")),  # Default 45 seconds
                     'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "5")),  # Number of threads, lower is better for slow connections
-                    'default_ua': {
-                        'requests': getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", DEFAULT_SETTINGS_HEADERS_USERAGENT),
-                        'playwright': None,
-                        'selenium': None,
-                        'puppeteer': None,
-                    }
                 },
                 'application': {
                     # Custom notification content

changedetectionio/model/browser_profile.py

@@ -39,12 +39,20 @@ references.  After that migration no legacy paths are needed here.
 
 from __future__ import annotations
 
+import os
 import re
 from typing import Optional
 
 from loguru import logger
 from pydantic import BaseModel, field_validator
 
+# Default User-Agent for the built-in plaintext requests profile.
+# Overridable via environment variable for deployments that need a custom UA.
+_DEFAULT_REQUESTS_UA = os.getenv(
+    "DEFAULT_SETTINGS_HEADERS_USERAGENT",
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'
+)
+
 # ---------------------------------------------------------------------------
 # Constants
 # ---------------------------------------------------------------------------
@@ -139,6 +147,13 @@ class BrowserProfile(BaseModel):
     Some sites serve different prices or copy based on locale.
     """
 
+    custom_headers: str = ''
+    """
+    Extra HTTP headers sent with every request using this profile, in ``Key: Value`` format
+    (one per line, ``#`` lines are ignored).  Applied before per-watch headers so
+    individual watches can override them.
+    """
+
     service_workers: str = 'allow'
     """
     Whether to allow Service Workers in the browser context.
@@ -241,6 +256,7 @@ BUILTIN_REQUESTS = BrowserProfile(
     name='Direct HTTP (requests)',
     fetch_backend='requests',
     is_builtin=True,
+    user_agent=_DEFAULT_REQUESTS_UA,
 )
 
 BUILTIN_PLAYWRIGHT = BrowserProfile(

changedetectionio/processors/base.py

@@ -186,10 +186,17 @@ class difference_detection_processor():
         from changedetectionio.jinja2_custom import render as jinja_render
         request_headers = CaseInsensitiveDict()
 
-        ua = self.datastore.data['settings']['requests'].get('default_ua')
-        ua_key = getattr(fetcher_obj, 'ua_settings_key', fetcher_class_name)
-        if ua and ua.get(ua_key):
-            request_headers.update({'User-Agent': ua.get(ua_key)})
+        # Browser profile: UA override (lowest priority — watch headers override this)
+        if profile.user_agent:
+            request_headers['User-Agent'] = profile.user_agent
+
+        # Browser profile: custom headers (override profile UA, but watch headers override these)
+        if profile.custom_headers:
+            for line in profile.custom_headers.splitlines():
+                line = line.strip()
+                if not line.startswith('#') and ':' in line:
+                    k, v = line.split(':', 1)
+                    request_headers[k.strip()] = v.strip()
 
         request_headers.update(self.watch.get('headers', {}))
         request_headers.update(self.datastore.get_all_base_headers())

changedetectionio/tests/test_browser_profile_status_icon.py

@@ -34,6 +34,7 @@ def create_custom_browser_profile(client, name='My Custom Chrome'):
             'ignore_https_errors': '',
             'user_agent': '',
             'locale': '',
+            'custom_headers': '',
             'original_machine_name': '',
         },
         follow_redirects=True,
@@ -43,6 +44,31 @@ def create_custom_browser_profile(client, name='My Custom Chrome'):
     return BrowserProfile(name=name, fetch_backend='playwright_cdp').get_machine_name()
 
 
+def create_requests_browser_profile(client, name, user_agent='', custom_headers=''):
+    """Create a requests-type browser profile with optional UA and custom headers."""
+    res = client.post(
+        url_for('settings.settings_browsers.save'),
+        data={
+            'name': name,
+            'fetch_backend': 'requests',
+            'browser_connection_url': '',
+            'viewport_width': 1280,
+            'viewport_height': 1000,
+            'block_images': '',
+            'block_fonts': '',
+            'ignore_https_errors': '',
+            'user_agent': user_agent,
+            'locale': '',
+            'custom_headers': custom_headers,
+            'original_machine_name': '',
+        },
+        follow_redirects=True,
+    )
+    assert b'saved.' in res.data
+    from changedetectionio.model.browser_profile import BrowserProfile
+    return BrowserProfile(name=name, fetch_backend='requests').get_machine_name()
+
+
 # ---------------------------------------------------------------------------
 # Unit tests — status_icon attribute on fetcher classes
 # ---------------------------------------------------------------------------
@@ -155,3 +181,101 @@ def test_system_default_browser_inherited_by_watch(client, live_server, measure_
     datastore.delete(uuid)
     set_system_default_profile(client, 'direct_http_requests')
     client.get(url_for('settings.settings_browsers.delete', machine_name=machine_name), follow_redirects=True)
+
+# ---------------------------------------------------------------------------
+# Integration tests — BrowserProfile UA and custom_headers applied to requests
+# ---------------------------------------------------------------------------
+
+def test_browser_profile_user_agent_applied(client, live_server, measure_memory_usage, datastore_path):
+    """User-Agent set on a BrowserProfile appears in the fetched request;
+    a per-watch User-Agent header overrides it."""
+    from changedetectionio.tests.util import wait_for_all_checks
+
+    datastore = client.application.config.get('DATASTORE')
+    test_url = url_for('test_headers', _external=True)
+
+    machine_name = create_requests_browser_profile(
+        client, name='UA Profile Test', user_agent='profile-ua/2.0'
+    )
+
+    uuid = datastore.add_watch(url=test_url, extras={'browser_profile': machine_name})
+    client.get(url_for('ui.form_watch_checknow'), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    res = client.get(url_for('ui.ui_preview.preview_page', uuid='first'), follow_redirects=True)
+    assert b'profile-ua/2.0' in res.data, "Profile UA should appear in the echoed request headers"
+
+    # Per-watch User-Agent header overrides the profile UA
+    client.post(
+        url_for('ui.ui_edit.edit_page', uuid='first'),
+        data={
+            'url': test_url,
+            'tags': '',
+            'browser_profile': machine_name,
+            'headers': 'User-Agent: watch-ua/3.0',
+            'time_between_check_use_default': 'y',
+        },
+        follow_redirects=True,
+    )
+    client.get(url_for('ui.form_watch_checknow'), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    res = client.get(url_for('ui.ui_preview.preview_page', uuid='first'), follow_redirects=True)
+    assert b'watch-ua/3.0' in res.data, "Watch-level UA should override profile UA"
+    assert b'profile-ua/2.0' not in res.data, "Profile UA should be superseded by watch-level header"
+
+    datastore.delete(uuid)
+    client.get(url_for('settings.settings_browsers.delete', machine_name=machine_name), follow_redirects=True)
+
+
+def test_browser_profile_custom_headers_applied(client, live_server, measure_memory_usage, datastore_path):
+    """Custom headers set on a BrowserProfile are sent with every request using that profile;
+    per-watch headers override them when the same header name is used."""
+    from changedetectionio.tests.util import wait_for_all_checks
+
+    datastore = client.application.config.get('DATASTORE')
+    test_url = url_for('test_headers', _external=True)
+
+    machine_name = create_requests_browser_profile(
+        client,
+        name='Headers Profile Test',
+        custom_headers='X-Profile-Header: profile-value\nX-Shared-Header: from-profile',
+    )
+
+    uuid = datastore.add_watch(url=test_url, extras={'browser_profile': machine_name})
+    client.get(url_for('ui.form_watch_checknow'), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    res = client.get(url_for('ui.ui_preview.preview_page', uuid='first'), follow_redirects=True)
+    assert b'X-Profile-Header:profile-value' in res.data, \
+        "Profile custom header should appear in the echoed request"
+    assert b'X-Shared-Header:from-profile' in res.data, \
+        "Second profile custom header should appear"
+
+    # Per-watch header for the same key overrides the profile header
+    client.post(
+        url_for('ui.ui_edit.edit_page', uuid='first'),
+        data={
+            'url': test_url,
+            'tags': '',
+            'browser_profile': machine_name,
+            'headers': 'X-Shared-Header: from-watch\nX-Watch-Only: watch-value',
+            'time_between_check_use_default': 'y',
+        },
+        follow_redirects=True,
+    )
+    client.get(url_for('ui.form_watch_checknow'), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    res = client.get(url_for('ui.ui_preview.preview_page', uuid='first'), follow_redirects=True)
+    assert b'X-Profile-Header:profile-value' in res.data, \
+        "Unrelated profile header should still be present"
+    assert b'X-Shared-Header:from-watch' in res.data, \
+        "Watch-level header should override the same-named profile header"
+    assert b'X-Shared-Header:from-profile' not in res.data, \
+        "Profile value for overridden header should be gone"
+    assert b'X-Watch-Only:watch-value' in res.data, \
+        "Watch-only header should appear"
+
+    datastore.delete(uuid)
+    client.get(url_for('settings.settings_browsers.delete', machine_name=machine_name), follow_redirects=True)

changedetectionio/tests/test_request.py

@@ -249,62 +249,60 @@ def test_method_in_request(client, live_server, measure_memory_usage, datastore_
 
     delete_all_watches(client)
 
-# Re #2408 - user-agent override test, also should handle case-insensitive header deduplication
+# Re #2408 - user-agent override via BrowserProfile; per-watch headers override the profile UA
 def test_ua_global_override(client, live_server, measure_memory_usage, datastore_path):
-
-    if os.getenv('WEBDRIVER_URL'):
-        print("Selenium doesnt support custom HTTP headers!!")
-        return
-
-
-    ##  live_server_setup(live_server) # Setup on conftest per function
     test_url = url_for('test_headers', _external=True)
+    datastore = client.application.config.get('DATASTORE')
 
+    # Create a requests-type browser profile with a custom UA
     res = client.post(
-        url_for("settings.settings_page"),
+        url_for('settings.settings_browsers.save'),
         data={
-            "application-minutes_between_check": 180,
-            "requests-default_ua-requests": "html-requests-user-agent"
+            'name': 'UA Test Profile',
+            'fetch_backend': 'requests',
+            'browser_connection_url': '',
+            'viewport_width': 1280,
+            'viewport_height': 1000,
+            'block_images': '',
+            'block_fonts': '',
+            'ignore_https_errors': '',
+            'user_agent': 'profile-ua-test/1.0',
+            'locale': '',
+            'custom_headers': '',
+            'original_machine_name': '',
         },
-        follow_redirects=True
+        follow_redirects=True,
     )
-    assert b'Settings updated' in res.data
+    assert b'saved.' in res.data
 
-    # Force requests fetcher so default_ua['requests'] applies regardless of system default browser
-    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url, extras={'browser_profile': 'direct_http_requests'})
+    from changedetectionio.model.browser_profile import BrowserProfile
+    profile_machine_name = BrowserProfile(name='UA Test Profile', fetch_backend='requests').get_machine_name()
+
+    uuid = datastore.add_watch(url=test_url, extras={'browser_profile': profile_machine_name})
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-
     wait_for_all_checks(client)
-    res = client.get(
-        url_for("ui.ui_preview.preview_page", uuid="first"),
-        follow_redirects=True
-    )
 
-    assert b"html-requests-user-agent" in res.data
-    # default user-agent should have shown by now
-    # now add a custom one in the headers
+    res = client.get(url_for("ui.ui_preview.preview_page", uuid="first"), follow_redirects=True)
+    assert b"profile-ua-test/1.0" in res.data
 
-
-    # Add some headers to a request
+    # Per-watch User-Agent header should override the profile UA (case-insensitive)
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={
             "url": test_url,
-            "tags": "testtag",
-            "browser_profile": 'direct_http_requests',
-            # Important - also test case-insensitive
+            "tags": "",
+            "browser_profile": profile_machine_name,
             "headers": "User-AGent: agent-from-watch",
             "time_between_check_use_default": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
-    res = client.get(
-        url_for("ui.ui_preview.preview_page", uuid="first"),
-        follow_redirects=True
-    )
+    res = client.get(url_for("ui.ui_preview.preview_page", uuid="first"), follow_redirects=True)
     assert b"agent-from-watch" in res.data
-    assert b"html-requests-user-agent" not in res.data
+    assert b"profile-ua-test/1.0" not in res.data
+
+    client.get(url_for('settings.settings_browsers.delete', machine_name=profile_machine_name), follow_redirects=True)
     delete_all_watches(client)
 
 def test_headers_textfile_in_request(client, live_server, measure_memory_usage, datastore_path):
@@ -314,40 +312,11 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage,
         print("Selenium doesnt support custom HTTP headers!!")
         return
 
-    # Add our URL to the import page
-    webdriver_ua = "Hello fancy webdriver UA 1.0"
-    requests_ua = "Hello basic requests UA 1.1"
-
     test_url = url_for('test_headers', _external=True)
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):
         # Because its no longer calling back to localhost but from the browser container, set in test-only.yml
         test_url = test_url.replace('localhost', 'cdio')
 
-    form_data = {
-        "application-minutes_between_check": 180,
-        "requests-default_ua-requests": requests_ua
-    }
-
-    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        form_data["requests-default_ua-playwright"] = webdriver_ua
-
-    res = client.post(
-        url_for("settings.settings_page"),
-        data=form_data,
-        follow_redirects=True
-    )
-    assert b'Settings updated' in res.data
-
-    res = client.get(url_for("settings.settings_page"))
-
-    # Only when some kind of real browser is setup
-    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        assert b'requests-default_ua-playwright' in res.data
-
-    # Field should always be there
-    assert b"requests-default_ua-requests" in res.data
-
-    # Add the test URL twice, we will check
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
@@ -410,12 +379,6 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage,
     assert b"Url-Header-Global:http://example.com/global" in res.data
     assert b"Url-Header-Watch:http://example.com/watch" in res.data
 
-    # Check the custom UA from system settings page made it through
-    if os.getenv('PLAYWRIGHT_DRIVER_URL'):
-        assert "User-Agent:".encode('utf-8') + webdriver_ua.encode('utf-8') in res.data
-    else:
-        assert "User-Agent:".encode('utf-8') + requests_ua.encode('utf-8') in res.data
-
     # unlink headers.txt on start/stop
     delete_all_watches(client)