Adding help link

.github/workflows/test-only.yml

@@ -30,7 +30,10 @@ jobs:
 
           # Selenium+browserless
           docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
-          docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.60-chrome-stable
+          docker run --network changedet-network -d --name browserless --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.60-chrome-stable
+          
+          # For accessing custom browser tests
+          docker run --network changedet-network -d --name browserless-custom-url --hostname browserless-custom-url -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm --shm-size="2g"  browserless/chrome:1.60-chrome-stable
 
       - name: Build changedetection.io container for testing
         run: |         
@@ -48,6 +51,7 @@ jobs:
         run: |
           # Unit tests
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
           
           # All tests
           docker run --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
@@ -86,6 +90,12 @@ jobs:
           # And again with PLAYWRIGHT_DRIVER_URL=..
           cd ..
 
+      - name: Test custom browser URL
+        run: |
+          cd changedetectionio
+          ./run_custom_browser_url_tests.sh
+          cd ..
+
       - name: Test changedetection.io container starts+runs basically without error
         run: |
           docker run -p 5556:5000 -d test-changedetectionio

Dockerfile

@@ -25,7 +25,7 @@ RUN pip install --target=/dependencies -r /requirements.txt
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN pip install --target=/dependencies playwright~=1.39 \
+RUN pip install --target=/dependencies playwright~=1.40 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage

README.md

@@ -268,3 +268,7 @@ I offer commercial support, this software is depended on by network security, ae
 [license-shield]: https://img.shields.io/github/license/dgtlmoon/changedetection.io.svg?style=for-the-badge
 [release-link]: https://github.com/dgtlmoon/changedetection.io/releases
 [docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io
+
+## Third-party licenses
+
+changedetectionio.html_tools.elementpath_tostring: Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati), Licensed under [MIT license](https://github.com/sissaschool/elementpath/blob/master/LICENSE)

changedetectionio/__init__.py

@@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.45.7.3'
+__version__ = '0.45.8.1'
 
 from changedetectionio.store import BASE_URL_NOT_SET_TEXT
 
@@ -614,6 +614,8 @@ def changedetection_app(config=None, datastore_o=None):
         # For the form widget tag uuid lookup
         form.tags.datastore = datastore # in _value
 
+        for p in datastore.extra_browsers:
+            form.fetch_backend.choices.append(p)
 
         form.fetch_backend.choices.append(("system", 'System settings default'))
 
@@ -714,7 +716,7 @@ def changedetection_app(config=None, datastore_o=None):
             system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
 
             is_html_webdriver = False
-            if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver':
+            if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
                 is_html_webdriver = True
 
             # Only works reliably with Playwright
@@ -819,6 +821,16 @@ def changedetection_app(config=None, datastore_o=None):
 
         return output
 
+    @app.route("/settings/reset-api-key", methods=['GET'])
+    @login_optionally_required
+    def settings_reset_api_key():
+        import secrets
+        secret = secrets.token_hex(16)
+        datastore.data['settings']['application']['api_access_token'] = secret
+        datastore.needs_write_urgent = True
+        flash("API Key was regenerated.")
+        return redirect(url_for('settings_page')+'#api')
+
     @app.route("/import", methods=['GET', "POST"])
     @login_optionally_required
     def import_page():
@@ -949,7 +961,7 @@ def changedetection_app(config=None, datastore_o=None):
         # Read as binary and force decode as UTF-8
         # Windows may fail decode in python if we just use 'r' mode (chardet decode exception)
         from_version = request.args.get('from_version')
-        from_version_index = -2 # second newest
+        from_version_index = -2  # second newest
         if from_version and from_version in dates:
             from_version_index = dates.index(from_version)
         else:
@@ -958,7 +970,7 @@ def changedetection_app(config=None, datastore_o=None):
         try:
             from_version_file_contents = watch.get_history_snapshot(dates[from_version_index])
         except Exception as e:
-            from_version_file_contents = "Unable to read to-version at index{}.\n".format(dates[from_version_index])
+            from_version_file_contents = f"Unable to read to-version at index {dates[from_version_index]}.\n"
 
         to_version = request.args.get('to_version')
         to_version_index = -1
@@ -977,7 +989,7 @@ def changedetection_app(config=None, datastore_o=None):
         system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
 
         is_html_webdriver = False
-        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver':
+        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
             is_html_webdriver = True
 
         password_enabled_and_share_is_off = False
@@ -1031,7 +1043,7 @@ def changedetection_app(config=None, datastore_o=None):
 
 
         is_html_webdriver = False
-        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver':
+        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
             is_html_webdriver = True
 
         # Never requested successfully, but we detected a fetch error

changedetectionio/api/api_v1.py

@@ -76,7 +76,7 @@ class Watch(Resource):
         # Properties are not returned as a JSON, so add the required props manually
         watch['history_n'] = watch.history_n
         watch['last_changed'] = watch.last_changed
-
+        watch['viewed'] = watch.viewed
         return watch
 
     @auth.check_token
@@ -280,11 +280,14 @@ class CreateWatch(Resource):
             if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
                 continue
 
-            list[uuid] = {'url': watch['url'],
+            list[uuid] = {
-                       'title': watch['title'],
+                'last_changed': watch.last_changed,
-                       'last_checked': watch['last_checked'],
+                'last_checked': watch['last_checked'],
-                       'last_changed': watch.last_changed,
+                'last_error': watch['last_error'],
-                       'last_error': watch['last_error']}
+                'title': watch['title'],
+                'url': watch['url'],
+                'viewed': watch.viewed
+            }
 
         if request.args.get('recheck_all'):
             for uuid in self.datastore.data['watching'].keys():

changedetectionio/blueprint/browser_steps/__init__.py

@@ -152,7 +152,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])
     def browsersteps_ui_update():
         import base64
-        import playwright._impl._api_types
+        import playwright._impl._errors
         global browsersteps_sessions
         from changedetectionio.blueprint.browser_steps import browser_steps
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -110,7 +110,7 @@ class steppable_browser_interface():
         self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
 
     def action_click_element_if_exists(self, selector, value):
-        import playwright._impl._api_types as _api_types
+        import playwright._impl._errors as _api_types
         print("Clicking element if exists")
         if not len(selector.strip()):
             return
@@ -123,6 +123,9 @@ class steppable_browser_interface():
             return
 
     def action_click_x_y(self, selector, value):
+        if not re.match(r'^\s?\d+\s?,\s?\d+\s?$', value):
+            raise Exception("'Click X,Y' step should be in the format of '100 , 90'")
+
         x, y = value.strip().split(',')
         x = int(float(x.strip()))
         y = int(float(y.strip()))

changedetectionio/blueprint/check_proxies/__init__.py

@@ -40,8 +40,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         contents = ''
         now = time.time()
         try:
-            update_handler = text_json_diff.perform_site_check(datastore=datastore)
+            update_handler = text_json_diff.perform_site_check(datastore=datastore, watch_uuid=uuid)
-            changed_detected, update_obj, contents = update_handler.run(uuid, preferred_proxy=preferred_proxy, skip_when_checksum_same=False)
+            update_handler.call_browser()
         # title, size is len contents not len xfer
         except content_fetcher.Non200ErrorCodeReceived as e:
             if e.status_code == 404:

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -69,11 +69,12 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                                 {% endif %}
                             </ul>
                         </li>
-                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
+                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash. To specify XPath to be used explicitly or the XPath rule starts with an XPath function: Prefix with <code>xpath:</code>
                             <ul>
-                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
+                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath:count(//*[contains(@class, 'sametext')])</code>, <a
                                 href="http://xpather.com/" target="new">test your XPath here</a></li>
                                 <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
+                                <li>To use XPath1.0: Prefix with <code>xpath1:</code></li>
                             </ul>
                             </li>
                     </ul>

changedetectionio/content_fetcher.py

@@ -96,6 +96,7 @@ class Fetcher():
     content = None
     error = None
     fetcher_description = "No description"
+    browser_connection_url = None
     headers = {}
     status_code = None
     webdriver_js_execute_code = None
@@ -171,7 +172,7 @@ class Fetcher():
 
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-        from playwright._impl._api_types import TimeoutError
+        from playwright._impl._errors import TimeoutError
         from jinja2 import Environment
         jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
 
@@ -251,14 +252,16 @@ class base_html_playwright(Fetcher):
 
     proxy = None
 
-    def __init__(self, proxy_override=None):
+    def __init__(self, proxy_override=None, browser_connection_url=None):
         super().__init__()
-        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+
         self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
-        self.command_executor = os.getenv(
+
-            "PLAYWRIGHT_DRIVER_URL",
+        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-            'ws://playwright-chrome:3000'
+        if not browser_connection_url:
-        ).strip('"')
+            self.browser_connection_url = os.getenv("PLAYWRIGHT_DRIVER_URL", 'ws://playwright-chrome:3000').strip('"')
+        else:
+            self.browser_connection_url = browser_connection_url
 
         # If any proxy settings are enabled, then we should setup the proxy object
         proxy_args = {}
@@ -419,11 +422,7 @@ class base_html_playwright(Fetcher):
             is_binary=False):
 
         # For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!)
-        has_browser_steps = self.browser_steps and list(filter(
+        if not self.browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-                self.browser_steps))
-
-        if not has_browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
             if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')):
                 # Temporary backup solution until we rewrite the playwright code
                 return self.run_fetch_browserless_puppeteer(
@@ -437,7 +436,7 @@ class base_html_playwright(Fetcher):
                     is_binary)
 
         from playwright.sync_api import sync_playwright
-        import playwright._impl._api_types
+        import playwright._impl._errors
 
         self.delete_browser_steps_screenshots()
         response = None
@@ -448,7 +447,7 @@ class base_html_playwright(Fetcher):
             # Seemed to cause a connection Exception even tho I can see it connect
             # self.browser = browser_type.connect(self.command_executor, timeout=timeout*1000)
             # 60,000 connection timeout only
-            browser = browser_type.connect_over_cdp(self.command_executor, timeout=60000)
+            browser = browser_type.connect_over_cdp(self.browser_connection_url, timeout=60000)
 
             # SOCKS5 with authentication is not supported (yet)
             # https://github.com/microsoft/playwright/issues/10567
@@ -490,7 +489,7 @@ class base_html_playwright(Fetcher):
             try:
                 if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
                     browsersteps_interface.action_execute_js(value=self.webdriver_js_execute_code, selector=None)
-            except playwright._impl._api_types.TimeoutError as e:
+            except playwright._impl._errors.TimeoutError as e:
                 context.close()
                 browser.close()
                 # This can be ok, we will try to grab what we could retrieve
@@ -508,7 +507,11 @@ class base_html_playwright(Fetcher):
             self.status_code = response.status
 
             if self.status_code != 200 and not ignore_status_codes:
-                raise Non200ErrorCodeReceived(url=url, status_code=self.status_code)
+
+                screenshot=self.page.screenshot(type='jpeg', full_page=True,
+                                     quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
+
+                raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
 
             if len(self.page.content().strip()) == 0:
                 context.close()
@@ -559,8 +562,6 @@ class base_html_webdriver(Fetcher):
     else:
         fetcher_description = "WebDriver Chrome/Javascript"
 
-    command_executor = ''
-
     # Configs for Proxy setup
     # In the ENV vars, is prefixed with "webdriver_", so it is for example "webdriver_sslProxy"
     selenium_proxy_settings_mappings = ['proxyType', 'ftpProxy', 'httpProxy', 'noProxy',
@@ -568,12 +569,15 @@ class base_html_webdriver(Fetcher):
                                         'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
     proxy = None
 
-    def __init__(self, proxy_override=None):
+    def __init__(self, proxy_override=None, browser_connection_url=None):
         super().__init__()
         from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
 
         # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-        self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
+        if not browser_connection_url:
+            self.browser_connection_url = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
+        else:
+            self.browser_connection_url = browser_connection_url
 
         # If any proxy settings are enabled, then we should setup the proxy object
         proxy_args = {}
@@ -615,7 +619,7 @@ class base_html_webdriver(Fetcher):
             options.proxy = self.proxy
 
         self.driver = webdriver.Remote(
-            command_executor=self.command_executor,
+            command_executor=self.browser_connection_url,
             options=options)
 
         try:
@@ -670,8 +674,10 @@ class base_html_webdriver(Fetcher):
 class html_requests(Fetcher):
     fetcher_description = "Basic fast Plaintext/HTTP Client"
 
-    def __init__(self, proxy_override=None):
+    def __init__(self, proxy_override=None, browser_connection_url=None):
+        super().__init__()
         self.proxy_override = proxy_override
+        # browser_connection_url is none because its always 'launched locally'
 
     def run(self,
             url,

changedetectionio/forms.py

@@ -168,7 +168,9 @@ class ValidateContentFetcherIsReady(object):
     def __call__(self, form, field):
         import urllib3.exceptions
         from changedetectionio import content_fetcher
+        return
 
+# AttributeError: module 'changedetectionio.content_fetcher' has no attribute 'extra_browser_unlocked<>ASDF213r123r'
         # Better would be a radiohandler that keeps a reference to each class
         if field.data is not None and field.data != 'system':
             klass = getattr(content_fetcher, field.data)
@@ -326,11 +328,30 @@ class ValidateCSSJSONXPATHInput(object):
                 return
 
             # Does it look like XPath?
-            if line.strip()[0] == '/':
+            if line.strip()[0] == '/' or line.strip().startswith('xpath:'):
+                if not self.allow_xpath:
+                    raise ValidationError("XPath not permitted in this field!")
+                from lxml import etree, html
+                import elementpath
+                # xpath 2.0-3.1
+                from elementpath.xpath3 import XPath3Parser
+                tree = html.fromstring("<html></html>")
+                line = line.replace('xpath:', '')
+
+                try:
+                    elementpath.select(tree, line.strip(), parser=XPath3Parser)
+                except elementpath.ElementPathError as e:
+                    message = field.gettext('\'%s\' is not a valid XPath expression. (%s)')
+                    raise ValidationError(message % (line, str(e)))
+                except:
+                    raise ValidationError("A system-error occurred when validating your XPath expression")
+
+            if line.strip().startswith('xpath1:'):
                 if not self.allow_xpath:
                     raise ValidationError("XPath not permitted in this field!")
                 from lxml import etree, html
                 tree = html.fromstring("<html></html>")
+                line = re.sub(r'^xpath1:', '', line)
 
                 try:
                     tree.xpath(line.strip())
@@ -496,6 +517,12 @@ class SingleExtraProxy(Form):
     proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "socks5:// or regular proxy http://user:pass@...:3128", "size":50})
     # @todo do the validation here instead
 
+class SingleExtraBrowser(Form):
+    browser_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
+    browser_connection_url = StringField('Browser connection URL', [validators.Optional()], render_kw={"placeholder": "wss://brightdata... wss://oxylabs etc", "size":50})
+    # @todo do the validation here instead
+
+
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
     time_between_check = FormField(TimeBetweenCheckForm)
@@ -504,6 +531,7 @@ class globalSettingsRequestForm(Form):
                                   render_kw={"style": "width: 5em;"},
                                   validators=[validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5)
+    extra_browsers = FieldList(FormField(SingleExtraBrowser), min_entries=5)
 
     def validate_extra_proxies(self, extra_validators=None):
         for e in self.data['extra_proxies']:

changedetectionio/html_tools.py

@@ -69,10 +69,89 @@ def element_removal(selectors: List[str], html_content):
     selector = ",".join(selectors)
     return subtractive_css_selector(selector, html_content)
 
+def elementpath_tostring(obj):
+    """
+    change elementpath.select results to string type
+    # The MIT License (MIT), Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati)
+    # https://github.com/sissaschool/elementpath/blob/dfcc2fd3d6011b16e02bf30459a7924f547b47d0/elementpath/xpath_tokens.py#L1038
+    """
+
+    import elementpath
+    from decimal import Decimal
+    import math
+
+    if obj is None:
+        return ''
+    # https://elementpath.readthedocs.io/en/latest/xpath_api.html#elementpath.select
+    elif isinstance(obj, elementpath.XPathNode):
+        return obj.string_value
+    elif isinstance(obj, bool):
+        return 'true' if obj else 'false'
+    elif isinstance(obj, Decimal):
+        value = format(obj, 'f')
+        if '.' in value:
+            return value.rstrip('0').rstrip('.')
+        return value
+
+    elif isinstance(obj, float):
+        if math.isnan(obj):
+            return 'NaN'
+        elif math.isinf(obj):
+            return str(obj).upper()
+
+        value = str(obj)
+        if '.' in value:
+            value = value.rstrip('0').rstrip('.')
+        if '+' in value:
+            value = value.replace('+', '')
+        if 'e' in value:
+            return value.upper()
+        return value
+
+    return str(obj)
 
 # Return str Utf-8 of matched rules
 def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False, is_rss=False):
     from lxml import etree, html
+    import elementpath
+    # xpath 2.0-3.1
+    from elementpath.xpath3 import XPath3Parser
+
+    parser = etree.HTMLParser()
+    if is_rss:
+        # So that we can keep CDATA for cdata_in_document_to_text() to process
+        parser = etree.XMLParser(strip_cdata=False)
+
+    tree = html.fromstring(bytes(html_content, encoding='utf-8'), parser=parser)
+    html_block = ""
+
+    r = elementpath.select(tree, xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'}, parser=XPath3Parser)
+    #@note: //title/text() wont work where <title>CDATA..
+
+    if type(r) != list:
+        r = [r]
+
+    for element in r:
+        # When there's more than 1 match, then add the suffix to separate each line
+        # And where the matched result doesn't include something that will cause Inscriptis to add a newline
+        # (This way each 'match' reliably has a new-line in the diff)
+        # Divs are converted to 4 whitespaces by inscriptis
+        if append_pretty_line_formatting and len(html_block) and (not hasattr( element, 'tag' ) or not element.tag in (['br', 'hr', 'div', 'p'])):
+            html_block += TEXT_FILTER_LIST_LINE_SUFFIX
+
+        if type(element) == str:
+            html_block += element
+        elif issubclass(type(element), etree._Element) or issubclass(type(element), etree._ElementTree):
+            html_block += etree.tostring(element, pretty_print=True).decode('utf-8')
+        else:
+            html_block += elementpath_tostring(element)
+
+    return html_block
+
+# Return str Utf-8 of matched rules
+# 'xpath1:'
+def xpath1_filter(xpath_filter, html_content, append_pretty_line_formatting=False, is_rss=False):
+    from lxml import etree, html
 
     parser = None
     if is_rss:

changedetectionio/model/App.py

@@ -16,6 +16,7 @@ class model(dict):
                 },
                 'requests': {
                     'extra_proxies': [], # Configurable extra proxies via the UI
+                    'extra_browsers': [],  # Configurable extra proxies via the UI
                     'jitter_seconds': 0,
                     'proxy': None, # Preferred proxy connection
                     'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},

changedetectionio/model/Watch.py

@@ -19,6 +19,7 @@ from changedetectionio.notification import (
 
 base_config = {
     'body': None,
+    'browser_steps': [],
     'browser_steps_last_error_step': None,
     'check_unique_lines': False,  # On change-detected, compare against all history if its something new
     'check_count': 0,
@@ -112,7 +113,8 @@ class model(dict):
 
     @property
     def viewed(self):
-        if int(self['last_viewed']) >= int(self.newest_history_key) :
+        # Don't return viewed when last_viewed is 0 and newest_key is 0
+        if int(self['last_viewed']) and int(self['last_viewed']) >= int(self.newest_history_key) :
             return True
 
         return False
@@ -145,8 +147,14 @@ class model(dict):
                 flash(message, 'error')
                 return ''
 
+        if ready_url.startswith('source:'):
+            ready_url=ready_url.replace('source:', '')
         return ready_url
 
+    @property
+    def is_source_type_url(self):
+        return self.get('url', '').startswith('source:')
+
     @property
     def get_fetch_backend(self):
         """
@@ -234,6 +242,14 @@ class model(dict):
         fname = os.path.join(self.watch_data_dir, "history.txt")
         return os.path.isfile(fname)
 
+    @property
+    def has_browser_steps(self):
+        has_browser_steps = self.get('browser_steps') and list(filter(
+                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
+                self.get('browser_steps')))
+
+        return  has_browser_steps
+
     # Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0.
     @property
     def newest_history_key(self):
@@ -247,6 +263,38 @@ class model(dict):
         bump = self.history
         return self.__newest_history_key
 
+    # Given an arbitrary timestamp, find the closest next key
+    # For example, last_viewed = 1000 so it should return the next 1001 timestamp
+    #
+    # used for the [diff] button so it can preset a smarter from_version
+    @property
+    def get_next_snapshot_key_to_last_viewed(self):
+
+        """Unfortunately for now timestamp is stored as string key"""
+        keys = list(self.history.keys())
+        if not keys:
+            return None
+
+        last_viewed = int(self.get('last_viewed'))
+        prev_k = keys[0]
+        sorted_keys = sorted(keys, key=lambda x: int(x))
+        sorted_keys.reverse()
+
+        # When the 'last viewed' timestamp is greater than the newest snapshot, return second last
+        if last_viewed > int(sorted_keys[0]):
+            return sorted_keys[1]
+
+        for k in sorted_keys:
+            if int(k) < last_viewed:
+                if prev_k == sorted_keys[0]:
+                    # Return the second last one so we dont recommend the same version compares itself
+                    return sorted_keys[1]
+
+                return prev_k
+            prev_k = k
+
+        return keys[0]
+
     def get_history_snapshot(self, timestamp):
         import brotli
         filepath = self.history[timestamp]

changedetectionio/notification.py

@@ -46,6 +46,9 @@ from apprise.decorators import notify
 @notify(on="puts")
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
+    from apprise.utils import parse_url as apprise_parse_url
+    from apprise.URLBase import URLBase
+
     url = kwargs['meta'].get('url')
 
     if url.startswith('post'):
@@ -68,16 +71,45 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     url = url.replace('delete://', 'http://')
     url = url.replace('deletes://', 'https://')
 
-    # Try to auto-guess if it's JSON
     headers = {}
+    params = {}
+    auth = None
+
+    # Convert /foobar?+some-header=hello to proper header dictionary
+    results = apprise_parse_url(url)
+    if results:
+        # Add our headers that the user can potentially over-ride if they wish
+        # to to our returned result set and tidy entries by unquoting them
+        headers = {URLBase.unquote(x): URLBase.unquote(y)
+                   for x, y in results['qsd+'].items()}
+
+        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
+        # but here we are making straight requests, so we need todo convert this against apprise's logic
+        for k, v in results['qsd'].items():
+            if not k.strip('+-') in results['qsd+'].keys():
+                params[URLBase.unquote(k)] = URLBase.unquote(v)
+
+        # Determine Authentication
+        auth = ''
+        if results.get('user') and results.get('password'):
+            auth = (URLBase.unquote(results.get('user')), URLBase.unquote(results.get('user')))
+        elif results.get('user'):
+            auth = (URLBase.unquote(results.get('user')))
+
+    # Try to auto-guess if it's JSON
     try:
         json.loads(body)
-        headers = {'Content-Type': 'application/json; charset=utf-8'}
+        headers['Content-Type'] = 'application/json; charset=utf-8'
     except ValueError as e:
         pass
 
-
+    r(results.get('url'),
-    r(url, headers=headers, data=body)
+      auth=auth,
+      data=body,
+      headers=headers,
+      params=params
+      )
 
 
 def process_notification(n_object, datastore):

changedetectionio/processors/__init__.py

@@ -1,15 +1,127 @@
 from abc import abstractmethod
+import os
 import hashlib
-
+import re
+from changedetectionio import content_fetcher
+from copy import deepcopy
+from distutils.util import strtobool
 
 class difference_detection_processor():
 
+    browser_steps = None
+    datastore = None
+    fetcher = None
+    screenshot = None
+    watch = None
+    xpath_data = None
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, datastore, watch_uuid, **kwargs):
         super().__init__(*args, **kwargs)
+        self.datastore = datastore
+        self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
+
+    def call_browser(self):
+
+        # Protect against file:// access
+        if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE):
+            if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
+                raise Exception(
+                    "file:// type access is denied for security reasons."
+                )
+
+        url = self.watch.link
+
+        # Requests, playwright, other browser via wss:// etc, fetch_extra_something
+        prefer_fetch_backend = self.watch.get('fetch_backend', 'system')
+
+        # Proxy ID "key"
+        preferred_proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=self.watch.get('uuid'))
+
+        # Pluggable content self.fetcher
+        if not prefer_fetch_backend or prefer_fetch_backend == 'system':
+            prefer_fetch_backend = self.datastore.data['settings']['application'].get('fetch_backend')
+
+        # In the case that the preferred fetcher was a browser config with custom connection URL..
+        # @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..)
+        browser_connection_url = None
+        if prefer_fetch_backend.startswith('extra_browser_'):
+            (t, key) = prefer_fetch_backend.split('extra_browser_')
+            connection = list(
+                filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', [])))
+            if connection:
+                prefer_fetch_backend = 'base_html_playwright'
+                browser_connection_url = connection[0].get('browser_connection_url')
+
+        # PDF should be html_requests because playwright will serve it up (so far) in a embedded page
+        # @todo https://github.com/dgtlmoon/changedetection.io/issues/2019
+        # @todo needs test to or a fix
+        if self.watch.is_pdf:
+           prefer_fetch_backend = "html_requests"
+
+        # Grab the right kind of 'fetcher', (playwright, requests, etc)
+        if hasattr(content_fetcher, prefer_fetch_backend):
+            fetcher_obj = getattr(content_fetcher, prefer_fetch_backend)
+        else:
+            # If the klass doesnt exist, just use a default
+            fetcher_obj = getattr(content_fetcher, "html_requests")
+
+
+        proxy_url = None
+        if preferred_proxy_id:
+            proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
+            print(f"Using proxy Key: {preferred_proxy_id} as Proxy URL {proxy_url}")
+
+        # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
+        # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
+        self.fetcher = fetcher_obj(proxy_override=proxy_url,
+                                   browser_connection_url=browser_connection_url
+                                   )
+
+        if self.watch.has_browser_steps:
+            self.fetcher.browser_steps = self.watch.get('browser_steps', [])
+            self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))
+
+        # Tweak the base config with the per-watch ones
+        request_headers = self.watch.get('headers', [])
+        request_headers.update(self.datastore.get_all_base_headers())
+        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=self.watch.get('uuid')))
+
+        # https://github.com/psf/requests/issues/4525
+        # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
+        # do this by accident.
+        if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
+            request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
+
+        timeout = self.datastore.data['settings']['requests'].get('timeout')
+
+        request_body = self.watch.get('body')
+        request_method = self.watch.get('method')
+        ignore_status_codes = self.watch.get('ignore_status_codes', False)
+
+        # Configurable per-watch or global extra delay before extracting text (for webDriver types)
+        system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)
+        if self.watch.get('webdriver_delay'):
+            self.fetcher.render_extract_delay = self.watch.get('webdriver_delay')
+        elif system_webdriver_delay is not None:
+            self.fetcher.render_extract_delay = system_webdriver_delay
+
+        if self.watch.get('webdriver_js_execute_code') is not None and self.watch.get('webdriver_js_execute_code').strip():
+            self.fetcher.webdriver_js_execute_code = self.watch.get('webdriver_js_execute_code')
+
+        # Requests for PDF's, images etc should be passwd the is_binary flag
+        is_binary = self.watch.is_pdf
+
+        # And here we go! call the right browser with browser-specific settings
+        self.fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, self.watch.get('include_filters'),
+                    is_binary=is_binary)
+
+        #@todo .quit here could go on close object, so we can run JS if change-detected
+        self.fetcher.quit()
+
+        # After init, call run_changedetection() which will do the actual change-detection
 
     @abstractmethod
-    def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
+    def run_changedetection(self, uuid, skip_when_checksum_same=True):
         update_obj = {'last_notification_error': False, 'last_error': False}
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()

changedetectionio/processors/restock_diff.py

@@ -1,10 +1,7 @@
 
 import hashlib
-import os
-import re
 import urllib3
 from . import difference_detection_processor
-from changedetectionio import content_fetcher
 from copy import deepcopy
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@@ -22,11 +19,7 @@ class perform_site_check(difference_detection_processor):
     screenshot = None
     xpath_data = None
 
-    def __init__(self, *args, datastore, **kwargs):
+    def run_changedetection(self, uuid, skip_when_checksum_same=True):
-        super().__init__(*args, **kwargs)
-        self.datastore = datastore
-
-    def run(self, uuid, skip_when_checksum_same=True):
 
         # DeepCopy so we can be sure we don't accidently change anything by reference
         watch = deepcopy(self.datastore.data['watching'].get(uuid))
@@ -34,84 +27,24 @@ class perform_site_check(difference_detection_processor):
         if not watch:
             raise Exception("Watch no longer exists.")
 
-        # Protect against file:// access
-        if re.search(r'^file', watch.get('url', ''), re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
-            raise Exception(
-                "file:// type access is denied for security reasons."
-            )
-
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
 
-        request_headers = watch.get('headers', [])
+        self.screenshot = self.fetcher.screenshot
-        request_headers.update(self.datastore.get_all_base_headers())
+        self.xpath_data = self.fetcher.xpath_data
-        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
-
-        # https://github.com/psf/requests/issues/4525
-        # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
-        # do this by accident.
-        if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
-            request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
-
-        timeout = self.datastore.data['settings']['requests'].get('timeout')
-
-        url = watch.link
-
-        request_body = self.datastore.data['watching'][uuid].get('body')
-        request_method = self.datastore.data['watching'][uuid].get('method')
-        ignore_status_codes = self.datastore.data['watching'][uuid].get('ignore_status_codes', False)
-
-        # Pluggable content fetcher
-        prefer_backend = watch.get_fetch_backend
-        if not prefer_backend or prefer_backend == 'system':
-            prefer_backend = self.datastore.data['settings']['application']['fetch_backend']
-
-        if hasattr(content_fetcher, prefer_backend):
-            klass = getattr(content_fetcher, prefer_backend)
-        else:
-            # If the klass doesnt exist, just use a default
-            klass = getattr(content_fetcher, "html_requests")
-
-        proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
-        proxy_url = None
-        if proxy_id:
-            proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')
-            print("UUID {} Using proxy {}".format(uuid, proxy_url))
-
-        fetcher = klass(proxy_override=proxy_url)
-
-        # Configurable per-watch or global extra delay before extracting text (for webDriver types)
-        system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)
-        if watch['webdriver_delay'] is not None:
-            fetcher.render_extract_delay = watch.get('webdriver_delay')
-        elif system_webdriver_delay is not None:
-            fetcher.render_extract_delay = system_webdriver_delay
-
-        # Could be removed if requests/plaintext could also return some info?
-        if prefer_backend != 'html_webdriver':
-            raise Exception("Re-stock detection requires Chrome or compatible webdriver/playwright fetcher to work")
-
-        if watch.get('webdriver_js_execute_code') is not None and watch.get('webdriver_js_execute_code').strip():
-            fetcher.webdriver_js_execute_code = watch.get('webdriver_js_execute_code')
-
-        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'))
-        fetcher.quit()
-
-        self.screenshot = fetcher.screenshot
-        self.xpath_data = fetcher.xpath_data
 
         # Track the content type
-        update_obj['content_type'] = fetcher.headers.get('Content-Type', '')
+        update_obj['content_type'] = self.fetcher.headers.get('Content-Type', '')
-        update_obj["last_check_status"] = fetcher.get_last_status_code()
+        update_obj["last_check_status"] = self.fetcher.get_last_status_code()
 
         # Main detection method
         fetched_md5 = None
-        if fetcher.instock_data:
+        if self.fetcher.instock_data:
-            fetched_md5 = hashlib.md5(fetcher.instock_data.encode('utf-8')).hexdigest()
+            fetched_md5 = hashlib.md5(self.fetcher.instock_data.encode('utf-8')).hexdigest()
             # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
-            update_obj["in_stock"] = True if fetcher.instock_data == 'Possibly in stock' else False
+            update_obj["in_stock"] = True if self.fetcher.instock_data == 'Possibly in stock' else False
         else:
-            raise UnableToExtractRestockData(status_code=fetcher.status_code)
+            raise UnableToExtractRestockData(status_code=self.fetcher.status_code)
 
         # The main thing that all this at the moment comes down to :)
         changed_detected = False
@@ -128,4 +61,4 @@ class perform_site_check(difference_detection_processor):
         # Always record the new checksum
         update_obj["previous_md5"] = fetched_md5
 
-        return changed_detected, update_obj, fetcher.instock_data.encode('utf-8')
+        return changed_detected, update_obj, self.fetcher.instock_data.encode('utf-8')

changedetectionio/processors/text_json_diff.py

@@ -1,4 +1,4 @@
-# HTML to TEXT/JSON DIFFERENCE FETCHER
+# HTML to TEXT/JSON DIFFERENCE self.fetcher
 
 import hashlib
 import json
@@ -32,15 +32,10 @@ class PDFToHTMLToolNotFound(ValueError):
 # Some common stuff here that can be moved to a base class
 # (set_proxy_from_list)
 class perform_site_check(difference_detection_processor):
-    screenshot = None
-    xpath_data = None
 
-    def __init__(self, *args, datastore, **kwargs):
+    def run_changedetection(self, uuid, skip_when_checksum_same=True):
-        super().__init__(*args, **kwargs)
-        self.datastore = datastore
-
-    def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
         changed_detected = False
+        html_content = ""
         screenshot = False  # as bytes
         stripped_text_from_html = ""
 
@@ -49,100 +44,25 @@ class perform_site_check(difference_detection_processor):
         if not watch:
             raise Exception("Watch no longer exists.")
 
-        # Protect against file:// access
-        if re.search(r'^file', watch.get('url', ''), re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
-            raise Exception(
-                "file:// type access is denied for security reasons."
-            )
-
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
 
-        # Tweak the base config with the per-watch ones
-        request_headers = watch.get('headers', [])
-        request_headers.update(self.datastore.get_all_base_headers())
-        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
-
-        # https://github.com/psf/requests/issues/4525
-        # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
-        # do this by accident.
-        if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
-            request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
-
-        timeout = self.datastore.data['settings']['requests'].get('timeout')
-
         url = watch.link
 
-        request_body = self.datastore.data['watching'][uuid].get('body')
+        self.screenshot = self.fetcher.screenshot
-        request_method = self.datastore.data['watching'][uuid].get('method')
+        self.xpath_data = self.fetcher.xpath_data
-        ignore_status_codes = self.datastore.data['watching'][uuid].get('ignore_status_codes', False)
-
-        # source: support
-        is_source = False
-        if url.startswith('source:'):
-            url = url.replace('source:', '')
-            is_source = True
-
-        # Pluggable content fetcher
-        prefer_backend = watch.get_fetch_backend
-        if not prefer_backend or prefer_backend == 'system':
-            prefer_backend = self.datastore.data['settings']['application']['fetch_backend']
-
-        if hasattr(content_fetcher, prefer_backend):
-            klass = getattr(content_fetcher, prefer_backend)
-        else:
-            # If the klass doesnt exist, just use a default
-            klass = getattr(content_fetcher, "html_requests")
-
-        if preferred_proxy:
-            proxy_id = preferred_proxy
-        else:
-            proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
-
-        proxy_url = None
-        if proxy_id:
-            proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')
-            print("UUID {} Using proxy {}".format(uuid, proxy_url))
-
-        fetcher = klass(proxy_override=proxy_url)
-
-        # Configurable per-watch or global extra delay before extracting text (for webDriver types)
-        system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)
-        if watch['webdriver_delay'] is not None:
-            fetcher.render_extract_delay = watch.get('webdriver_delay')
-        elif system_webdriver_delay is not None:
-            fetcher.render_extract_delay = system_webdriver_delay
-
-        # Possible conflict
-        if prefer_backend == 'html_webdriver':
-            fetcher.browser_steps = watch.get('browser_steps', None)
-            fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, uuid)
-
-        if watch.get('webdriver_js_execute_code') is not None and watch.get('webdriver_js_execute_code').strip():
-            fetcher.webdriver_js_execute_code = watch.get('webdriver_js_execute_code')
-
-        # requests for PDF's, images etc should be passwd the is_binary flag
-        is_binary = watch.is_pdf
-
-        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'),
-                    is_binary=is_binary)
-        fetcher.quit()
-
-        self.screenshot = fetcher.screenshot
-        self.xpath_data = fetcher.xpath_data
 
         # Track the content type
-        update_obj['content_type'] = fetcher.get_all_headers().get('content-type', '').lower()
+        update_obj['content_type'] = self.fetcher.get_all_headers().get('content-type', '').lower()
 
         # Watches added automatically in the queue manager will skip if its the same checksum as the previous run
         # Saves a lot of CPU
-        update_obj['previous_md5_before_filters'] = hashlib.md5(fetcher.content.encode('utf-8')).hexdigest()
+        update_obj['previous_md5_before_filters'] = hashlib.md5(self.fetcher.content.encode('utf-8')).hexdigest()
         if skip_when_checksum_same:
             if update_obj['previous_md5_before_filters'] == watch.get('previous_md5_before_filters'):
                 raise content_fetcher.checksumFromPreviousCheckWasTheSame()
 
         # Fetching complete, now filters
-        # @todo move to class / maybe inside of fetcher abstract base?
 
         # @note: I feel like the following should be in a more obvious chain system
         #  - Check filter text
@@ -151,24 +71,24 @@ class perform_site_check(difference_detection_processor):
         # https://stackoverflow.com/questions/41817578/basic-method-chaining ?
         # return content().textfilter().jsonextract().checksumcompare() ?
 
-        is_json = 'application/json' in fetcher.get_all_headers().get('content-type', '').lower()
+        is_json = 'application/json' in self.fetcher.get_all_headers().get('content-type', '').lower()
         is_html = not is_json
         is_rss = False
 
-        ctype_header = fetcher.get_all_headers().get('content-type', '').lower()
+        ctype_header = self.fetcher.get_all_headers().get('content-type', '').lower()
         # Go into RSS preprocess for converting CDATA/comment to usable text
         if any(substring in ctype_header for substring in ['application/xml', 'application/rss', 'text/xml']):
-            if '<rss' in fetcher.content[:100].lower():
+            if '<rss' in self.fetcher.content[:100].lower():
-                fetcher.content = cdata_in_document_to_text(html_content=fetcher.content)
+                self.fetcher.content = cdata_in_document_to_text(html_content=self.fetcher.content)
                 is_rss = True
 
         # source: support, basically treat it as plaintext
-        if is_source:
+        if watch.is_source_type_url:
             is_html = False
             is_json = False
 
-        inline_pdf = fetcher.get_all_headers().get('content-disposition', '') and '%PDF-1' in fetcher.content[:10]
+        inline_pdf = self.fetcher.get_all_headers().get('content-disposition', '') and '%PDF-1' in self.fetcher.content[:10]
-        if watch.is_pdf or 'application/pdf' in fetcher.get_all_headers().get('content-type', '').lower() or inline_pdf:
+        if watch.is_pdf or 'application/pdf' in self.fetcher.get_all_headers().get('content-type', '').lower() or inline_pdf:
             from shutil import which
             tool = os.getenv("PDF_TO_HTML_TOOL", "pdftohtml")
             if not which(tool):
@@ -179,18 +99,18 @@ class perform_site_check(difference_detection_processor):
                 [tool, '-stdout', '-', '-s', 'out.pdf', '-i'],
                 stdout=subprocess.PIPE,
                 stdin=subprocess.PIPE)
-            proc.stdin.write(fetcher.raw_content)
+            proc.stdin.write(self.fetcher.raw_content)
             proc.stdin.close()
-            fetcher.content = proc.stdout.read().decode('utf-8')
+            self.fetcher.content = proc.stdout.read().decode('utf-8')
             proc.wait(timeout=60)
 
             # Add a little metadata so we know if the file changes (like if an image changes, but the text is the same
             # @todo may cause problems with non-UTF8?
             metadata = "<p>Added by changedetection.io: Document checksum - {} Filesize - {} bytes</p>".format(
-                hashlib.md5(fetcher.raw_content).hexdigest().upper(),
+                hashlib.md5(self.fetcher.raw_content).hexdigest().upper(),
-                len(fetcher.content))
+                len(self.fetcher.content))
 
-            fetcher.content = fetcher.content.replace('</body>', metadata + '</body>')
+            self.fetcher.content = self.fetcher.content.replace('</body>', metadata + '</body>')
 
         # Better would be if Watch.model could access the global data also
         # and then use getattr https://docs.python.org/3/reference/datamodel.html#object.__getitem__
@@ -217,7 +137,7 @@ class perform_site_check(difference_detection_processor):
         if is_json:
             # Sort the JSON so we dont get false alerts when the content is just re-ordered
             try:
-                fetcher.content = json.dumps(json.loads(fetcher.content), sort_keys=True)
+                self.fetcher.content = json.dumps(json.loads(self.fetcher.content), sort_keys=True)
             except Exception as e:
                 # Might have just been a snippet, or otherwise bad JSON, continue
                 pass
@@ -225,22 +145,22 @@ class perform_site_check(difference_detection_processor):
         if has_filter_rule:
             for filter in include_filters_rule:
                 if any(prefix in filter for prefix in json_filter_prefixes):
-                    stripped_text_from_html += html_tools.extract_json_as_string(content=fetcher.content, json_filter=filter)
+                    stripped_text_from_html += html_tools.extract_json_as_string(content=self.fetcher.content, json_filter=filter)
                     is_html = False
 
-        if is_html or is_source:
+        if is_html or watch.is_source_type_url:
 
             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
-            fetcher.content = html_tools.workarounds_for_obfuscations(fetcher.content)
+            self.fetcher.content = html_tools.workarounds_for_obfuscations(self.fetcher.content)
-            html_content = fetcher.content
+            html_content = self.fetcher.content
 
             # If not JSON,  and if it's not text/plain..
-            if 'text/plain' in fetcher.get_all_headers().get('content-type', '').lower():
+            if 'text/plain' in self.fetcher.get_all_headers().get('content-type', '').lower():
                 # Don't run get_text or xpath/css filters on plaintext
                 stripped_text_from_html = html_content
             else:
                 # Does it have some ld+json price data? used for easier monitoring
-                update_obj['has_ldjson_price_data'] = html_tools.has_ldjson_product_info(fetcher.content)
+                update_obj['has_ldjson_price_data'] = html_tools.has_ldjson_product_info(self.fetcher.content)
 
                 # Then we assume HTML
                 if has_filter_rule:
@@ -250,14 +170,19 @@ class perform_site_check(difference_detection_processor):
                         # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
                         if filter_rule[0] == '/' or filter_rule.startswith('xpath:'):
                             html_content += html_tools.xpath_filter(xpath_filter=filter_rule.replace('xpath:', ''),
-                                                                    html_content=fetcher.content,
+                                                                    html_content=self.fetcher.content,
-                                                                    append_pretty_line_formatting=not is_source,
+                                                                    append_pretty_line_formatting=not watch.is_source_type_url,
+                                                                    is_rss=is_rss)
+                        elif filter_rule.startswith('xpath1:'):
+                            html_content += html_tools.xpath1_filter(xpath_filter=filter_rule.replace('xpath1:', ''),
+                                                                    html_content=self.fetcher.content,
+                                                                    append_pretty_line_formatting=not watch.is_source_type_url,
                                                                     is_rss=is_rss)
                         else:
                             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
                             html_content += html_tools.include_filters(include_filters=filter_rule,
-                                                                       html_content=fetcher.content,
+                                                                       html_content=self.fetcher.content,
-                                                                       append_pretty_line_formatting=not is_source)
+                                                                       append_pretty_line_formatting=not watch.is_source_type_url)
 
                     if not html_content.strip():
                         raise FilterNotFoundInResponse(include_filters_rule)
@@ -265,7 +190,7 @@ class perform_site_check(difference_detection_processor):
                 if has_subtractive_selectors:
                     html_content = html_tools.element_removal(subtractive_selectors, html_content)
 
-                if is_source:
+                if watch.is_source_type_url:
                     stripped_text_from_html = html_content
                 else:
                     # extract text
@@ -311,7 +236,7 @@ class perform_site_check(difference_detection_processor):
         empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
         if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
             raise content_fetcher.ReplyWithContentButNoText(url=url,
-                                                            status_code=fetcher.get_last_status_code(),
+                                                            status_code=self.fetcher.get_last_status_code(),
                                                             screenshot=screenshot,
                                                             has_filters=has_filter_rule,
                                                             html_content=html_content
@@ -320,7 +245,7 @@ class perform_site_check(difference_detection_processor):
         # We rely on the actual text in the html output.. many sites have random script vars etc,
         # in the future we'll implement other mechanisms.
 
-        update_obj["last_check_status"] = fetcher.get_last_status_code()
+        update_obj["last_check_status"] = self.fetcher.get_last_status_code()
 
         # If there's text to skip
         # @todo we could abstract out the get_text() to handle this cleaner
@@ -408,7 +333,7 @@ class perform_site_check(difference_detection_processor):
         if is_html:
             if self.datastore.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
                 if not watch['title'] or not len(watch['title']):
-                    update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
+                    update_obj['title'] = html_tools.extract_element(find='title', html_content=self.fetcher.content)
 
         if changed_detected:
             if watch.get('check_unique_lines', False):

changedetectionio/res/stock-not-in-stock.js

@@ -1,6 +1,7 @@
 function isItemInStock() {
   // @todo Pass these in so the same list can be used in non-JS fetchers
   const outOfStockTexts = [
+    ' أخبرني عندما يتوفر',
     '0 in stock',
     'agotado',
     'artikel zurzeit vergriffen',
@@ -16,9 +17,12 @@ function isItemInStock() {
     'currently have any tickets for this',
     'currently unavailable',
     'dostępne wkrótce',
+    'dostępne wkrótce',
     'en rupture de stock',
     'ist derzeit nicht auf lager',
+    'ist derzeit nicht auf lager',
     'item is no longer available',
+    'let me know when it\'s available',
     'message if back in stock',
     'nachricht bei',
     'nicht auf lager',
@@ -42,7 +46,9 @@ function isItemInStock() {
     'unavailable tickets',
     'we do not currently have an estimate of when this product will be back in stock.',
     'zur zeit nicht an lager',
+    '品切れ',
     '已售完',
+    '품절'
   ];
 
 

changedetectionio/res/xpath_element_scraper.js

@@ -170,9 +170,12 @@ if (include_filters.length) {
 
         try {
             // is it xpath?
-            if (f.startsWith('/') || f.startsWith('xpath:')) {
+            if (f.startsWith('/') || f.startsWith('xpath')) {
-                q = document.evaluate(f.replace('xpath:', ''), document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+                var qry_f = f.replace(/xpath(:|\d:)/, '')
+                console.log("[xpath] Scanning for included filter " + qry_f)
+                q = document.evaluate(qry_f, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
             } else {
+                console.log("[css] Scanning for included filter " + f)
                 q = document.querySelector(f);
             }
         } catch (e) {
@@ -182,8 +185,18 @@ if (include_filters.length) {
         }
 
         if (q) {
+            // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
+            try {
+                if (typeof q.nodeName == 'string' && q.nodeName === '#text') {
+                    q = q.parentElement
+                }
+            } catch (e) {
+                console.log(e)
+                console.log("xpath_element_scraper: #text resolver")
+            }
+
             // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
-            if (q.hasOwnProperty('getBoundingClientRect')) {
+            if (typeof q.getBoundingClientRect == 'function') {
                 bbox = q.getBoundingClientRect();
                 console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
             } else {
@@ -192,7 +205,8 @@ if (include_filters.length) {
                     bbox = q.ownerElement.getBoundingClientRect();
                     console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
                 } catch (e) {
-                    console.log("xpath_element_scraper: error looking up ownerElement")
+                    console.log(e)
+                    console.log("xpath_element_scraper: error looking up q.ownerElement")
                 }
             }
         }

changedetectionio/run_custom_browser_url_tests.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# run some tests and look if the 'custom-browser-search-string=1' connect string appeared in the correct containers
+
+# enable debug
+set -x
+
+# A extra browser is configured, but we never chose to use it, so it should NOT show in the logs
+docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_not_via_custom_browser_url'
+docker logs browserless-custom-url &>log.txt
+grep 'custom-browser-search-string=1' log.txt
+if [ $? -ne 1 ]
+then
+  echo "Saw a request in 'browserless-custom-url' container with 'custom-browser-search-string=1' when I should not"
+  exit 1
+fi
+
+docker logs browserless &>log.txt
+grep 'custom-browser-search-string=1' log.txt
+if [ $? -ne 1 ]
+then
+  echo "Saw a request in 'browser' container with 'custom-browser-search-string=1' when I should not"
+  exit 1
+fi
+
+# Special connect string should appear in the custom-url container, but not in the 'default' one
+docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/custom_browser_url/test_custom_browser_url.py::test_request_via_custom_browser_url'
+docker logs browserless-custom-url &>log.txt
+grep 'custom-browser-search-string=1' log.txt
+if [ $? -ne 0 ]
+then
+  echo "Did not see request in 'browserless-custom-url' container with 'custom-browser-search-string=1' when I should"
+  exit 1
+fi
+
+docker logs browserless &>log.txt
+grep 'custom-browser-search-string=1' log.txt
+if [ $? -ne 1 ]
+then
+  echo "Saw a request in 'browser' container with 'custom-browser-search-string=1' when I should not"
+  exit 1
+fi
+
+

changedetectionio/static/images/bell-off.svg

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg width="15" height="16.363636" viewBox="0 0 15 16.363636" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg">
+<svg width="15" height="16.363636" viewBox="0 0 15 16.363636" xmlns="http://www.w3.org/2000/svg" >
   <path d="m 14.318182,11.762045 v 1.1925 H 5.4102273 L 11.849318,7.1140909 C 12.234545,9.1561364 12.54,11.181818 14.318182,11.762045 Z m -6.7984093,4.601591 c 1.0759091,0 2.0256823,-0.955909 2.0256823,-2.045454 H 5.4545455 c 0,1.089545 0.9879545,2.045454 2.0652272,2.045454 z M 15,2.8622727 0.9177273,15.636136 0,14.627045 l 1.8443182,-1.6725 h -1.1625 v -1.1925 C 4.0070455,10.677273 2.1784091,4.5388636 5.3611364,2.6897727 5.8009091,2.4347727 6.0709091,1.9609091 6.0702273,1.4488636 v -0.00205 C 6.0702273,0.64772727 6.7104545,0 7.5,0 8.2895455,0 8.9297727,0.64772727 8.9297727,1.4468182 v 0.00205 C 8.9290909,1.9602319 9.199773,2.4354591 9.638864,2.6897773 10.364318,3.111141 10.827273,3.7568228 11.1525,4.5129591 L 14.085682,1.8531818 Z M 6.8181818,1.3636364 C 6.8181818,1.74 7.1236364,2.0454545 7.5,2.0454545 7.8763636,2.0454545 8.1818182,1.74 8.1818182,1.3636364 8.1818182,0.98795455 7.8763636,0.68181818 7.5,0.68181818 c -0.3763636,0 -0.6818182,0.30613637 -0.6818182,0.68181822 z" id="path2" style="fill:#f8321b;stroke-width:0.681818;fill-opacity:1"/>
 </svg>

changedetectionio/static/images/brightdata.svg

@@ -10,7 +10,7 @@
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg">
+   >
   <defs
      id="defs16" />
   <sodipodi:namedview

changedetectionio/static/images/copy.svg

@@ -12,7 +12,7 @@
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg"><defs
+   ><defs
      id="defs11" /><sodipodi:namedview
      id="namedview9"
      pagecolor="#ffffff"

changedetectionio/static/images/email.svg

@@ -10,7 +10,7 @@
    viewBox="0 0 7.1975545 4.7993639"
    xml:space="preserve"
    xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg"><defs
+   ><defs
    id="defs19" />
 <g
    id="g14"

changedetectionio/static/images/notice.svg

@@ -9,7 +9,7 @@
    id="svg5"
    xmlns:xlink="http://www.w3.org/1999/xlink"
    xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg">
+   >
   <defs
      id="defs2" />
   <g

changedetectionio/static/images/oxylabs.svg

@@ -10,7 +10,7 @@
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg">
+   >
   <defs
      id="defs12" />
   <sodipodi:namedview

changedetectionio/static/images/pause.svg

@@ -3,7 +3,6 @@
    xmlns:dc="http://purl.org/dc/elements/1.1/"
    xmlns:cc="http://creativecommons.org/ns#"
    xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
    xmlns="http://www.w3.org/2000/svg"
    version="1.1"
    id="Capa_1"

changedetectionio/static/images/play.svg

@@ -13,7 +13,6 @@
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg"
    xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
    xmlns:cc="http://creativecommons.org/ns#"
    xmlns:dc="http://purl.org/dc/elements/1.1/"><sodipodi:namedview

changedetectionio/static/images/spread-white.svg

@@ -6,7 +6,7 @@
    version="1.1"
    id="svg6"
    xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg">
+   >
   <defs
      id="defs10" />
   <path

changedetectionio/static/images/spread.svg

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg width="18" height="19.92" viewBox="0 0 18 19.92" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg">
+<svg width="18" height="19.92" viewBox="0 0 18 19.92" xmlns="http://www.w3.org/2000/svg" >
   <path d="M -3,-2 H 21 V 22 H -3 Z" fill="none" id="path2"/>
   <path d="m 15,14.08 c -0.76,0 -1.44,0.3 -1.96,0.77 L 5.91,10.7 C 5.96,10.47 6,10.24 6,10 6,9.76 5.96,9.53 5.91,9.3 L 12.96,5.19 C 13.5,5.69 14.21,6 15,6 16.66,6 18,4.66 18,3 18,1.34 16.66,0 15,0 c -1.66,0 -3,1.34 -3,3 0,0.24 0.04,0.47 0.09,0.7 L 5.04,7.81 C 4.5,7.31 3.79,7 3,7 1.34,7 0,8.34 0,10 c 0,1.66 1.34,3 3,3 0.79,0 1.5,-0.31 2.04,-0.81 l 7.12,4.16 c -0.05,0.21 -0.08,0.43 -0.08,0.65 0,1.61 1.31,2.92 2.92,2.92 1.61,0 2.92,-1.31 2.92,-2.92 0,-1.61 -1.31,-2.92 -2.92,-2.92 z" id="path4" style="fill:#0078e7;fill-opacity:1"/>
 </svg>

changedetectionio/static/js/global-settings.js

@@ -1,19 +1,4 @@
 $(document).ready(function () {
-    function toggle() {
-        if ($('input[name="application-fetch_backend"]:checked').val() != 'html_requests') {
-            $('#requests-override-options').hide();
-            $('#webdriver-override-options').show();
-        } else {
-            $('#requests-override-options').show();
-            $('#webdriver-override-options').hide();
-        }
-    }
-
-    $('input[name="application-fetch_backend"]').click(function (e) {
-        toggle();
-    });
-    toggle();
-
     $("#api-key").hover(
         function () {
             $("#api-key-copy").html('copy').fadeIn();

changedetectionio/static/js/vis.js

@@ -0,0 +1,29 @@
+$(document).ready(function () {
+
+    // Lazy Hide/Show elements mechanism
+    $('[data-visible-for]').hide();
+    function show_related_elem(e) {
+        var n = $(e).attr('name') + "=" + $(e).val();
+        if (n === 'fetch_backend=system') {
+            n = "fetch_backend=" + default_system_fetch_backend;
+        }
+        $(`[data-visible-for~="${n}"]`).show();
+    }
+    $(':radio').on('keyup keypress blur change click', function (e) {
+        $(`[data-visible-for]`).hide();
+        $('.advanced-options').hide();
+        show_related_elem(this);
+    });
+
+    $(':radio:checked').each(function (e) {
+       show_related_elem(this);
+    })
+
+
+    // Show advanced
+    $('.show-advanced').click(function (e) {
+        $(this).closest('.tab-pane-inner').find('.advanced-options').each(function (e) {
+            $(this).toggle();
+        })
+    });
+});

changedetectionio/static/js/visual-selector.js

@@ -149,7 +149,7 @@ $(document).ready(function () {
             // @todo In the future paint all that match
             for (const c of current_default_xpath) {
                 for (var i = selector_data['size_pos'].length; i !== 0; i--) {
-                    if (selector_data['size_pos'][i - 1].xpath === c) {
+                    if (selector_data['size_pos'][i - 1].xpath.trim() === c.trim()) {
                         console.log("highlighting " + c);
                         current_selected_i = i - 1;
                         highlight_current_selected_i();

changedetectionio/static/js/watch-settings.js

@@ -1,40 +1,4 @@
 $(document).ready(function () {
-    function toggle() {
-        if ($('input[name="fetch_backend"]:checked').val() == 'html_webdriver') {
-            if (playwright_enabled) {
-                // playwright supports headers, so hide everything else
-                // See #664
-                $('#requests-override-options #request-method').hide();
-                $('#requests-override-options #request-body').hide();
-
-                // @todo connect this one up
-                $('#ignore-status-codes-option').hide();
-            } else {
-                // selenium/webdriver doesnt support anything afaik, hide it all
-                $('#requests-override-options').hide();
-            }
-
-            $('#webdriver-override-options').show();
-
-        } else if ($('input[name="fetch_backend"]:checked').val() == 'system') {
-            $('#requests-override-options #request-method').hide();
-            $('#requests-override-options #request-body').hide();
-            $('#ignore-status-codes-option').hide();
-            $('#requests-override-options').hide();
-            $('#webdriver-override-options').hide();
-        } else {
-
-            $('#requests-override-options').show();
-            $('#requests-override-options *:hidden').show();
-            $('#webdriver-override-options').hide();
-        }
-    }
-
-    $('input[name="fetch_backend"]').click(function (e) {
-        toggle();
-    });
-    toggle();
-
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
         $('#notification_body').val('');

changedetectionio/static/styles/scss/parts/_extra_browsers.scss

@@ -0,0 +1,24 @@
+ul#requests-extra_browsers {
+  list-style: none;
+  /* tidy up the table to look more "inline" */
+  li {
+    > label {
+      display: none;
+    }
+
+  }
+
+  /* each proxy entry is a `table` */
+  table {
+    tr {
+      display: inline;
+    }
+  }
+}
+
+#extra-browsers-setting {
+  border: 1px solid var(--color-grey-800);
+  border-radius: 4px;
+  margin: 1em;
+   padding: 1em;
+}

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -60,3 +60,10 @@ body.proxy-check-active {
 
   padding-bottom: 1em;
 }
+
+#extra-proxies-setting {
+  border: 1px solid var(--color-grey-800);
+  border-radius: 4px;
+    margin: 1em;
+   padding: 1em;
+}

changedetectionio/static/styles/scss/styles.scss

@@ -5,6 +5,7 @@
 @import "parts/_arrows";
 @import "parts/_browser-steps";
 @import "parts/_extra_proxies";
+@import "parts/_extra_browsers";
 @import "parts/_pagination";
 @import "parts/_spinners";
 @import "parts/_variables";
@@ -401,8 +402,24 @@ label {
   }
 
   #watch-add-wrapper-zone {
-    >div {
+
-      display: inline-block;
+    @media only screen and (min-width: 760px) {
+      display: flex;
+      gap: 0.3rem;
+      flex-direction: row;
+    }
+    /* URL field grows always, other stay static in width */
+    > span {
+      flex-grow: 0;
+
+      input {
+        width: 100%;
+        padding-right: 1em;
+      }
+
+      &:first-child {
+        flex-grow: 1;
+      }
     }
 
     @media only screen and (max-width: 760px) {
@@ -943,10 +960,8 @@ ul {
 
 @import "parts/_visualselector";
 
-#webdriver-override-options {
+#webdriver_delay {
-  input[type="number"] {
     width: 5em;
-  }
 }
 
 #api-key {

changedetectionio/static/styles/styles.css

@@ -128,6 +128,27 @@ body.proxy-check-active #request .proxy-timing {
     border-radius: 4px;
     padding: 1em; }
 
+#extra-proxies-setting {
+  border: 1px solid var(--color-grey-800);
+  border-radius: 4px;
+  margin: 1em;
+  padding: 1em; }
+
+ul#requests-extra_browsers {
+  list-style: none;
+  /* tidy up the table to look more "inline" */
+  /* each proxy entry is a `table` */ }
+  ul#requests-extra_browsers li > label {
+    display: none; }
+  ul#requests-extra_browsers table tr {
+    display: inline; }
+
+#extra-browsers-setting {
+  border: 1px solid var(--color-grey-800);
+  border-radius: 4px;
+  margin: 1em;
+  padding: 1em; }
+
 .pagination-page-info {
   color: #fff;
   font-size: 0.85rem;
@@ -662,11 +683,23 @@ label:hover {
   #new-watch-form legend {
     color: var(--color-text-legend);
     font-weight: bold; }
-  #new-watch-form #watch-add-wrapper-zone > div {
+  #new-watch-form #watch-add-wrapper-zone {
-    display: inline-block; }
+    /* URL field grows always, other stay static in width */ }
-  @media only screen and (max-width: 760px) {
+    @media only screen and (min-width: 760px) {
-    #new-watch-form #watch-add-wrapper-zone #url {
+      #new-watch-form #watch-add-wrapper-zone {
-      width: 100%; } }
+        display: flex;
+        gap: 0.3rem;
+        flex-direction: row; } }
+    #new-watch-form #watch-add-wrapper-zone > span {
+      flex-grow: 0; }
+      #new-watch-form #watch-add-wrapper-zone > span input {
+        width: 100%;
+        padding-right: 1em; }
+      #new-watch-form #watch-add-wrapper-zone > span:first-child {
+        flex-grow: 1; }
+    @media only screen and (max-width: 760px) {
+      #new-watch-form #watch-add-wrapper-zone #url {
+        width: 100%; } }
 
 #diff-col {
   padding-left: 40px; }
@@ -1044,7 +1077,7 @@ ul {
 #selector-current-xpath {
   font-size: 80%; }
 
-#webdriver-override-options input[type="number"] {
+#webdriver_delay {
   width: 5em; }
 
 #api-key:hover {

changedetectionio/store.py

@@ -633,6 +633,18 @@ class ChangeDetectionStore:
 
         return {}
 
+    @property
+    def extra_browsers(self):
+        res = []
+        p = list(filter(
+            lambda s: (s.get('browser_name') and s.get('browser_connection_url')),
+            self.__data['settings']['requests'].get('extra_browsers', [])))
+        if p:
+            for i in p:
+                res.append(("extra_browser_"+i['browser_name'], i['browser_name']))
+
+        return res
+
     def tag_exists_by_name(self, tag_name):
         return any(v.get('title', '').lower() == tag_name.lower() for k, v in self.__data['settings']['application']['tags'].items())
 
@@ -835,4 +847,14 @@ class ChangeDetectionStore:
             if not watch.get('date_created'):
                 self.data['watching'][uuid]['date_created'] = i
             i+=1
-        return
+        return
+
+    # #1774 - protect xpath1 against migration
+    def update_14(self):
+        for awatch in self.__data["watching"]:
+            if self.__data["watching"][awatch]['include_filters']:
+                for num, selector in enumerate(self.__data["watching"][awatch]['include_filters']):
+                    if selector.startswith('/'):
+                        self.__data["watching"][awatch]['include_filters'][num] = 'xpath1:' + selector
+                    if selector.startswith('xpath:'):
+                        self.__data["watching"][awatch]['include_filters'][num] = selector.replace('xpath:', 'xpath1:', 1)

changedetectionio/templates/_common_fields.jinja

@@ -13,10 +13,10 @@
                             <div class="pure-form-message-inline">
                               <ul>
                                 <li>Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) </code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
-                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
+                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">more help here</a></li>
                                   <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
                             </div>

changedetectionio/templates/_helpers.jinja

@@ -39,6 +39,24 @@
 {% endmacro %}
 
 
+{% macro render_nolabel_field(field) %}
+    <span>
+    {{ field(**kwargs)|safe }}
+        {% if field.errors %}
+            <span class="error">
+      {% if field.errors %}
+          <ul class=errors>
+        {% for error in field.errors %}
+            <li>{{ error }}</li>
+        {% endfor %}
+        </ul>
+      {% endif %}
+      </span>
+        {% endif %}
+    </span>
+{% endmacro %}
+
+
 {% macro render_button(field) %}
   {{ field(**kwargs)|safe }}
 {% endmacro %}

changedetectionio/templates/base.html

@@ -116,7 +116,7 @@
                    viewBox="0 0 16.9 16.1"
                    id="svg-heart"
                    xmlns="http://www.w3.org/2000/svg"
-                   xmlns:svg="http://www.w3.org/2000/svg">
+                   >
                   <path id="heartpath" d="M 5.338316,0.50302766 C 0.71136983,0.50647126 -3.9576371,7.2707777 8.5004254,15.503028 23.833425,5.3700277 13.220206,-2.5384409 8.6762066,1.6475589 c -0.060791,0.054322 -0.11943,0.1110064 -0.1757812,0.1699219 -0.057,-0.059 -0.1157813,-0.116875 -0.1757812,-0.171875 C 7.4724566,0.86129334 6.4060729,0.50223298 5.338316,0.50302766 Z"
                      style="fill:var(--color-background);fill-opacity:1;stroke:#ff0000;stroke-opacity:1" />
                 </svg>
@@ -170,7 +170,6 @@
                     And tell your friends and colleagues :)
                 </li>
                 </ul>
-                </p>
                 <p>
                     The more popular changedetection.io is, the more time we can dedicate to adding amazing features!
                 </p>

changedetectionio/templates/edit.html

@@ -3,6 +3,7 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script>
     const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
@@ -19,7 +20,7 @@
     const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
     const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
     const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
-
+    const default_system_fetch_backend="{{ settings_application['fetch_backend'] }}";
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
@@ -124,10 +125,9 @@
                         </span>
                     </div>
                 {% endif %}
-                    <div  class="pure-control-group inline-radio">
+
-                        {{ render_checkbox_field(form.ignore_status_codes) }}
+                <!-- webdriver always -->
-                    </div>
+                <fieldset data-visible-for="fetch_backend=html_webdriver"  style="display: none;">
-                <fieldset id="webdriver-override-options">
                     <div class="pure-control-group">
                         {{ render_field(form.webdriver_delay) }}
                         <div class="pure-form-message-inline">
@@ -140,23 +140,40 @@
                         </div>
                     </div>
                     <div class="pure-control-group">
+                        <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
+                    </div>
+                    <div class="advanced-options"  style="display: none;">
                         {{ render_field(form.webdriver_js_execute_code) }}
                         <div class="pure-form-message-inline">
-                            Run this code before performing change detection, handy for filling in fields and other actions <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Run-JavaScript-before-change-detection">More help and examples here</a>
+                            Run this code before performing change detection, handy for filling in fields and other
+                            actions <a
+                                href="https://github.com/dgtlmoon/changedetection.io/wiki/Run-JavaScript-before-change-detection">More
+                            help and examples here</a>
                         </div>
                     </div>
                 </fieldset>
-                <fieldset class="pure-group" id="requests-override-options">
+                <!-- html requests always -->
-                    {% if not playwright_enabled %}
+                <fieldset data-visible-for="fetch_backend=html_requests">
-                        <div class="pure-form-message-inline">
+                    <div class="pure-control-group">
-                            <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
+                        <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
-                        </div>
-                    {% endif %}
-                    <div class="pure-control-group" id="request-method">
-                        {{ render_field(form.method) }}
                     </div>
-                    <div class="pure-control-group" id="request-headers">
+                    <div class="advanced-options"  style="display: none;">
-{{ render_field(form.headers, rows=5, placeholder="Example
+                        <div class="pure-control-group" id="request-method">
+                            {{ render_field(form.method) }}
+                        </div>
+                        <div id="request-body">
+                                            {{ render_field(form.body, rows=5, placeholder="Example
+{
+   \"name\":\"John\",
+   \"age\":30,
+   \"car\":null
+}") }}
+                        </div>
+                    </div>
+                </fieldset>
+            <!-- hmm -->
+                <div class="pure-control-group advanced-options"  style="display: none;">
+                    {{ render_field(form.headers, rows=5, placeholder="Example
 Cookie: foobar
 User-Agent: wonderbra 1.0") }}
 
@@ -169,17 +186,12 @@ User-Agent: wonderbra 1.0") }}
                             <br>
                             (Not supported by Selenium browser)
                         </div>
-
                     </div>
-                    <div class="pure-control-group" id="request-body">
+            <fieldset data-visible-for="fetch_backend=html_requests fetch_backend=html_webdriver" >
-                                        {{ render_field(form.body, rows=5, placeholder="Example
+                    <div class="pure-control-group inline-radio advanced-options"  style="display: none;">
-{
+                    {{ render_checkbox_field(form.ignore_status_codes) }}
-   \"name\":\"John\",
-   \"age\":30,
-   \"car\":null
-}") }}
                     </div>
-                </fieldset>
+            </fieldset>
             </div>
             {% if playwright_enabled %}
             <div class="tab-pane-inner" id="browser-steps">
@@ -290,11 +302,12 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                                 {% endif %}
                             </ul>
                         </li>
-                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
+                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash. To specify XPath to be used explicitly or the XPath rule starts with an XPath function: Prefix with <code>xpath:</code>
                             <ul>
-                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
+                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath:count(//*[contains(@class, 'sametext')])</code>, <a
                                 href="http://xpather.com/" target="new">test your XPath here</a></li>
                                 <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
+                                <li>To use XPath1.0: Prefix with <code>xpath1:</code></li>
                             </ul>
                             </li>
                     </ul>

changedetectionio/templates/settings.html

@@ -11,7 +11,7 @@
 </script>
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
-
+<script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
 <div class="edit-form">
     <div class="tabs collapsable">
@@ -111,7 +111,7 @@
                     <br>
                     Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
                 </div>
-                <fieldset class="pure-group" id="webdriver-override-options">
+                <fieldset class="pure-group" id="webdriver-override-options" data-visible-for="application-fetch_backend=html_webdriver">
                     <div class="pure-form-message-inline">
                         <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
                         <br>
@@ -178,6 +178,9 @@ nav
                         <span style="display:none;" id="api-key-copy" >copy</span>
                     </div>
                 </div>
+                <div class="pure-control-group">
+                    <a href="{{url_for('settings_reset_api_key')}}" class="pure-button button-small button-cancel">Regenerate API key</a>
+                </div>
             </div>
             <div class="tab-pane-inner" id="proxies">
                 <div id="recommended-proxy">
@@ -227,11 +230,18 @@ nav
                 </p>
                <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
 
-                <div class="pure-control-group">
+                <div class="pure-control-group" id="extra-proxies-setting">
                 {{ render_field(form.requests.form.extra_proxies) }}
                 <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span><br>
                 <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
                 </div>
+                <div class="pure-control-group" id="extra-browsers-setting">
+                    <p>
+                    <span class="pure-form-message-inline"><i>Extra Browsers</i> can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.</span><br>
+                    <span class="pure-form-message-inline">Simply paste the connection address into the box, <a href="https://changedetection.io/tutorial/using-bright-datas-scraping-browser-pass-captchas-and-other-protection-when-monitoring">More instructions and examples here</a> </span>
+                    </p>
+                    {{ render_field(form.requests.form.extra_browsers) }}
+                </div>
             </div>
             <div id="actions">
                 <div class="pure-control-group">

changedetectionio/templates/svgs/search-icon.svg

@@ -1 +1 @@
-<?xml version="1.0" encoding="utf-8"?><svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>

changedetectionio/templates/watch-overview.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.jinja' import render_simple_field, render_field %}
+{% from '_helpers.jinja' import render_simple_field, render_field, render_nolabel_field %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 
@@ -11,17 +11,14 @@
         <fieldset>
             <legend>Add a new change detection watch</legend>
             <div id="watch-add-wrapper-zone">
-                <div>
+
-                    {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
+                    {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_simple_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
+                    {{ render_nolabel_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
-                </div>
+                    {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }}
-                <div>
+                    {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
-                    {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
-                    {{ render_simple_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
-                </div>
             </div>
             <div id="quick-watch-processor-type">
-                {{ render_simple_field(form.processor, title="Edit first then Watch") }}
+                {{ render_simple_field(form.processor) }}
             </div>
 
         </fieldset>
@@ -82,12 +79,15 @@
             </tr>
             {% endif %}
             {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}
+
+                {% set is_unviewed =  watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}
+
             <tr id="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} processor-{{ watch['processor'] }}
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
                 {% if watch.last_notification_error is defined and watch.last_notification_error != False %}error{% endif %}
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
-                {% if watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}unviewed{% endif %}
+                {% if is_unviewed %}unviewed{% endif %}
                 {% if watch.uuid in queued_uuids %}queued{% endif %}">
                 <td class="inline checkbox-uuid" ><input name="uuids"  type="checkbox" value="{{ watch.uuid}} " > <span>{{ loop.index+pagination.skip }}</span></td>
                 <td class="inline watch-controls">
@@ -104,8 +104,9 @@
 
                     {% if watch.get_fetch_backend == "html_webdriver"
                          or (  watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
+                         or "extra_browser_" in watch.get_fetch_backend
                     %}
-                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a chrome browser" >
+                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a Chrome browser" >
                     {% endif %}
 
                     {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" >{% endif %}
@@ -166,7 +167,13 @@
                        class="recheck pure-button pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
                     <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button pure-button-primary">Edit</a>
                     {% if watch.history_n >= 2 %}
-                    <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">Diff</a>
+
+                        {%  if is_unviewed %}
+                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid, from_version=watch.get_next_snapshot_key_to_last_viewed) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">Diff</a>
+                        {% else %}
+                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">Diff</a>
+                        {% endif %}
+
                     {% else %}
                         {% if watch.history_n == 1 or (watch.history_n ==0 and watch.error_text_ctime )%}
                             <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary">Preview</a>

changedetectionio/tests/conftest.py

@@ -13,22 +13,17 @@ global app
 
 
 def cleanup(datastore_path):
+    import glob
     # Unlink test output files
-    files = [
+
-        'count.txt',
+    for g in ["*.txt", "*.json", "*.pdf"]:
-        'endpoint-content.txt'
+        files = glob.glob(os.path.join(datastore_path, g))
-        'headers.txt',
+        for f in files:
-        'headers-testtag.txt',
+            if 'proxies.json' in f:
-        'notification.txt',
+                # Usually mounted by docker container during test time
-        'secret.txt',
+                continue
-        'url-watches.json',
+            if os.path.isfile(f):
-        'output.txt',
+                os.unlink(f)
-    ]
-    for file in files:
-        try:
-            os.unlink("{}/{}".format(datastore_path, file))
-        except FileNotFoundError:
-            pass
 
 @pytest.fixture(scope='session')
 def app(request):

changedetectionio/tests/custom_browser_url/__init__.py

@@ -0,0 +1 @@
+# placeholder

changedetectionio/tests/custom_browser_url/test_custom_browser_url.py

@@ -0,0 +1,89 @@
+# !/usr/bin/python3
+import os
+
+from flask import url_for
+from ..util import live_server_setup, wait_for_all_checks
+
+def do_test(client, live_server, make_test_use_extra_browser=False):
+
+    # Grep for this string in the logs?
+    test_url = f"https://changedetection.io/ci-test.html"
+    custom_browser_name = 'custom browser URL'
+
+    # needs to be set and something like 'ws://127.0.0.1:3000?stealth=1&--disable-web-security=true'
+    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
+
+    #####################
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-empty_pages_are_a_change": "",
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_webdriver",
+              # browserless-custom-url is setup in  .github/workflows/test-only.yml
+              # the test script run_custom_browser_url_test.sh will look for 'custom-browser-search-string' in the container logs
+              'requests-extra_browsers-0-browser_connection_url': 'ws://browserless-custom-url:3000?stealth=1&--disable-web-security=true&custom-browser-search-string=1',
+              'requests-extra_browsers-0-browser_name': custom_browser_name
+              },
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    if make_test_use_extra_browser:
+
+        # So the name should appear in the edit page under "Request" > "Fetch Method"
+        res = client.get(
+            url_for("edit_page", uuid="first"),
+            follow_redirects=True
+        )
+        assert b'custom browser URL' in res.data
+
+        res = client.post(
+            url_for("edit_page", uuid="first"),
+            data={
+                  "url": test_url,
+                  "tags": "",
+                  "headers": "",
+                  'fetch_backend': f"extra_browser_{custom_browser_name}",
+                  'webdriver_js_execute_code': ''
+            },
+            follow_redirects=True
+        )
+
+        assert b"Updated watch." in res.data
+        wait_for_all_checks(client)
+
+    # Force recheck
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    assert b'1 watches queued for rechecking.' in res.data
+
+    wait_for_all_checks(client)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b'cool it works' in res.data
+
+
+# Requires playwright to be installed
+def test_request_via_custom_browser_url(client, live_server):
+    live_server_setup(live_server)
+    # We do this so we can grep the logs of the custom container and see if the request actually went through that container
+    do_test(client, live_server, make_test_use_extra_browser=True)
+
+
+def test_request_not_via_custom_browser_url(client, live_server):
+    live_server_setup(live_server)
+    # We do this so we can grep the logs of the custom container and see if the request actually went through that container
+    do_test(client, live_server, make_test_use_extra_browser=False)

changedetectionio/tests/test_api.py

@@ -96,7 +96,9 @@ def test_api_simple(client, live_server):
     )
     assert watch_uuid in res.json.keys()
     before_recheck_info = res.json[watch_uuid]
+
     assert before_recheck_info['last_checked'] != 0
+
     #705 `last_changed` should be zero on the first check
     assert before_recheck_info['last_changed'] == 0
     assert before_recheck_info['title'] == 'My test URL'
@@ -157,6 +159,18 @@ def test_api_simple(client, live_server):
     # @todo how to handle None/default global values?
     assert watch['history_n'] == 2, "Found replacement history section, which is in its own API"
 
+    assert watch.get('viewed') == False
+    # Loading the most recent snapshot should force viewed to become true
+    client.get(url_for("diff_history_page", uuid="first"), follow_redirects=True)
+
+    # Fetch the whole watch again, viewed should be true
+    res = client.get(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key}
+    )
+    watch = res.json
+    assert watch.get('viewed') == True
+
     # basic systeminfo check
     res = client.get(
         url_for("systeminfo"),

changedetectionio/tests/test_extract_csv.py

@@ -24,7 +24,7 @@ def test_check_extract_text_from_diff(client, live_server):
     )
 
     assert b"1 Imported" in res.data
-    time.sleep(1)
+    wait_for_all_checks(client)
 
     # Load in 5 different numbers/changes
     last_date=""

changedetectionio/tests/test_extract_regex.py

@@ -227,9 +227,6 @@ def test_regex_error_handling(client, live_server):
         follow_redirects=True
     )
 
-    with open('/tmp/fuck.html', 'wb') as f:
-        f.write(res.data)
-
     assert b'is not a valid regular expression.' in res.data
 
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_ignore_regex_text.py

@@ -33,8 +33,6 @@ def test_strip_regex_text_func():
         "/not"
     ]
 
-
-    fetcher = fetch_site_status.perform_site_check(datastore=False)
     stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
 
     assert b"but 1 lines" in stripped_content

changedetectionio/tests/test_ignore_text.py

@@ -24,7 +24,6 @@ def test_strip_text_func():
 
     ignore_lines = ["sometimes"]
 
-    fetcher = fetch_site_status.perform_site_check(datastore=False)
     stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
 
     assert b"sometimes" not in stripped_content

changedetectionio/tests/test_notification.py

@@ -281,7 +281,8 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
 
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}"
+    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
 
     res = client.post(
         url_for("settings_page"),
@@ -297,10 +298,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         follow_redirects=True
     )
     assert b'Settings updated' in res.data
-    client.get(
+
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )
     # Add a watch and trigger a HTTP POST
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -315,7 +313,9 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     set_modified_response()
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
+    wait_for_all_checks(client)
+
+    time.sleep(2) # plus extra delay for notifications to fire
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
@@ -328,6 +328,13 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     with open("test-datastore/notification-url.txt", 'r') as f:
         notification_url = f.read()
         assert 'xxx=http' in notification_url
+        # apprise style headers should be stripped
+        assert 'custom-header' not in notification_url
+
+    with open("test-datastore/notification-headers.txt", 'r') as f:
+        notification_headers = f.read()
+        assert 'custom-header: 123' in notification_headers.lower()
+
 
     # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
     assert os.path.isfile("test-datastore/notification-content-type.txt")
@@ -335,3 +342,8 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         assert 'application/json' in f.read()
 
     os.unlink("test-datastore/notification-url.txt")
+
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )

changedetectionio/tests/test_pdf.py

@@ -2,9 +2,8 @@
 
 import time
 from flask import url_for
-from .util import set_original_response, set_modified_response, live_server_setup
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 
-sleep_time_for_fetch_thread = 3
 
 # `subtractive_selectors` should still work in `source:` type requests
 def test_fetch_pdf(client, live_server):
@@ -22,7 +21,9 @@ def test_fetch_pdf(client, live_server):
 
     assert b"1 Imported" in res.data
 
-    time.sleep(sleep_time_for_fetch_thread)
+
+    wait_for_all_checks(client)
+
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True
@@ -33,8 +34,42 @@ def test_fetch_pdf(client, live_server):
 
     # So we know if the file changes in other ways
     import hashlib
-    md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
+    original_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
     # We should have one
-    assert len(md5) >0
+    assert len(original_md5) >0
     # And it's going to be in the document
-    assert b'Document checksum - '+bytes(str(md5).encode('utf-8')) in res.data
+    assert b'Document checksum - '+bytes(str(original_md5).encode('utf-8')) in res.data
+
+
+    shutil.copy("tests/test2.pdf", "test-datastore/endpoint-test.pdf")
+    changed_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    assert b'1 watches queued for rechecking.' in res.data
+
+    wait_for_all_checks(client)
+
+    # Now something should be ready, indicated by having a 'unviewed' class
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    # The original checksum should be not be here anymore (cdio adds it to the bottom of the text)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert original_md5.encode('utf-8') not in res.data
+    assert changed_md5.encode('utf-8') in res.data
+
+
+    res = client.get(
+        url_for("diff_history_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert original_md5.encode('utf-8') in res.data
+    assert changed_md5.encode('utf-8') in res.data
+
+    assert b'here is a change' in res.data
+    

changedetectionio/tests/test_request.py

@@ -80,8 +80,11 @@ def test_headers_in_request(client, live_server):
 
     # Should be only one with headers set
     assert watches_with_headers==1
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
 
 def test_body_in_request(client, live_server):
+
     # Add our URL to the import page
     test_url = url_for('test_body', _external=True)
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):
@@ -170,7 +173,8 @@ def test_body_in_request(client, live_server):
         follow_redirects=True
     )
     assert b"Body must be empty when Request Method is set to GET" in res.data
-
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
 
 def test_method_in_request(client, live_server):
     # Add our URL to the import page

changedetectionio/tests/test_security.py

@@ -1,5 +1,5 @@
 from flask import url_for
-from . util import set_original_response, set_modified_response, live_server_setup
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 import time
 
 
@@ -12,6 +12,7 @@ def test_bad_access(client, live_server):
     )
 
     assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
 
     # Attempt to add a body with a GET method
     res = client.post(
@@ -59,7 +60,7 @@ def test_bad_access(client, live_server):
         data={"url": 'file:///tasty/disk/drive', "tags": ''},
         follow_redirects=True
     )
-    time.sleep(1)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
 
     assert b'file:// type access is denied for security reasons.' in res.data

changedetectionio/tests/test_xpath_selector.py

@@ -6,9 +6,11 @@ from .util import live_server_setup, wait_for_all_checks
 
 from ..html_tools import *
 
+
 def test_setup(live_server):
     live_server_setup(live_server)
 
+
 def set_original_response():
     test_return_data = """<html>
        <body>
@@ -26,6 +28,7 @@ def set_original_response():
         f.write(test_return_data)
     return None
 
+
 def set_modified_response():
     test_return_data = """<html>
        <body>
@@ -44,11 +47,12 @@ def set_modified_response():
 
     return None
 
+
 # Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613
 def test_check_xpath_filter_utf8(client, live_server):
-    filter='//item/*[self::description]'
+    filter = '//item/*[self::description]'
 
-    d='''<?xml version="1.0" encoding="UTF-8"?>
+    d = '''<?xml version="1.0" encoding="UTF-8"?>
 <rss xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
 	<channel>
 		<title>rpilocator.com</title>
@@ -102,9 +106,9 @@ def test_check_xpath_filter_utf8(client, live_server):
 
 # Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613
 def test_check_xpath_text_function_utf8(client, live_server):
-    filter='//item/title/text()'
+    filter = '//item/title/text()'
 
-    d='''<?xml version="1.0" encoding="UTF-8"?>
+    d = '''<?xml version="1.0" encoding="UTF-8"?>
 <rss xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
 	<channel>
 		<title>rpilocator.com</title>
@@ -163,15 +167,12 @@ def test_check_xpath_text_function_utf8(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_check_markup_xpath_filter_restriction(client, live_server):
 
+def test_check_markup_xpath_filter_restriction(client, live_server):
     xpath_filter = "//*[contains(@class, 'sametext')]"
 
     set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -214,7 +215,6 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
 
 
 def test_xpath_validation(client, live_server):
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -235,6 +235,48 @@ def test_xpath_validation(client, live_server):
     assert b'Deleted' in res.data
 
 
+def test_xpath23_prefix_validation(client, live_server):
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"include_filters": "xpath:/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"is not a valid XPath expression" in res.data
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+
+def test_xpath1_validation(client, live_server):
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"include_filters": "xpath1:/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"is not a valid XPath expression" in res.data
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+
 # actually only really used by the distll.io importer, but could be handy too
 def test_check_with_prefix_include_filters(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
@@ -254,7 +296,8 @@ def test_check_with_prefix_include_filters(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "",
+              'fetch_backend': "html_requests"},
         follow_redirects=True
     )
 
@@ -266,13 +309,15 @@ def test_check_with_prefix_include_filters(client, live_server):
         follow_redirects=True
     )
 
-    assert b"Some text thats the same" in res.data #in selector
+    assert b"Some text thats the same" in res.data  # in selector
-    assert b"Some text that will change" not in res.data #not in selector
+    assert b"Some text that will change" not in res.data  # not in selector
 
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+
+
 def test_various_rules(client, live_server):
     # Just check these don't error
-    #live_server_setup(live_server)
+    # live_server_setup(live_server)
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("""<html>
        <body>
@@ -285,10 +330,11 @@ def test_various_rules(client, live_server):
      <a href=''>some linky </a>
      <a href=''>another some linky </a>
      <!-- related to https://github.com/dgtlmoon/changedetection.io/pull/1774 -->
-     <input   type="email"   id="email" />
+     <input   type="email"   id="email" />     
      </body>
      </html>
     """)
+
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("import_page"),
@@ -298,7 +344,6 @@ def test_various_rules(client, live_server):
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
 
-
     for r in ['//div', '//a', 'xpath://div', 'xpath://a']:
         res = client.post(
             url_for("edit_page", uuid="first"),
@@ -313,3 +358,153 @@ def test_various_rules(client, live_server):
         assert b"Updated watch." in res.data
         res = client.get(url_for("index"))
         assert b'fetch-error' not in res.data, f"Should not see errors after '{r} filter"
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+
+def test_xpath_20(client, live_server):
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    set_original_response()
+
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"include_filters": "//*[contains(@class, 'sametext')]|//*[contains(@class, 'changetext')]",
+              "url": test_url,
+              "tags": "",
+              "headers": "",
+              'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Updated watch." in res.data
+    wait_for_all_checks(client)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"Some text thats the same" in res.data  # in selector
+    assert b"Some text that will change" in res.data  # in selector
+
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+
+
+def test_xpath_20_function_count(client, live_server):
+    set_original_response()
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"include_filters": "xpath:count(//div) * 123456789987654321",
+              "url": test_url,
+              "tags": "",
+              "headers": "",
+              'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Updated watch." in res.data
+    wait_for_all_checks(client)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"246913579975308642" in res.data  # in selector
+
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+
+
+def test_xpath_20_function_count2(client, live_server):
+    set_original_response()
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"include_filters": "/html/body/count(div) * 123456789987654321",
+              "url": test_url,
+              "tags": "",
+              "headers": "",
+              'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Updated watch." in res.data
+    wait_for_all_checks(client)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"246913579975308642" in res.data  # in selector
+
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+
+
+def test_xpath_20_function_string_join_matches(client, live_server):
+    set_original_response()
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "include_filters": "xpath:string-join(//*[contains(@class, 'sametext')]|//*[matches(@class, 'changetext')], 'specialconjunction')",
+            "url": test_url,
+            "tags": "",
+            "headers": "",
+            'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    assert b"Updated watch." in res.data
+    wait_for_all_checks(client)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"Some text thats the samespecialconjunctionSome text that will change" in res.data  # in selector
+
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+

changedetectionio/tests/test_xpath_selector_unit.py

@@ -0,0 +1,203 @@
+import sys
+import os
+import pytest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import html_tools
+
+# test generation guide.
+# 1. Do not include encoding in the xml declaration if the test object is a str type.
+# 2. Always paraphrase test.
+
+hotels = """
+<hotel>
+  <branch location="California">
+    <staff>
+      <given_name>Christopher</given_name>
+      <surname>Anderson</surname>
+      <age>25</age>
+    </staff>
+    <staff>
+      <given_name>Christopher</given_name>
+      <surname>Carter</surname>
+      <age>30</age>
+    </staff>
+  </branch>
+  <branch location="Las Vegas">
+    <staff>
+      <given_name>Lisa</given_name>
+      <surname>Walker</surname>
+      <age>60</age>
+    </staff>
+    <staff>
+      <given_name>Jessica</given_name>
+      <surname>Walker</surname>
+      <age>32</age>
+    </staff>
+    <staff>
+      <given_name>Jennifer</given_name>
+      <surname>Roberts</surname>
+      <age>50</age>
+    </staff>
+  </branch>
+</hotel>"""
+
+@pytest.mark.parametrize("html_content", [hotels])
+@pytest.mark.parametrize("xpath, answer", [('(//staff/given_name, //staff/age)', '25'),
+                          ("xs:date('2023-10-10')", '2023-10-10'),
+                          ("if (/hotel/branch[@location = 'California']/staff[1]/age = 25) then 'is 25' else 'is not 25'", 'is 25'),
+                          ("if (//hotel/branch[@location = 'California']/staff[1]/age = 25) then 'is 25' else 'is not 25'", 'is 25'),
+                          ("if (count(/hotel/branch/staff) = 5) then true() else false()", 'true'),
+                          ("if (count(//hotel/branch/staff) = 5) then true() else false()", 'true'),
+                          ("for $i in /hotel/branch/staff return if ($i/age >= 40) then upper-case($i/surname) else lower-case($i/surname)", 'anderson'),
+                          ("given_name  =  'Christopher' and age  =  40", 'false'),
+                          ("//given_name  =  'Christopher' and //age  =  40", 'false'),
+                          #("(staff/given_name, staff/age)", 'Lisa'),
+                          ("(//staff/given_name, //staff/age)", 'Lisa'),
+                          #("hotel/branch[@location = 'California']/staff/age union hotel/branch[@location = 'Las Vegas']/staff/age", ''),
+                          ("(//hotel/branch[@location = 'California']/staff/age union //hotel/branch[@location = 'Las Vegas']/staff/age)", '60'),
+                          ("(200 to 210)", "205"),
+                          ("(//hotel/branch[@location = 'California']/staff/age union //hotel/branch[@location = 'Las Vegas']/staff/age)", "50"),
+                          ("(1, 9, 9, 5)", "5"),
+                          ("(3, (), (14, 15), 92, 653)", "653"),
+                          ("for $i in /hotel/branch/staff return $i/given_name", "Christopher"),
+                          ("for $i in //hotel/branch/staff return $i/given_name", "Christopher"),
+                          ("distinct-values(for $i in /hotel/branch/staff return $i/given_name)", "Jessica"),
+                          ("distinct-values(for $i in //hotel/branch/staff return $i/given_name)", "Jessica"),
+                          ("for $i in (7 to  15) return $i*10", "130"),
+                          ("some $i in /hotel/branch/staff satisfies $i/age < 20", "false"),
+                          ("some $i in //hotel/branch/staff satisfies $i/age < 20", "false"),
+                          ("every $i in /hotel/branch/staff satisfies $i/age > 20", "true"),
+                          ("every $i in //hotel/branch/staff satisfies $i/age > 20 ", "true"),
+                          ("let $x := branch[@location = 'California'], $y := branch[@location = 'Las Vegas'] return (avg($x/staff/age), avg($y/staff/age))", "27.5"),
+                          ("let $x := //branch[@location = 'California'], $y := //branch[@location = 'Las Vegas'] return (avg($x/staff/age), avg($y/staff/age))", "27.5"),
+                          ("let $nu := 1, $de := 1000 return  'probability = ' || $nu div $de * 100 || '%'", "0.1%"),
+                          ("let $nu := 2, $probability := function ($argument) { 'probability = ' ||  $nu div $argument  * 100 || '%'}, $de := 5 return $probability($de)", "40%"),
+                          ("'XPATH2.0-3.1 dissemination' instance of xs:string ", "true"),
+                          ("'new stackoverflow question incoming' instance of xs:integer ", "false"),
+                          ("'50000' cast as xs:integer", "50000"),
+                          ("//branch[@location = 'California']/staff[1]/surname eq 'Anderson'", "true"),
+                          ("fn:false()", "false")])
+def test_hotels(html_content, xpath, answer):
+    html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True)
+    assert type(html_content) == str
+    assert answer in html_content
+
+
+
+branches_to_visit = """<?xml version="1.0" ?>
+  <branches_to_visit>
+     <manager name="Godot" room_no="501">
+         <branch>Area 51</branch>
+         <branch>A place with no name</branch>
+         <branch>Stalsk12</branch>
+     </manager>
+      <manager name="Freya" room_no="305">
+         <branch>Stalsk12</branch>
+         <branch>Barcelona</branch>
+         <branch>Paris</branch>
+     </manager>
+ </branches_to_visit>"""
+@pytest.mark.parametrize("html_content", [branches_to_visit])
+@pytest.mark.parametrize("xpath, answer", [
+    ("manager[@name = 'Godot']/branch union manager[@name = 'Freya']/branch", "Area 51"),
+    ("//manager[@name = 'Godot']/branch union //manager[@name = 'Freya']/branch", "Stalsk12"),
+    ("manager[@name = 'Godot']/branch | manager[@name = 'Freya']/branch", "Stalsk12"),
+    ("//manager[@name = 'Godot']/branch | //manager[@name = 'Freya']/branch", "Stalsk12"),
+    ("manager/branch intersect manager[@name = 'Godot']/branch", "A place with no name"),
+    ("//manager/branch intersect //manager[@name = 'Godot']/branch", "A place with no name"),
+    ("manager[@name = 'Godot']/branch intersect manager[@name = 'Freya']/branch", ""),
+    ("manager/branch except manager[@name = 'Godot']/branch", "Barcelona"),
+    ("manager[@name = 'Godot']/branch[1]  eq 'Area 51'", "true"),
+    ("//manager[@name = 'Godot']/branch[1]  eq 'Area 51'", "true"),
+    ("manager[@name = 'Godot']/branch[1]  eq 'Seoul'", "false"),
+    ("//manager[@name = 'Godot']/branch[1]  eq 'Seoul'", "false"),
+    ("manager[@name = 'Godot']/branch[2] eq manager[@name = 'Freya']/branch[2]", "false"),
+    ("//manager[@name = 'Godot']/branch[2] eq //manager[@name = 'Freya']/branch[2]", "false"),
+    ("manager[1]/@room_no lt manager[2]/@room_no", "false"),
+    ("//manager[1]/@room_no lt //manager[2]/@room_no", "false"),
+    ("manager[1]/@room_no gt manager[2]/@room_no", "true"),
+    ("//manager[1]/@room_no gt //manager[2]/@room_no", "true"),
+    ("manager[@name = 'Godot']/branch[1]  = 'Area 51'", "true"),
+    ("//manager[@name = 'Godot']/branch[1]  = 'Area 51'", "true"),
+    ("manager[@name = 'Godot']/branch[1]  = 'Seoul'", "false"),
+    ("//manager[@name = 'Godot']/branch[1]  = 'Seoul'", "false"),
+    ("manager[@name = 'Godot']/branch  = 'Area 51'", "true"),
+    ("//manager[@name = 'Godot']/branch  = 'Area 51'", "true"),
+    ("manager[@name = 'Godot']/branch  = 'Barcelona'", "false"),
+    ("//manager[@name = 'Godot']/branch  = 'Barcelona'", "false"),
+    ("manager[1]/@room_no > manager[2]/@room_no", "true"),
+    ("//manager[1]/@room_no > //manager[2]/@room_no", "true"),
+    ("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[1]/branch[1]", "false"),
+    ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[1]/branch[1]", "false"),
+    ("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[1]/branch[3]", "true"),
+    ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[1]/branch[3]", "true"),
+    ("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] <<  manager[1]/branch[1]", "false"),
+    ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] <<  //manager[1]/branch[1]", "false"),
+    ("manager[@name = 'Godot']/branch[ . = 'Stalsk12']  >>  manager[1]/branch[1]", "true"),
+    ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] >>  //manager[1]/branch[1]", "true"),
+    ("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[@name = 'Freya']/branch[ . = 'Stalsk12']", "false"),
+    ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[@name = 'Freya']/branch[ . = 'Stalsk12']", "false"),
+    ("manager[1]/@name || manager[2]/@name", "GodotFreya"),
+    ("//manager[1]/@name || //manager[2]/@name", "GodotFreya"),
+                          ])
+def test_branches_to_visit(html_content, xpath, answer):
+    html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True)
+    assert type(html_content) == str
+    assert answer in html_content
+
+trips = """
+<trips>
+   <trip reservation_number="10">
+       <depart>2023-10-06</depart>
+       <arrive>2023-10-10</arrive>
+       <traveler name="Christopher Anderson">
+           <duration>4</duration>
+           <price>2000.00</price>
+       </traveler>
+   </trip>
+   <trip reservation_number="12">
+       <depart>2023-10-06</depart>
+       <arrive>2023-10-12</arrive>
+       <traveler name="Frank Carter">
+           <duration>6</duration>
+           <price>3500.34</price>
+       </traveler>
+   </trip>
+</trips>"""
+@pytest.mark.parametrize("html_content", [trips])
+@pytest.mark.parametrize("xpath, answer", [
+    ("1 + 9 * 9 + 5 div 5", "83"),
+    ("(1 + 9 * 9 + 5) div 6", "14.5"),
+    ("23 idiv 3", "7"),
+    ("23 div 3", "7.66666666"),
+    ("for $i in ./trip return $i/traveler/duration * $i/traveler/price", "21002.04"),
+    ("for $i in ./trip return $i/traveler/duration ", "4"),
+    ("for $i in .//trip return $i/traveler/duration * $i/traveler/price", "21002.04"),
+    ("sum(for $i in ./trip return $i/traveler/duration * $i/traveler/price)", "29002.04"),
+    ("sum(for $i in .//trip return $i/traveler/duration * $i/traveler/price)", "29002.04"),
+    #("trip[1]/depart - trip[1]/arrive", "fail_to_get_answer"),
+    #("//trip[1]/depart - //trip[1]/arrive", "fail_to_get_answer"),
+    #("trip[1]/depart + trip[1]/arrive", "fail_to_get_answer"),
+    #("xs:date(trip[1]/depart) + xs:date(trip[1]/arrive)", "fail_to_get_answer"),
+    ("(//trip[1]/arrive cast as xs:date) - (//trip[1]/depart cast as xs:date)", "P4D"),
+    ("(//trip[1]/depart cast as xs:date) - (//trip[1]/arrive cast as xs:date)", "-P4D"),
+    ("(//trip[1]/depart cast as xs:date) + xs:dayTimeDuration('P3D')", "2023-10-09"),
+    ("(//trip[1]/depart cast as xs:date) - xs:dayTimeDuration('P3D')", "2023-10-03"),
+    ("(456, 623) instance of xs:integer", "false"),
+    ("(456, 623) instance of xs:integer*", "true"),
+    ("/trips/trip instance of element()", "false"),
+    ("/trips/trip instance of element()*", "true"),
+    ("/trips/trip[1]/arrive instance of xs:date", "false"),
+    ("date(/trips/trip[1]/arrive) instance of xs:date", "true"),
+    ("'8' cast as xs:integer", "8"),
+    ("'11.1E3' cast as xs:double", "11100"),
+    ("6.5 cast as xs:integer", "6"),
+    #("/trips/trip[1]/arrive cast as xs:dateTime", "fail_to_get_answer"),
+    ("/trips/trip[1]/arrive cast as xs:date", "2023-10-10"),
+    ("('2023-10-12') cast as xs:date", "2023-10-12"),
+    ("for $i in //trip return concat($i/depart, '  ', $i/arrive)", "2023-10-06  2023-10-10"),
+                          ])
+def test_trips(html_content, xpath, answer):
+    html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True)
+    assert type(html_content) == str
+    assert answer in html_content

changedetectionio/tests/unit/test_watch_model.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python3
+
+# run from dir above changedetectionio/ dir
+# python3 -m unittest changedetectionio.tests.unit.test_notification_diff
+
+import unittest
+import os
+
+from changedetectionio.model import Watch
+
+# mostly
+class TestDiffBuilder(unittest.TestCase):
+
+    def test_watch_get_suggested_from_diff_timestamp(self):
+        import uuid as uuid_builder
+        watch = Watch.model(datastore_path='/tmp', default={})
+        watch.ensure_data_dir_exists()
+
+        watch['last_viewed'] = 110
+
+        watch.save_history_text(contents=b"hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
+        watch.save_history_text(contents=b"hello world", timestamp=105, snapshot_id=str(uuid_builder.uuid4()))
+        watch.save_history_text(contents=b"hello world", timestamp=109, snapshot_id=str(uuid_builder.uuid4()))
+        watch.save_history_text(contents=b"hello world", timestamp=112, snapshot_id=str(uuid_builder.uuid4()))
+        watch.save_history_text(contents=b"hello world", timestamp=115, snapshot_id=str(uuid_builder.uuid4()))
+        watch.save_history_text(contents=b"hello world", timestamp=117, snapshot_id=str(uuid_builder.uuid4()))
+
+        p = watch.get_next_snapshot_key_to_last_viewed
+        assert p == "112", "Correct last-viewed timestamp was detected"
+
+        # When there is only one step of difference from the end of the list, it should return second-last change
+        watch['last_viewed'] = 116
+        p = watch.get_next_snapshot_key_to_last_viewed
+        assert p == "115", "Correct 'second last' last-viewed timestamp was detected when using the last timestamp"
+
+        watch['last_viewed'] = 99
+        p = watch.get_next_snapshot_key_to_last_viewed
+        assert p == "100"
+
+        watch['last_viewed'] = 200
+        p = watch.get_next_snapshot_key_to_last_viewed
+        assert p == "115", "When the 'last viewed' timestamp is greater than the newest snapshot, return second last "
+
+        watch['last_viewed'] = 109
+        p = watch.get_next_snapshot_key_to_last_viewed
+        assert p == "109", "Correct when its the same time"
+
+        # new empty one
+        watch = Watch.model(datastore_path='/tmp', default={})
+        p = watch.get_next_snapshot_key_to_last_viewed
+        assert p == None, "None when no history available"
+
+if __name__ == '__main__':
+    unittest.main()

changedetectionio/tests/util.py

@@ -205,6 +205,9 @@ def live_server_setup(live_server):
         with open("test-datastore/notification-url.txt", "w") as f:
             f.write(request.url)
 
+        with open("test-datastore/notification-headers.txt", "w") as f:
+            f.write(str(request.headers))
+
         if request.content_type:
             with open("test-datastore/notification-content-type.txt", "w") as f:
                 f.write(request.content_type)

changedetectionio/update_worker.py

@@ -209,6 +209,7 @@ class update_worker(threading.Thread):
         from .processors import text_json_diff, restock_diff
 
         while not self.app.config.exit.is_set():
+            update_handler = None
 
             try:
                 queued_item_data = self.q.get(block=False)
@@ -229,17 +230,35 @@ class update_worker(threading.Thread):
                     now = time.time()
 
                     try:
-                        processor = self.datastore.data['watching'][uuid].get('processor','text_json_diff')
+                        # Processor is what we are using for detecting the "Change"
+                        processor = self.datastore.data['watching'][uuid].get('processor', 'text_json_diff')
+                        # if system...
+
+                        # Abort processing when the content was the same as the last fetch
+                        skip_when_same_checksum = queued_item_data.item.get('skip_when_checksum_same')
+
 
                         # @todo some way to switch by name
+                        # Init a new 'difference_detection_processor'
+
                         if processor == 'restock_diff':
-                            update_handler = restock_diff.perform_site_check(datastore=self.datastore)
+                            update_handler = restock_diff.perform_site_check(datastore=self.datastore,
+                                                                             watch_uuid=uuid
+                                                                             )
                         else:
                             # Used as a default and also by some tests
-                            update_handler = text_json_diff.perform_site_check(datastore=self.datastore)
+                            update_handler = text_json_diff.perform_site_check(datastore=self.datastore,
+                                                                               watch_uuid=uuid
+                                                                               )
 
+                        # Clear last errors (move to preflight func?)
                         self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
-                        changed_detected, update_obj, contents = update_handler.run(uuid, skip_when_checksum_same=queued_item_data.item.get('skip_when_checksum_same'))
+
+                        update_handler.call_browser()
+
+                        changed_detected, update_obj, contents = update_handler.run_changedetection(uuid,
+                                                                                    skip_when_checksum_same=skip_when_same_checksum,
+                                                                                    )
 
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
@@ -391,6 +410,9 @@ class update_worker(threading.Thread):
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
                         # Other serious error
                         process_changedetection_results = False
+#                        import traceback
+#                        print(traceback.format_exc())
+
                     else:
                         # Crash protection, the watch entry could have been removed by this point (during a slow chrome fetch etc)
                         if not self.datastore.data['watching'].get(uuid):

requirements.txt

@@ -46,6 +46,9 @@ beautifulsoup4
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 lxml
 
+# XPath 2.0-3.1 support
+elementpath
+
 selenium~=4.14.0
 
 werkzeug~=3.0