fix exit code checks

.github/workflows/test-only.yml

@@ -51,7 +51,6 @@ jobs:
         run: |
           # Unit tests
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
           
           # All tests
           docker run --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
@@ -98,58 +97,14 @@ jobs:
 
       - name: Test changedetection.io container starts+runs basically without error
         run: |
-          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
+          docker run -p 5556:5000 -d test-changedetectionio
           sleep 3
           # Should return 0 (no error) when grep finds it
           curl -s http://localhost:5556 |grep -q checkbox-uuid
           
           # and IPv6
           curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
-          docker kill test-changedetectionio
-
-      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
-        run: |
           
-          echo SIGINT Shutdown request test
-          docker run --name sig-test -d test-changedetectionio
-          sleep 3
-          echo ">>> Sending SIGINT to sig-test container"
-          docker kill --signal=SIGINT sig-test
-          sleep 3
-          # invert the check (it should be not 0/not running)
-          docker ps
-          # check signal catch(STDOUT) log
-          docker logs sig-test | grep 'Shutdown: Got Signal - SIGINT' || exit 1
-          test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
-            echo "Looks like container was running when it shouldnt be"
-            docker ps
-            exit 1
-          fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists 
-          docker rm sig-test
-          
-          echo SIGTERM Shutdown request test
-          docker run --name sig-test -d test-changedetectionio
-          sleep 3
-          echo ">>> Sending SIGTERM to sig-test container"
-          docker kill --signal=SIGTERM sig-test
-          sleep 3
-          # invert the check (it should be not 0/not running)
-          docker ps
-          docker logs sig-test | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
-          test -z "`docker ps|grep sig-test`"
-          if [ $? -ne 0 ]
-          then
-            echo "Looks like container was running when it shouldnt be"
-            docker ps
-            exit 1
-          fi
-          
-          # @todo - scan the container log to see the right "graceful shutdown" text exists           
-          docker rm sig-test
 
 #export WEBDRIVER_URL=http://localhost:4444/wd/hub
 #pytest tests/fetchers/test_content.py

Dockerfile

@@ -25,7 +25,7 @@ RUN pip install --target=/dependencies -r /requirements.txt
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN pip install --target=/dependencies playwright~=1.40 \
+RUN pip install --target=/dependencies playwright~=1.39 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
@@ -53,12 +53,12 @@ ENV PYTHONPATH=/usr/local
 
 EXPOSE 5000
 
-# The actual flask app module
+# The actual flask app
 COPY changedetectionio /app/changedetectionio
-# Starting wrapper
+
+# The eventlet server wrapper
 COPY changedetection.py /app/changedetection.py
 
 WORKDIR /app
-CMD ["python", "./changedetection.py", "-d", "/datastore"]
-
 
+CMD [ "python", "./changedetection.py" , "-d", "/datastore"]

README.md

@@ -11,10 +11,9 @@ _Live your data-life pro-actively._
 
 ![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 
-[**Get started with website page change monitoring straight away. Don't have time? Try our $8.99/month subscription, use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
+[**Don't have time? Let us host it for you! try our $8.99/month subscription - use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
 
 - Chrome browser included.
-- Nothing to install, access via browser login after signup.
 - Super fast, no registration needed setup.
 - Get started watching and receiving website change notifications straight away.
 
@@ -269,7 +268,3 @@ I offer commercial support, this software is depended on by network security, ae
 [license-shield]: https://img.shields.io/github/license/dgtlmoon/changedetection.io.svg?style=for-the-badge
 [release-link]: https://github.com/dgtlmoon/changedetection.io/releases
 [docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io
-
-## Third-party licenses
-
-changedetectionio.html_tools.elementpath_tostring: Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati), Licensed under [MIT license](https://github.com/sissaschool/elementpath/blob/master/LICENSE)

changedetection.py

@@ -1,6 +1,44 @@
 #!/usr/bin/python3
 
-# Only exists for direct CLI usage
+# Entry-point for running from the CLI when not installed via Pip, Pip will handle the console_scripts entry_points's from setup.py
+# It's recommended to use `pip3 install changedetection.io` and start with `changedetection.py` instead, it will be linkd to your global path.
+# or Docker.
+# Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-import changedetectionio
-changedetectionio.main()
+from changedetectionio import changedetection
+import multiprocessing
+import sys
+import os
+
+def sigchld_handler(_signo, _stack_frame):
+    import sys
+    print('Shutdown: Got SIGCHLD')
+    # https://stackoverflow.com/questions/40453496/python-multiprocessing-capturing-signals-to-restart-child-processes-or-shut-do
+    pid, status = os.waitpid(-1, os.WNOHANG | os.WUNTRACED | os.WCONTINUED)
+
+    print('Sub-process: pid %d status %d' % (pid, status))
+    if status != 0:
+        sys.exit(1)
+
+    raise SystemExit
+
+if __name__ == '__main__':
+
+    #signal.signal(signal.SIGCHLD, sigchld_handler)
+
+    # The only way I could find to get Flask to shutdown, is to wrap it and then rely on the subsystem issuing SIGTERM/SIGKILL
+    parse_process = multiprocessing.Process(target=changedetection.main)
+    parse_process.daemon = True
+    parse_process.start()
+    import time
+
+    try:
+        while True:
+            time.sleep(1)
+            if not parse_process.is_alive():
+                # Process died/crashed for some reason, exit with error set
+                sys.exit(1)
+
+    except KeyboardInterrupt:
+        #parse_process.terminate() not needed, because this process will issue it to the sub-process anyway
+        print ("Exited - CTRL+C")

changedetectionio/api/api_v1.py

@@ -76,7 +76,7 @@ class Watch(Resource):
         # Properties are not returned as a JSON, so add the required props manually
         watch['history_n'] = watch.history_n
         watch['last_changed'] = watch.last_changed
-        watch['viewed'] = watch.viewed
+
         return watch
 
     @auth.check_token
@@ -280,14 +280,11 @@ class CreateWatch(Resource):
             if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
                 continue
 
-            list[uuid] = {
-                'last_changed': watch.last_changed,
-                'last_checked': watch['last_checked'],
-                'last_error': watch['last_error'],
-                'title': watch['title'],
-                'url': watch['url'],
-                'viewed': watch.viewed
-            }
+            list[uuid] = {'url': watch['url'],
+                       'title': watch['title'],
+                       'last_checked': watch['last_checked'],
+                       'last_changed': watch.last_changed,
+                       'last_error': watch['last_error']}
 
         if request.args.get('recheck_all'):
             for uuid in self.datastore.data['watching'].keys():
@@ -296,61 +293,6 @@ class CreateWatch(Resource):
 
         return list, 200
 
-class Import(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-
-    @auth.check_token
-    def post(self):
-        """
-        @api {post} /api/v1/import Import a list of watched URLs
-        @apiDescription Accepts a line-feed separated list of URLs to import, additionally with ?tag_uuids=(tag  id), ?tag=(name), ?proxy={key}, ?dedupe=true (default true) one URL per line.
-        @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/import --data-binary @list-of-sites.txt -H"x-api-key:8a111a21bc2f8f1dd9b9353bbd46049a"
-        @apiName Import
-        @apiGroup Watch
-        @apiSuccess (200) {List} OK List of watch UUIDs added
-        @apiSuccess (500) {String} ERR Some other error
-        """
-
-        extras = {}
-
-        if request.args.get('proxy'):
-            plist = self.datastore.proxy_list
-            if not request.args.get('proxy') in plist:
-                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
-            else:
-                extras['proxy'] = request.args.get('proxy')
-
-        dedupe = strtobool(request.args.get('dedupe', 'true'))
-
-        tags = request.args.get('tag')
-        tag_uuids = request.args.get('tag_uuids')
-
-        if tag_uuids:
-            tag_uuids = tag_uuids.split(',')
-
-        urls = request.get_data().decode('utf8').splitlines()
-        added = []
-        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
-        for url in urls:
-            url = url.strip()
-            if not len(url):
-                continue
-
-            # If hosts that only contain alphanumerics are allowed ("localhost" for example)
-            if not validators.url(url, simple_host=allow_simplehost):
-                return f"Invalid or unsupported URL - {url}", 400
-
-            if dedupe and self.datastore.url_exists(url):
-                continue
-
-            new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags, tag_uuids=tag_uuids)
-            added.append(new_uuid)
-
-        return added
-
 class SystemInfo(Resource):
     def __init__(self, **kwargs):
         # datastore is a black box dependency

changedetectionio/blueprint/browser_steps/__init__.py

@@ -25,9 +25,10 @@ from distutils.util import strtobool
 from flask import Blueprint, request, make_response
 import logging
 import os
+import re
 
 from changedetectionio.store import ChangeDetectionStore
-from changedetectionio.flask_app import login_optionally_required
+from changedetectionio import login_optionally_required
 
 browsersteps_sessions = {}
 io_interface_context = None
@@ -151,7 +152,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])
     def browsersteps_ui_update():
         import base64
-        import playwright._impl._errors
+        import playwright._impl._api_types
         global browsersteps_sessions
         from changedetectionio.blueprint.browser_steps import browser_steps
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -110,7 +110,7 @@ class steppable_browser_interface():
         self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
 
     def action_click_element_if_exists(self, selector, value):
-        import playwright._impl._errors as _api_types
+        import playwright._impl._api_types as _api_types
         print("Clicking element if exists")
         if not len(selector.strip()):
             return
@@ -123,9 +123,6 @@ class steppable_browser_interface():
             return
 
     def action_click_x_y(self, selector, value):
-        if not re.match(r'^\s?\d+\s?,\s?\d+\s?$', value):
-            raise Exception("'Click X,Y' step should be in the format of '100 , 90'")
-
         x, y = value.strip().split(',')
         x = int(float(x.strip()))
         y = int(float(y.strip()))

changedetectionio/blueprint/tags/__init__.py

@@ -1,6 +1,6 @@
 from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
 from changedetectionio.store import ChangeDetectionStore
-from changedetectionio.flask_app import login_optionally_required
+from changedetectionio import login_optionally_required
 
 
 def construct_blueprint(datastore: ChangeDetectionStore):

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -69,12 +69,11 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                                 {% endif %}
                             </ul>
                         </li>
-                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash. To specify XPath to be used explicitly or the XPath rule starts with an XPath function: Prefix with <code>xpath:</code>
+                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
                             <ul>
-                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath:count(//*[contains(@class, 'sametext')])</code>, <a
+                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
                                 href="http://xpather.com/" target="new">test your XPath here</a></li>
                                 <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
-                                <li>To use XPath1.0: Prefix with <code>xpath1:</code></li>
                             </ul>
                             </li>
                     </ul>

changedetectionio/changedetection.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python3
+
+# Launch as a eventlet.wsgi server instance.
+
+from distutils.util import strtobool
+from json.decoder import JSONDecodeError
+
+import eventlet
+import eventlet.wsgi
+import getopt
+import os
+import signal
+import socket
+import sys
+
+from . import store, changedetection_app, content_fetcher
+from . import __version__
+
+# Only global so we can access it in the signal handler
+app = None
+datastore = None
+
+def sigterm_handler(_signo, _stack_frame):
+    global app
+    global datastore
+#    app.config.exit.set()
+    print('Shutdown: Got SIGTERM, DB saved to disk')
+    datastore.sync_to_json()
+#    raise SystemExit
+
+def main():
+    global datastore
+    global app
+
+    datastore_path = None
+    do_cleanup = False
+    host = ''
+    ipv6_enabled = False
+    port = os.environ.get('PORT') or 5000
+    ssl_mode = False
+
+    # On Windows, create and use a default path.
+    if os.name == 'nt':
+        datastore_path = os.path.expandvars(r'%APPDATA%\changedetection.io')
+        os.makedirs(datastore_path, exist_ok=True)
+    else:
+        # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
+        datastore_path = os.path.join(os.getcwd(), "../datastore")
+
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:", "port")
+    except getopt.GetoptError:
+        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
+        sys.exit(2)
+
+    create_datastore_dir = False
+
+    for opt, arg in opts:
+        if opt == '-s':
+            ssl_mode = True
+
+        if opt == '-h':
+            host = arg
+
+        if opt == '-p':
+            port = int(arg)
+
+        if opt == '-d':
+            datastore_path = arg
+
+        if opt == '-6':
+            print ("Enabling IPv6 listen support")
+            ipv6_enabled = True
+
+        # Cleanup (remove text files that arent in the index)
+        if opt == '-c':
+            do_cleanup = True
+
+        # Create the datadir if it doesnt exist
+        if opt == '-C':
+            create_datastore_dir = True
+
+    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
+    app_config = {'datastore_path': datastore_path}
+
+    if not os.path.isdir(app_config['datastore_path']):
+        if create_datastore_dir:
+            os.mkdir(app_config['datastore_path'])
+        else:
+            print(
+                "ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists or specify a directory with the -d option.\n"
+                "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
+            sys.exit(2)
+
+    try:
+        datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
+    except JSONDecodeError as e:
+        # Dont' start if the JSON DB looks corrupt
+        print ("ERROR: JSON DB or Proxy List JSON at '{}' appears to be corrupt, aborting".format(app_config['datastore_path']))
+        print(str(e))
+        return
+
+    app = changedetection_app(app_config, datastore)
+
+    signal.signal(signal.SIGTERM, sigterm_handler)
+
+    # Go into cleanup mode
+    if do_cleanup:
+        datastore.remove_unused_snapshots()
+
+    app.config['datastore_path'] = datastore_path
+
+
+    @app.context_processor
+    def inject_version():
+        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
+                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
+                    has_password=datastore.data['settings']['application']['password'] != False
+                    )
+
+    # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.
+    # @Note: Incompatible with password login (and maybe other features) for now, submit a PR!
+    @app.after_request
+    def hide_referrer(response):
+        if strtobool(os.getenv("HIDE_REFERER", 'false')):
+            response.headers["Referrer-Policy"] = "no-referrer"
+
+        return response
+
+    # Proxy sub-directory support
+    # Set environment var USE_X_SETTINGS=1 on this script
+    # And then in your proxy_pass settings
+    #
+    #         proxy_set_header Host "localhost";
+    #         proxy_set_header X-Forwarded-Prefix /app;
+
+    if os.getenv('USE_X_SETTINGS'):
+        print ("USE_X_SETTINGS is ENABLED\n")
+        from werkzeug.middleware.proxy_fix import ProxyFix
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
+
+    s_type = socket.AF_INET6 if ipv6_enabled else socket.AF_INET
+
+    if ssl_mode:
+        # @todo finalise SSL config, but this should get you in the right direction if you need it.
+        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), s_type),
+                                               certfile='cert.pem',
+                                               keyfile='privkey.pem',
+                                               server_side=True), app)
+
+    else:
+        eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app)
+

changedetectionio/content_fetcher.py

@@ -172,7 +172,7 @@ class Fetcher():
 
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-        from playwright._impl._errors import TimeoutError
+        from playwright._impl._api_types import TimeoutError
         from jinja2 import Environment
         jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
 
@@ -436,7 +436,7 @@ class base_html_playwright(Fetcher):
                     is_binary)
 
         from playwright.sync_api import sync_playwright
-        import playwright._impl._errors
+        import playwright._impl._api_types
 
         self.delete_browser_steps_screenshots()
         response = None
@@ -489,7 +489,7 @@ class base_html_playwright(Fetcher):
             try:
                 if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
                     browsersteps_interface.action_execute_js(value=self.webdriver_js_execute_code, selector=None)
-            except playwright._impl._errors.TimeoutError as e:
+            except playwright._impl._api_types.TimeoutError as e:
                 context.close()
                 browser.close()
                 # This can be ok, we will try to grab what we could retrieve

changedetectionio/forms.py

@@ -328,30 +328,11 @@ class ValidateCSSJSONXPATHInput(object):
                 return
 
             # Does it look like XPath?
-            if line.strip()[0] == '/' or line.strip().startswith('xpath:'):
-                if not self.allow_xpath:
-                    raise ValidationError("XPath not permitted in this field!")
-                from lxml import etree, html
-                import elementpath
-                # xpath 2.0-3.1
-                from elementpath.xpath3 import XPath3Parser
-                tree = html.fromstring("<html></html>")
-                line = line.replace('xpath:', '')
-
-                try:
-                    elementpath.select(tree, line.strip(), parser=XPath3Parser)
-                except elementpath.ElementPathError as e:
-                    message = field.gettext('\'%s\' is not a valid XPath expression. (%s)')
-                    raise ValidationError(message % (line, str(e)))
-                except:
-                    raise ValidationError("A system-error occurred when validating your XPath expression")
-
-            if line.strip().startswith('xpath1:'):
+            if line.strip()[0] == '/':
                 if not self.allow_xpath:
                     raise ValidationError("XPath not permitted in this field!")
                 from lxml import etree, html
                 tree = html.fromstring("<html></html>")
-                line = re.sub(r'^xpath1:', '', line)
 
                 try:
                     tree.xpath(line.strip())

changedetectionio/html_tools.py

@@ -69,89 +69,10 @@ def element_removal(selectors: List[str], html_content):
     selector = ",".join(selectors)
     return subtractive_css_selector(selector, html_content)
 
-def elementpath_tostring(obj):
-    """
-    change elementpath.select results to string type
-    # The MIT License (MIT), Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati)
-    # https://github.com/sissaschool/elementpath/blob/dfcc2fd3d6011b16e02bf30459a7924f547b47d0/elementpath/xpath_tokens.py#L1038
-    """
-
-    import elementpath
-    from decimal import Decimal
-    import math
-
-    if obj is None:
-        return ''
-    # https://elementpath.readthedocs.io/en/latest/xpath_api.html#elementpath.select
-    elif isinstance(obj, elementpath.XPathNode):
-        return obj.string_value
-    elif isinstance(obj, bool):
-        return 'true' if obj else 'false'
-    elif isinstance(obj, Decimal):
-        value = format(obj, 'f')
-        if '.' in value:
-            return value.rstrip('0').rstrip('.')
-        return value
-
-    elif isinstance(obj, float):
-        if math.isnan(obj):
-            return 'NaN'
-        elif math.isinf(obj):
-            return str(obj).upper()
-
-        value = str(obj)
-        if '.' in value:
-            value = value.rstrip('0').rstrip('.')
-        if '+' in value:
-            value = value.replace('+', '')
-        if 'e' in value:
-            return value.upper()
-        return value
-
-    return str(obj)
 
 # Return str Utf-8 of matched rules
 def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False, is_rss=False):
     from lxml import etree, html
-    import elementpath
-    # xpath 2.0-3.1
-    from elementpath.xpath3 import XPath3Parser
-
-    parser = etree.HTMLParser()
-    if is_rss:
-        # So that we can keep CDATA for cdata_in_document_to_text() to process
-        parser = etree.XMLParser(strip_cdata=False)
-
-    tree = html.fromstring(bytes(html_content, encoding='utf-8'), parser=parser)
-    html_block = ""
-
-    r = elementpath.select(tree, xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'}, parser=XPath3Parser)
-    #@note: //title/text() wont work where <title>CDATA..
-
-    if type(r) != list:
-        r = [r]
-
-    for element in r:
-        # When there's more than 1 match, then add the suffix to separate each line
-        # And where the matched result doesn't include something that will cause Inscriptis to add a newline
-        # (This way each 'match' reliably has a new-line in the diff)
-        # Divs are converted to 4 whitespaces by inscriptis
-        if append_pretty_line_formatting and len(html_block) and (not hasattr( element, 'tag' ) or not element.tag in (['br', 'hr', 'div', 'p'])):
-            html_block += TEXT_FILTER_LIST_LINE_SUFFIX
-
-        if type(element) == str:
-            html_block += element
-        elif issubclass(type(element), etree._Element) or issubclass(type(element), etree._ElementTree):
-            html_block += etree.tostring(element, pretty_print=True).decode('utf-8')
-        else:
-            html_block += elementpath_tostring(element)
-
-    return html_block
-
-# Return str Utf-8 of matched rules
-# 'xpath1:'
-def xpath1_filter(xpath_filter, html_content, append_pretty_line_formatting=False, is_rss=False):
-    from lxml import etree, html
 
     parser = None
     if is_rss:

changedetectionio/model/Watch.py

@@ -38,7 +38,6 @@ base_config = {
     'track_ldjson_price_data': None,
     'headers': {},  # Extra headers to send
     'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
-    'in_stock' : None,
     'in_stock_only' : True, # Only trigger change on going to instock from out-of-stock
     'include_filters': [],
     'last_checked': 0,
@@ -114,8 +113,7 @@ class model(dict):
 
     @property
     def viewed(self):
-        # Don't return viewed when last_viewed is 0 and newest_key is 0
-        if int(self['last_viewed']) and int(self['last_viewed']) >= int(self.newest_history_key) :
+        if int(self['last_viewed']) >= int(self.newest_history_key) :
             return True
 
         return False
@@ -264,38 +262,6 @@ class model(dict):
         bump = self.history
         return self.__newest_history_key
 
-    # Given an arbitrary timestamp, find the closest next key
-    # For example, last_viewed = 1000 so it should return the next 1001 timestamp
-    #
-    # used for the [diff] button so it can preset a smarter from_version
-    @property
-    def get_next_snapshot_key_to_last_viewed(self):
-
-        """Unfortunately for now timestamp is stored as string key"""
-        keys = list(self.history.keys())
-        if not keys:
-            return None
-
-        last_viewed = int(self.get('last_viewed'))
-        prev_k = keys[0]
-        sorted_keys = sorted(keys, key=lambda x: int(x))
-        sorted_keys.reverse()
-
-        # When the 'last viewed' timestamp is greater than the newest snapshot, return second last
-        if last_viewed > int(sorted_keys[0]):
-            return sorted_keys[1]
-
-        for k in sorted_keys:
-            if int(k) < last_viewed:
-                if prev_k == sorted_keys[0]:
-                    # Return the second last one so we dont recommend the same version compares itself
-                    return sorted_keys[1]
-
-                return prev_k
-            prev_k = k
-
-        return keys[0]
-
     def get_history_snapshot(self, timestamp):
         import brotli
         filepath = self.history[timestamp]

changedetectionio/notification.py

@@ -46,9 +46,6 @@ from apprise.decorators import notify
 @notify(on="puts")
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
-    from apprise.utils import parse_url as apprise_parse_url
-    from apprise.URLBase import URLBase
-
     url = kwargs['meta'].get('url')
 
     if url.startswith('post'):
@@ -71,45 +68,16 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     url = url.replace('delete://', 'http://')
     url = url.replace('deletes://', 'https://')
 
-    headers = {}
-    params = {}
-    auth = None
-
-    # Convert /foobar?+some-header=hello to proper header dictionary
-    results = apprise_parse_url(url)
-    if results:
-        # Add our headers that the user can potentially over-ride if they wish
-        # to to our returned result set and tidy entries by unquoting them
-        headers = {URLBase.unquote(x): URLBase.unquote(y)
-                   for x, y in results['qsd+'].items()}
-
-        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
-        # but here we are making straight requests, so we need todo convert this against apprise's logic
-        for k, v in results['qsd'].items():
-            if not k.strip('+-') in results['qsd+'].keys():
-                params[URLBase.unquote(k)] = URLBase.unquote(v)
-
-        # Determine Authentication
-        auth = ''
-        if results.get('user') and results.get('password'):
-            auth = (URLBase.unquote(results.get('user')), URLBase.unquote(results.get('user')))
-        elif results.get('user'):
-            auth = (URLBase.unquote(results.get('user')))
-
     # Try to auto-guess if it's JSON
+    headers = {}
     try:
         json.loads(body)
-        headers['Content-Type'] = 'application/json; charset=utf-8'
+        headers = {'Content-Type': 'application/json; charset=utf-8'}
     except ValueError as e:
         pass
 
-    r(results.get('url'),
-      auth=auth,
-      data=body,
-      headers=headers,
-      params=params
-      )
+
+    r(url, headers=headers, data=body)
 
 
 def process_notification(n_object, datastore):

changedetectionio/processors/__init__.py

@@ -52,11 +52,6 @@ class difference_detection_processor():
                 prefer_fetch_backend = 'base_html_playwright'
                 browser_connection_url = connection[0].get('browser_connection_url')
 
-        # PDF should be html_requests because playwright will serve it up (so far) in a embedded page
-        # @todo https://github.com/dgtlmoon/changedetection.io/issues/2019
-        # @todo needs test to or a fix
-        if self.watch.is_pdf:
-           prefer_fetch_backend = "html_requests"
 
         # Grab the right kind of 'fetcher', (playwright, requests, etc)
         if hasattr(content_fetcher, prefer_fetch_backend):

changedetectionio/processors/text_json_diff.py

@@ -173,11 +173,6 @@ class perform_site_check(difference_detection_processor):
                                                                     html_content=self.fetcher.content,
                                                                     append_pretty_line_formatting=not watch.is_source_type_url,
                                                                     is_rss=is_rss)
-                        elif filter_rule.startswith('xpath1:'):
-                            html_content += html_tools.xpath1_filter(xpath_filter=filter_rule.replace('xpath1:', ''),
-                                                                    html_content=self.fetcher.content,
-                                                                    append_pretty_line_formatting=not watch.is_source_type_url,
-                                                                    is_rss=is_rss)
                         else:
                             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
                             html_content += html_tools.include_filters(include_filters=filter_rule,

changedetectionio/res/stock-not-in-stock.js

@@ -1,7 +1,6 @@
 function isItemInStock() {
   // @todo Pass these in so the same list can be used in non-JS fetchers
   const outOfStockTexts = [
-    ' أخبرني عندما يتوفر',
     '0 in stock',
     'agotado',
     'artikel zurzeit vergriffen',
@@ -17,19 +16,14 @@ function isItemInStock() {
     'currently have any tickets for this',
     'currently unavailable',
     'dostępne wkrótce',
-    'dostępne wkrótce',
     'en rupture de stock',
     'ist derzeit nicht auf lager',
-    'ist derzeit nicht auf lager',
     'item is no longer available',
-    'let me know when it\'s available',
     'message if back in stock',
     'nachricht bei',
     'nicht auf lager',
     'nicht lieferbar',
     'nicht zur verfügung',
-    'niet leverbaar',
-    'niet beschikbaar',
     'no disponible temporalmente',
     'no longer in stock',
     'no tickets available',
@@ -42,17 +36,13 @@ function isItemInStock() {
     'out-of-stock',
     'produkt niedostępny',
     'sold out',
-    'sold-out',
     'temporarily out of stock',
     'temporarily unavailable',
     'tickets unavailable',
-    'tijdelijk uitverkocht',
     'unavailable tickets',
     'we do not currently have an estimate of when this product will be back in stock.',
     'zur zeit nicht an lager',
-    '品切れ',
     '已售完',
-    '품절'
   ];
 
 
@@ -115,4 +105,4 @@ function isItemInStock() {
 }
 
 // returns the element text that makes it think it's out of stock
-return isItemInStock();
+return isItemInStock();

changedetectionio/res/xpath_element_scraper.js

@@ -170,12 +170,9 @@ if (include_filters.length) {
 
         try {
             // is it xpath?
-            if (f.startsWith('/') || f.startsWith('xpath')) {
-                var qry_f = f.replace(/xpath(:|\d:)/, '')
-                console.log("[xpath] Scanning for included filter " + qry_f)
-                q = document.evaluate(qry_f, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+            if (f.startsWith('/') || f.startsWith('xpath:')) {
+                q = document.evaluate(f.replace('xpath:', ''), document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
             } else {
-                console.log("[css] Scanning for included filter " + f)
                 q = document.querySelector(f);
             }
         } catch (e) {
@@ -185,18 +182,8 @@ if (include_filters.length) {
         }
 
         if (q) {
-            // Try to resolve //something/text() back to its /something so we can atleast get the bounding box
-            try {
-                if (typeof q.nodeName == 'string' && q.nodeName === '#text') {
-                    q = q.parentElement
-                }
-            } catch (e) {
-                console.log(e)
-                console.log("xpath_element_scraper: #text resolver")
-            }
-
             // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
-            if (typeof q.getBoundingClientRect == 'function') {
+            if (q.hasOwnProperty('getBoundingClientRect')) {
                 bbox = q.getBoundingClientRect();
                 console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
             } else {
@@ -205,8 +192,7 @@ if (include_filters.length) {
                     bbox = q.ownerElement.getBoundingClientRect();
                     console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
                 } catch (e) {
-                    console.log(e)
-                    console.log("xpath_element_scraper: error looking up q.ownerElement")
+                    console.log("xpath_element_scraper: error looking up ownerElement")
                 }
             }
         }

changedetectionio/static/images/bell-off.svg

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg width="15" height="16.363636" viewBox="0 0 15 16.363636" xmlns="http://www.w3.org/2000/svg" >
+<svg width="15" height="16.363636" viewBox="0 0 15 16.363636" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg">
   <path d="m 14.318182,11.762045 v 1.1925 H 5.4102273 L 11.849318,7.1140909 C 12.234545,9.1561364 12.54,11.181818 14.318182,11.762045 Z m -6.7984093,4.601591 c 1.0759091,0 2.0256823,-0.955909 2.0256823,-2.045454 H 5.4545455 c 0,1.089545 0.9879545,2.045454 2.0652272,2.045454 z M 15,2.8622727 0.9177273,15.636136 0,14.627045 l 1.8443182,-1.6725 h -1.1625 v -1.1925 C 4.0070455,10.677273 2.1784091,4.5388636 5.3611364,2.6897727 5.8009091,2.4347727 6.0709091,1.9609091 6.0702273,1.4488636 v -0.00205 C 6.0702273,0.64772727 6.7104545,0 7.5,0 8.2895455,0 8.9297727,0.64772727 8.9297727,1.4468182 v 0.00205 C 8.9290909,1.9602319 9.199773,2.4354591 9.638864,2.6897773 10.364318,3.111141 10.827273,3.7568228 11.1525,4.5129591 L 14.085682,1.8531818 Z M 6.8181818,1.3636364 C 6.8181818,1.74 7.1236364,2.0454545 7.5,2.0454545 7.8763636,2.0454545 8.1818182,1.74 8.1818182,1.3636364 8.1818182,0.98795455 7.8763636,0.68181818 7.5,0.68181818 c -0.3763636,0 -0.6818182,0.30613637 -0.6818182,0.68181822 z" id="path2" style="fill:#f8321b;stroke-width:0.681818;fill-opacity:1"/>
 </svg>

changedetectionio/static/images/brightdata.svg

@@ -10,7 +10,7 @@
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns="http://www.w3.org/2000/svg"
-   >
+   xmlns:svg="http://www.w3.org/2000/svg">
   <defs
      id="defs16" />
   <sodipodi:namedview

changedetectionio/static/images/copy.svg

@@ -12,7 +12,7 @@
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns="http://www.w3.org/2000/svg"
-   ><defs
+   xmlns:svg="http://www.w3.org/2000/svg"><defs
      id="defs11" /><sodipodi:namedview
      id="namedview9"
      pagecolor="#ffffff"

changedetectionio/static/images/email.svg

@@ -10,7 +10,7 @@
    viewBox="0 0 7.1975545 4.7993639"
    xml:space="preserve"
    xmlns="http://www.w3.org/2000/svg"
-   ><defs
+   xmlns:svg="http://www.w3.org/2000/svg"><defs
    id="defs19" />
 <g
    id="g14"

changedetectionio/static/images/notice.svg

@@ -9,7 +9,7 @@
    id="svg5"
    xmlns:xlink="http://www.w3.org/1999/xlink"
    xmlns="http://www.w3.org/2000/svg"
-   >
+   xmlns:svg="http://www.w3.org/2000/svg">
   <defs
      id="defs2" />
   <g

changedetectionio/static/images/oxylabs.svg

@@ -10,7 +10,7 @@
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns="http://www.w3.org/2000/svg"
-   >
+   xmlns:svg="http://www.w3.org/2000/svg">
   <defs
      id="defs12" />
   <sodipodi:namedview

changedetectionio/static/images/pause.svg

@@ -3,6 +3,7 @@
    xmlns:dc="http://purl.org/dc/elements/1.1/"
    xmlns:cc="http://creativecommons.org/ns#"
    xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
    xmlns="http://www.w3.org/2000/svg"
    version="1.1"
    id="Capa_1"

changedetectionio/static/images/play.svg

@@ -13,6 +13,7 @@
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg"
    xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
    xmlns:cc="http://creativecommons.org/ns#"
    xmlns:dc="http://purl.org/dc/elements/1.1/"><sodipodi:namedview

changedetectionio/static/images/spread-white.svg

@@ -6,7 +6,7 @@
    version="1.1"
    id="svg6"
    xmlns="http://www.w3.org/2000/svg"
-   >
+   xmlns:svg="http://www.w3.org/2000/svg">
   <defs
      id="defs10" />
   <path

changedetectionio/static/images/spread.svg

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg width="18" height="19.92" viewBox="0 0 18 19.92" xmlns="http://www.w3.org/2000/svg" >
+<svg width="18" height="19.92" viewBox="0 0 18 19.92" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg">
   <path d="M -3,-2 H 21 V 22 H -3 Z" fill="none" id="path2"/>
   <path d="m 15,14.08 c -0.76,0 -1.44,0.3 -1.96,0.77 L 5.91,10.7 C 5.96,10.47 6,10.24 6,10 6,9.76 5.96,9.53 5.91,9.3 L 12.96,5.19 C 13.5,5.69 14.21,6 15,6 16.66,6 18,4.66 18,3 18,1.34 16.66,0 15,0 c -1.66,0 -3,1.34 -3,3 0,0.24 0.04,0.47 0.09,0.7 L 5.04,7.81 C 4.5,7.31 3.79,7 3,7 1.34,7 0,8.34 0,10 c 0,1.66 1.34,3 3,3 0.79,0 1.5,-0.31 2.04,-0.81 l 7.12,4.16 c -0.05,0.21 -0.08,0.43 -0.08,0.65 0,1.61 1.31,2.92 2.92,2.92 1.61,0 2.92,-1.31 2.92,-2.92 0,-1.61 -1.31,-2.92 -2.92,-2.92 z" id="path4" style="fill:#0078e7;fill-opacity:1"/>
 </svg>

changedetectionio/static/js/global-settings.js

@@ -1,4 +1,19 @@
 $(document).ready(function () {
+    function toggle() {
+        if ($('input[name="application-fetch_backend"]:checked').val() != 'html_requests') {
+            $('#requests-override-options').hide();
+            $('#webdriver-override-options').show();
+        } else {
+            $('#requests-override-options').show();
+            $('#webdriver-override-options').hide();
+        }
+    }
+
+    $('input[name="application-fetch_backend"]').click(function (e) {
+        toggle();
+    });
+    toggle();
+
     $("#api-key").hover(
         function () {
             $("#api-key-copy").html('copy').fadeIn();

changedetectionio/static/js/vis.js

@@ -1,29 +0,0 @@
-$(document).ready(function () {
-
-    // Lazy Hide/Show elements mechanism
-    $('[data-visible-for]').hide();
-    function show_related_elem(e) {
-        var n = $(e).attr('name') + "=" + $(e).val();
-        if (n === 'fetch_backend=system') {
-            n = "fetch_backend=" + default_system_fetch_backend;
-        }
-        $(`[data-visible-for~="${n}"]`).show();
-    }
-    $(':radio').on('keyup keypress blur change click', function (e) {
-        $(`[data-visible-for]`).hide();
-        $('.advanced-options').hide();
-        show_related_elem(this);
-    });
-
-    $(':radio:checked').each(function (e) {
-       show_related_elem(this);
-    })
-
-
-    // Show advanced
-    $('.show-advanced').click(function (e) {
-        $(this).closest('.tab-pane-inner').find('.advanced-options').each(function (e) {
-            $(this).toggle();
-        })
-    });
-});

changedetectionio/static/js/visual-selector.js

@@ -149,7 +149,7 @@ $(document).ready(function () {
             // @todo In the future paint all that match
             for (const c of current_default_xpath) {
                 for (var i = selector_data['size_pos'].length; i !== 0; i--) {
-                    if (selector_data['size_pos'][i - 1].xpath.trim() === c.trim()) {
+                    if (selector_data['size_pos'][i - 1].xpath === c) {
                         console.log("highlighting " + c);
                         current_selected_i = i - 1;
                         highlight_current_selected_i();

changedetectionio/static/js/watch-settings.js

@@ -1,4 +1,40 @@
 $(document).ready(function () {
+    function toggle() {
+        if ($('input[name="fetch_backend"]:checked').val() == 'html_webdriver') {
+            if (playwright_enabled) {
+                // playwright supports headers, so hide everything else
+                // See #664
+                $('#requests-override-options #request-method').hide();
+                $('#requests-override-options #request-body').hide();
+
+                // @todo connect this one up
+                $('#ignore-status-codes-option').hide();
+            } else {
+                // selenium/webdriver doesnt support anything afaik, hide it all
+                $('#requests-override-options').hide();
+            }
+
+            $('#webdriver-override-options').show();
+
+        } else if ($('input[name="fetch_backend"]:checked').val() == 'system') {
+            $('#requests-override-options #request-method').hide();
+            $('#requests-override-options #request-body').hide();
+            $('#ignore-status-codes-option').hide();
+            $('#requests-override-options').hide();
+            $('#webdriver-override-options').hide();
+        } else {
+
+            $('#requests-override-options').show();
+            $('#requests-override-options *:hidden').show();
+            $('#webdriver-override-options').hide();
+        }
+    }
+
+    $('input[name="fetch_backend"]').click(function (e) {
+        toggle();
+    });
+    toggle();
+
     $('#notification-setting-reset-to-default').click(function (e) {
         $('#notification_title').val('');
         $('#notification_body').val('');

changedetectionio/static/styles/scss/styles.scss

@@ -402,24 +402,8 @@ label {
   }
 
   #watch-add-wrapper-zone {
-
-    @media only screen and (min-width: 760px) {
-      display: flex;
-      gap: 0.3rem;
-      flex-direction: row;
-    }
-    /* URL field grows always, other stay static in width */
-    > span {
-      flex-grow: 0;
-
-      input {
-        width: 100%;
-        padding-right: 1em;
-      }
-
-      &:first-child {
-        flex-grow: 1;
-      }
+    >div {
+      display: inline-block;
     }
 
     @media only screen and (max-width: 760px) {
@@ -960,8 +944,10 @@ ul {
 
 @import "parts/_visualselector";
 
-#webdriver_delay {
+#webdriver-override-options {
+  input[type="number"] {
     width: 5em;
+  }
 }
 
 #api-key {

changedetectionio/static/styles/styles.css

@@ -683,23 +683,11 @@ label:hover {
   #new-watch-form legend {
     color: var(--color-text-legend);
     font-weight: bold; }
-  #new-watch-form #watch-add-wrapper-zone {
-    /* URL field grows always, other stay static in width */ }
-    @media only screen and (min-width: 760px) {
-      #new-watch-form #watch-add-wrapper-zone {
-        display: flex;
-        gap: 0.3rem;
-        flex-direction: row; } }
-    #new-watch-form #watch-add-wrapper-zone > span {
-      flex-grow: 0; }
-      #new-watch-form #watch-add-wrapper-zone > span input {
-        width: 100%;
-        padding-right: 1em; }
-      #new-watch-form #watch-add-wrapper-zone > span:first-child {
-        flex-grow: 1; }
-    @media only screen and (max-width: 760px) {
-      #new-watch-form #watch-add-wrapper-zone #url {
-        width: 100%; } }
+  #new-watch-form #watch-add-wrapper-zone > div {
+    display: inline-block; }
+  @media only screen and (max-width: 760px) {
+    #new-watch-form #watch-add-wrapper-zone #url {
+      width: 100%; } }
 
 #diff-col {
   padding-left: 40px; }
@@ -1077,7 +1065,7 @@ ul {
 #selector-current-xpath {
   font-size: 80%; }
 
-#webdriver_delay {
+#webdriver-override-options input[type="number"] {
   width: 5em; }
 
 #api-key:hover {

changedetectionio/store.py

@@ -234,7 +234,7 @@ class ChangeDetectionStore:
 
         # Probably their should be dict...
         for watch in self.data['watching'].values():
-            if watch['url'].lower() == url.lower():
+            if watch['url'] == url:
                 return True
 
         return False
@@ -248,7 +248,6 @@ class ChangeDetectionStore:
                 'check_count': 0,
                 'fetch_time' : 0.0,
                 'has_ldjson_price_data': None,
-                'in_stock': None,
                 'last_checked': 0,
                 'last_error': False,
                 'last_notification_error': False,
@@ -334,8 +333,7 @@ class ChangeDetectionStore:
 
         # Or if UUIDs given directly
         if tag_uuids:
-            for t in tag_uuids:
-                apply_extras['tags'] = list(set(apply_extras['tags'] + [t.strip()]))
+            apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids))
 
         # Make any uuids unique
         if apply_extras.get('tags'):
@@ -849,14 +847,4 @@ class ChangeDetectionStore:
             if not watch.get('date_created'):
                 self.data['watching'][uuid]['date_created'] = i
             i+=1
-        return
-
-    # #1774 - protect xpath1 against migration
-    def update_14(self):
-        for awatch in self.__data["watching"]:
-            if self.__data["watching"][awatch]['include_filters']:
-                for num, selector in enumerate(self.__data["watching"][awatch]['include_filters']):
-                    if selector.startswith('/'):
-                        self.__data["watching"][awatch]['include_filters'][num] = 'xpath1:' + selector
-                    if selector.startswith('xpath:'):
-                        self.__data["watching"][awatch]['include_filters'][num] = selector.replace('xpath:', 'xpath1:', 1)
+        return

changedetectionio/templates/_common_fields.jinja

@@ -13,10 +13,10 @@
                             <div class="pure-form-message-inline">
                               <ul>
                                 <li>Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) </code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
-                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">more help here</a></li>
+                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
                                   <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
                             </div>

changedetectionio/templates/_helpers.jinja

@@ -39,24 +39,6 @@
 {% endmacro %}
 
 
-{% macro render_nolabel_field(field) %}
-    <span>
-    {{ field(**kwargs)|safe }}
-        {% if field.errors %}
-            <span class="error">
-      {% if field.errors %}
-          <ul class=errors>
-        {% for error in field.errors %}
-            <li>{{ error }}</li>
-        {% endfor %}
-        </ul>
-      {% endif %}
-      </span>
-        {% endif %}
-    </span>
-{% endmacro %}
-
-
 {% macro render_button(field) %}
   {{ field(**kwargs)|safe }}
 {% endmacro %}

changedetectionio/templates/base.html

@@ -116,7 +116,7 @@
                    viewBox="0 0 16.9 16.1"
                    id="svg-heart"
                    xmlns="http://www.w3.org/2000/svg"
-                   >
+                   xmlns:svg="http://www.w3.org/2000/svg">
                   <path id="heartpath" d="M 5.338316,0.50302766 C 0.71136983,0.50647126 -3.9576371,7.2707777 8.5004254,15.503028 23.833425,5.3700277 13.220206,-2.5384409 8.6762066,1.6475589 c -0.060791,0.054322 -0.11943,0.1110064 -0.1757812,0.1699219 -0.057,-0.059 -0.1157813,-0.116875 -0.1757812,-0.171875 C 7.4724566,0.86129334 6.4060729,0.50223298 5.338316,0.50302766 Z"
                      style="fill:var(--color-background);fill-opacity:1;stroke:#ff0000;stroke-opacity:1" />
                 </svg>
@@ -170,6 +170,7 @@
                     And tell your friends and colleagues :)
                 </li>
                 </ul>
+                </p>
                 <p>
                     The more popular changedetection.io is, the more time we can dedicate to adding amazing features!
                 </p>

changedetectionio/templates/edit.html

@@ -3,7 +3,6 @@
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
 <script>
     const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
@@ -20,7 +19,7 @@
     const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
     const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
     const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
-    const default_system_fetch_backend="{{ settings_application['fetch_backend'] }}";
+
 </script>
 
 <script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
@@ -125,9 +124,10 @@
                         </span>
                     </div>
                 {% endif %}
-
-                <!-- webdriver always -->
-                <fieldset data-visible-for="fetch_backend=html_webdriver"  style="display: none;">
+                    <div  class="pure-control-group inline-radio">
+                        {{ render_checkbox_field(form.ignore_status_codes) }}
+                    </div>
+                <fieldset id="webdriver-override-options">
                     <div class="pure-control-group">
                         {{ render_field(form.webdriver_delay) }}
                         <div class="pure-form-message-inline">
@@ -140,40 +140,23 @@
                         </div>
                     </div>
                     <div class="pure-control-group">
-                        <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
-                    </div>
-                    <div class="advanced-options"  style="display: none;">
                         {{ render_field(form.webdriver_js_execute_code) }}
                         <div class="pure-form-message-inline">
-                            Run this code before performing change detection, handy for filling in fields and other
-                            actions <a
-                                href="https://github.com/dgtlmoon/changedetection.io/wiki/Run-JavaScript-before-change-detection">More
-                            help and examples here</a>
+                            Run this code before performing change detection, handy for filling in fields and other actions <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Run-JavaScript-before-change-detection">More help and examples here</a>
                         </div>
                     </div>
                 </fieldset>
-                <!-- html requests always -->
-                <fieldset data-visible-for="fetch_backend=html_requests">
-                    <div class="pure-control-group">
-                        <a class="pure-button button-secondary button-xsmall show-advanced">Show advanced options</a>
-                    </div>
-                    <div class="advanced-options"  style="display: none;">
-                        <div class="pure-control-group" id="request-method">
-                            {{ render_field(form.method) }}
-                        </div>
-                        <div id="request-body">
-                                            {{ render_field(form.body, rows=5, placeholder="Example
-{
-   \"name\":\"John\",
-   \"age\":30,
-   \"car\":null
-}") }}
+                <fieldset class="pure-group" id="requests-override-options">
+                    {% if not playwright_enabled %}
+                        <div class="pure-form-message-inline">
+                            <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
                         </div>
+                    {% endif %}
+                    <div class="pure-control-group" id="request-method">
+                        {{ render_field(form.method) }}
                     </div>
-                </fieldset>
-            <!-- hmm -->
-                <div class="pure-control-group advanced-options"  style="display: none;">
-                    {{ render_field(form.headers, rows=5, placeholder="Example
+                    <div class="pure-control-group" id="request-headers">
+{{ render_field(form.headers, rows=5, placeholder="Example
 Cookie: foobar
 User-Agent: wonderbra 1.0") }}
 
@@ -186,12 +169,17 @@ User-Agent: wonderbra 1.0") }}
                             <br>
                             (Not supported by Selenium browser)
                         </div>
+
                     </div>
-            <fieldset data-visible-for="fetch_backend=html_requests fetch_backend=html_webdriver" >
-                    <div class="pure-control-group inline-radio advanced-options"  style="display: none;">
-                    {{ render_checkbox_field(form.ignore_status_codes) }}
+                    <div class="pure-control-group" id="request-body">
+                                        {{ render_field(form.body, rows=5, placeholder="Example
+{
+   \"name\":\"John\",
+   \"age\":30,
+   \"car\":null
+}") }}
                     </div>
-            </fieldset>
+                </fieldset>
             </div>
             {% if playwright_enabled %}
             <div class="tab-pane-inner" id="browser-steps">
@@ -302,12 +290,11 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                                 {% endif %}
                             </ul>
                         </li>
-                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash. To specify XPath to be used explicitly or the XPath rule starts with an XPath function: Prefix with <code>xpath:</code>
+                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
                             <ul>
-                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath:count(//*[contains(@class, 'sametext')])</code>, <a
+                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
                                 href="http://xpather.com/" target="new">test your XPath here</a></li>
                                 <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
-                                <li>To use XPath1.0: Prefix with <code>xpath1:</code></li>
                             </ul>
                             </li>
                     </ul>

changedetectionio/templates/settings.html

@@ -11,7 +11,7 @@
 </script>
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
+
 <script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
 <div class="edit-form">
     <div class="tabs collapsable">
@@ -111,7 +111,7 @@
                     <br>
                     Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
                 </div>
-                <fieldset class="pure-group" id="webdriver-override-options" data-visible-for="application-fetch_backend=html_webdriver">
+                <fieldset class="pure-group" id="webdriver-override-options">
                     <div class="pure-form-message-inline">
                         <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
                         <br>
@@ -178,9 +178,6 @@ nav
                         <span style="display:none;" id="api-key-copy" >copy</span>
                     </div>
                 </div>
-                <div class="pure-control-group">
-                    <a href="{{url_for('settings_reset_api_key')}}" class="pure-button button-small button-cancel">Regenerate API key</a>
-                </div>
             </div>
             <div class="tab-pane-inner" id="proxies">
                 <div id="recommended-proxy">
@@ -236,11 +233,8 @@ nav
                 <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
                 </div>
                 <div class="pure-control-group" id="extra-browsers-setting">
-                    <p>
-                    <span class="pure-form-message-inline"><i>Extra Browsers</i> can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.</span><br>
-                    <span class="pure-form-message-inline">Simply paste the connection address into the box, <a href="https://changedetection.io/tutorial/using-bright-datas-scraping-browser-pass-captchas-and-other-protection-when-monitoring">More instructions and examples here</a> </span>
-                    </p>
-                    {{ render_field(form.requests.form.extra_browsers) }}
+                    <span class="pure-form-message-inline"><i>Extra Browsers</i> allow changedetection.io to communicate with a different web-browser.</span><br>
+                  {{ render_field(form.requests.form.extra_browsers) }}
                 </div>
             </div>
             <div id="actions">

changedetectionio/templates/svgs/search-icon.svg

@@ -1 +1 @@
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>
+<?xml version="1.0" encoding="utf-8"?><svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>

changedetectionio/templates/watch-overview.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.jinja' import render_simple_field, render_field, render_nolabel_field %}
+{% from '_helpers.jinja' import render_simple_field, render_field %}
 <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 <script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 
@@ -11,14 +11,17 @@
         <fieldset>
             <legend>Add a new change detection watch</legend>
             <div id="watch-add-wrapper-zone">
-
-                    {{ render_nolabel_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_nolabel_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
-                    {{ render_nolabel_field(form.watch_submit_button, title="Watch this URL!" ) }}
-                    {{ render_nolabel_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
+                <div>
+                    {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
+                    {{ render_simple_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
+                </div>
+                <div>
+                    {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
+                    {{ render_simple_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
+                </div>
             </div>
             <div id="quick-watch-processor-type">
-                {{ render_simple_field(form.processor) }}
+                {{ render_simple_field(form.processor, title="Edit first then Watch") }}
             </div>
 
         </fieldset>
@@ -79,15 +82,12 @@
             </tr>
             {% endif %}
             {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}
-
-                {% set is_unviewed =  watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}
-
             <tr id="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} processor-{{ watch['processor'] }}
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
                 {% if watch.last_notification_error is defined and watch.last_notification_error != False %}error{% endif %}
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
-                {% if is_unviewed %}unviewed{% endif %}
+                {% if watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}unviewed{% endif %}
                 {% if watch.uuid in queued_uuids %}queued{% endif %}">
                 <td class="inline checkbox-uuid" ><input name="uuids"  type="checkbox" value="{{ watch.uuid}} " > <span>{{ loop.index+pagination.skip }}</span></td>
                 <td class="inline watch-controls">
@@ -141,7 +141,7 @@
                     {% if watch['processor'] == 'restock_diff'  %}
                     <span class="restock-label {{'in-stock' if watch['in_stock'] else 'not-in-stock' }}" title="detecting restock conditions">
                         <!-- maybe some object watch['processor'][restock_diff] or.. -->
-                        {% if watch['last_checked'] and watch['in_stock'] != None %}
+                        {% if watch['last_checked'] %}
                             {% if watch['in_stock'] %} In stock {% else %} Not in stock {% endif %}
                         {% else %}
                             Not yet checked
@@ -167,13 +167,7 @@
                        class="recheck pure-button pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
                     <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button pure-button-primary">Edit</a>
                     {% if watch.history_n >= 2 %}
-
-                        {%  if is_unviewed %}
-                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid, from_version=watch.get_next_snapshot_key_to_last_viewed) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">Diff</a>
-                        {% else %}
-                           <a href="{{ url_for('diff_history_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">Diff</a>
-                        {% endif %}
-
+                    <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">Diff</a>
                     {% else %}
                         {% if watch.history_n == 1 or (watch.history_n ==0 and watch.error_text_ctime )%}
                             <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary">Preview</a>

changedetectionio/tests/conftest.py

@@ -13,17 +13,22 @@ global app
 
 
 def cleanup(datastore_path):
-    import glob
     # Unlink test output files
-
-    for g in ["*.txt", "*.json", "*.pdf"]:
-        files = glob.glob(os.path.join(datastore_path, g))
-        for f in files:
-            if 'proxies.json' in f:
-                # Usually mounted by docker container during test time
-                continue
-            if os.path.isfile(f):
-                os.unlink(f)
+    files = [
+        'count.txt',
+        'endpoint-content.txt'
+        'headers.txt',
+        'headers-testtag.txt',
+        'notification.txt',
+        'secret.txt',
+        'url-watches.json',
+        'output.txt',
+    ]
+    for file in files:
+        try:
+            os.unlink("{}/{}".format(datastore_path, file))
+        except FileNotFoundError:
+            pass
 
 @pytest.fixture(scope='session')
 def app(request):

changedetectionio/tests/test_api.py

@@ -96,9 +96,7 @@ def test_api_simple(client, live_server):
     )
     assert watch_uuid in res.json.keys()
     before_recheck_info = res.json[watch_uuid]
-
     assert before_recheck_info['last_checked'] != 0
-
     #705 `last_changed` should be zero on the first check
     assert before_recheck_info['last_changed'] == 0
     assert before_recheck_info['title'] == 'My test URL'
@@ -159,18 +157,6 @@ def test_api_simple(client, live_server):
     # @todo how to handle None/default global values?
     assert watch['history_n'] == 2, "Found replacement history section, which is in its own API"
 
-    assert watch.get('viewed') == False
-    # Loading the most recent snapshot should force viewed to become true
-    client.get(url_for("diff_history_page", uuid="first"), follow_redirects=True)
-
-    # Fetch the whole watch again, viewed should be true
-    res = client.get(
-        url_for("watch", uuid=watch_uuid),
-        headers={'x-api-key': api_key}
-    )
-    watch = res.json
-    assert watch.get('viewed') == True
-
     # basic systeminfo check
     res = client.get(
         url_for("systeminfo"),
@@ -357,25 +343,3 @@ def test_api_watch_PUT_update(client, live_server):
     # Cleanup everything
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
-
-
-def test_api_import(client, live_server):
-    api_key = extract_api_key_from_UI(client)
-
-    res = client.post(
-        url_for("import") + "?tag=import-test",
-        data='https://website1.com\r\nhttps://website2.com',
-        headers={'x-api-key': api_key},
-        follow_redirects=True
-    )
-
-    assert res.status_code == 200
-    assert len(res.json) == 2
-    res = client.get(url_for("index"))
-    assert b"https://website1.com" in res.data
-    assert b"https://website2.com" in res.data
-
-    # Should see the new tag in the tag/groups list
-    res = client.get(url_for('tags.tags_overview_page'))
-    assert b'import-test' in res.data
-    

changedetectionio/tests/test_extract_regex.py

@@ -227,6 +227,9 @@ def test_regex_error_handling(client, live_server):
         follow_redirects=True
     )
 
+    with open('/tmp/fuck.html', 'wb') as f:
+        f.write(res.data)
+
     assert b'is not a valid regular expression.' in res.data
 
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_notification.py

@@ -281,8 +281,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
 
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
-    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}"
 
     res = client.post(
         url_for("settings_page"),
@@ -298,7 +297,10 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         follow_redirects=True
     )
     assert b'Settings updated' in res.data
-
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )
     # Add a watch and trigger a HTTP POST
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -313,9 +315,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     set_modified_response()
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(2) # plus extra delay for notifications to fire
+    time.sleep(2)
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
@@ -328,13 +328,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     with open("test-datastore/notification-url.txt", 'r') as f:
         notification_url = f.read()
         assert 'xxx=http' in notification_url
-        # apprise style headers should be stripped
-        assert 'custom-header' not in notification_url
-
-    with open("test-datastore/notification-headers.txt", 'r') as f:
-        notification_headers = f.read()
-        assert 'custom-header: 123' in notification_headers.lower()
-
 
     # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
     assert os.path.isfile("test-datastore/notification-content-type.txt")
@@ -342,8 +335,3 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         assert 'application/json' in f.read()
 
     os.unlink("test-datastore/notification-url.txt")
-
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )

changedetectionio/tests/test_pdf.py

@@ -2,8 +2,9 @@
 
 import time
 from flask import url_for
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, live_server_setup
 
+sleep_time_for_fetch_thread = 3
 
 # `subtractive_selectors` should still work in `source:` type requests
 def test_fetch_pdf(client, live_server):
@@ -21,9 +22,7 @@ def test_fetch_pdf(client, live_server):
 
     assert b"1 Imported" in res.data
 
-
-    wait_for_all_checks(client)
-
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True
@@ -34,42 +33,8 @@ def test_fetch_pdf(client, live_server):
 
     # So we know if the file changes in other ways
     import hashlib
-    original_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
+    md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
     # We should have one
-    assert len(original_md5) >0
+    assert len(md5) >0
     # And it's going to be in the document
-    assert b'Document checksum - '+bytes(str(original_md5).encode('utf-8')) in res.data
-
-
-    shutil.copy("tests/test2.pdf", "test-datastore/endpoint-test.pdf")
-    changed_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
-
-    wait_for_all_checks(client)
-
-    # Now something should be ready, indicated by having a 'unviewed' class
-    res = client.get(url_for("index"))
-    assert b'unviewed' in res.data
-
-    # The original checksum should be not be here anymore (cdio adds it to the bottom of the text)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert original_md5.encode('utf-8') not in res.data
-    assert changed_md5.encode('utf-8') in res.data
-
-
-    res = client.get(
-        url_for("diff_history_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert original_md5.encode('utf-8') in res.data
-    assert changed_md5.encode('utf-8') in res.data
-
-    assert b'here is a change' in res.data
-    
+    assert b'Document checksum - '+bytes(str(md5).encode('utf-8')) in res.data

changedetectionio/tests/test_xpath_selector.py

@@ -6,11 +6,9 @@ from .util import live_server_setup, wait_for_all_checks
 
 from ..html_tools import *
 
-
 def test_setup(live_server):
     live_server_setup(live_server)
 
-
 def set_original_response():
     test_return_data = """<html>
        <body>
@@ -28,7 +26,6 @@ def set_original_response():
         f.write(test_return_data)
     return None
 
-
 def set_modified_response():
     test_return_data = """<html>
        <body>
@@ -47,12 +44,11 @@ def set_modified_response():
 
     return None
 
-
 # Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613
 def test_check_xpath_filter_utf8(client, live_server):
-    filter = '//item/*[self::description]'
+    filter='//item/*[self::description]'
 
-    d = '''<?xml version="1.0" encoding="UTF-8"?>
+    d='''<?xml version="1.0" encoding="UTF-8"?>
 <rss xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
 	<channel>
 		<title>rpilocator.com</title>
@@ -106,9 +102,9 @@ def test_check_xpath_filter_utf8(client, live_server):
 
 # Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613
 def test_check_xpath_text_function_utf8(client, live_server):
-    filter = '//item/title/text()'
+    filter='//item/title/text()'
 
-    d = '''<?xml version="1.0" encoding="UTF-8"?>
+    d='''<?xml version="1.0" encoding="UTF-8"?>
 <rss xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
 	<channel>
 		<title>rpilocator.com</title>
@@ -167,12 +163,15 @@ def test_check_xpath_text_function_utf8(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-
 def test_check_markup_xpath_filter_restriction(client, live_server):
+
     xpath_filter = "//*[contains(@class, 'sametext')]"
 
     set_original_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -215,6 +214,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
 
 
 def test_xpath_validation(client, live_server):
+
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -235,48 +235,6 @@ def test_xpath_validation(client, live_server):
     assert b'Deleted' in res.data
 
 
-def test_xpath23_prefix_validation(client, live_server):
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"include_filters": "xpath:/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"is not a valid XPath expression" in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-
-def test_xpath1_validation(client, live_server):
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"include_filters": "xpath1:/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"is not a valid XPath expression" in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-
 # actually only really used by the distll.io importer, but could be handy too
 def test_check_with_prefix_include_filters(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
@@ -296,8 +254,7 @@ def test_check_with_prefix_include_filters(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "",
-              'fetch_backend': "html_requests"},
+        data={"include_filters":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
 
@@ -309,15 +266,13 @@ def test_check_with_prefix_include_filters(client, live_server):
         follow_redirects=True
     )
 
-    assert b"Some text thats the same" in res.data  # in selector
-    assert b"Some text that will change" not in res.data  # not in selector
+    assert b"Some text thats the same" in res.data #in selector
+    assert b"Some text that will change" not in res.data #not in selector
 
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-
-
 def test_various_rules(client, live_server):
     # Just check these don't error
-    # live_server_setup(live_server)
+    #live_server_setup(live_server)
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("""<html>
        <body>
@@ -330,11 +285,10 @@ def test_various_rules(client, live_server):
      <a href=''>some linky </a>
      <a href=''>another some linky </a>
      <!-- related to https://github.com/dgtlmoon/changedetection.io/pull/1774 -->
-     <input   type="email"   id="email" />     
+     <input   type="email"   id="email" />
      </body>
      </html>
     """)
-
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("import_page"),
@@ -344,6 +298,7 @@ def test_various_rules(client, live_server):
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
 
+
     for r in ['//div', '//a', 'xpath://div', 'xpath://a']:
         res = client.post(
             url_for("edit_page", uuid="first"),
@@ -358,153 +313,3 @@ def test_various_rules(client, live_server):
         assert b"Updated watch." in res.data
         res = client.get(url_for("index"))
         assert b'fetch-error' not in res.data, f"Should not see errors after '{r} filter"
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-
-def test_xpath_20(client, live_server):
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-    set_original_response()
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"include_filters": "//*[contains(@class, 'sametext')]|//*[contains(@class, 'changetext')]",
-              "url": test_url,
-              "tags": "",
-              "headers": "",
-              'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b"Some text thats the same" in res.data  # in selector
-    assert b"Some text that will change" in res.data  # in selector
-
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-
-
-def test_xpath_20_function_count(client, live_server):
-    set_original_response()
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"include_filters": "xpath:count(//div) * 123456789987654321",
-              "url": test_url,
-              "tags": "",
-              "headers": "",
-              'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b"246913579975308642" in res.data  # in selector
-
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-
-
-def test_xpath_20_function_count2(client, live_server):
-    set_original_response()
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"include_filters": "/html/body/count(div) * 123456789987654321",
-              "url": test_url,
-              "tags": "",
-              "headers": "",
-              'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b"246913579975308642" in res.data  # in selector
-
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-
-
-def test_xpath_20_function_string_join_matches(client, live_server):
-    set_original_response()
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-            "include_filters": "xpath:string-join(//*[contains(@class, 'sametext')]|//*[matches(@class, 'changetext')], 'specialconjunction')",
-            "url": test_url,
-            "tags": "",
-            "headers": "",
-            'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b"Some text thats the samespecialconjunctionSome text that will change" in res.data  # in selector
-
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-

changedetectionio/tests/test_xpath_selector_unit.py

@@ -1,203 +0,0 @@
-import sys
-import os
-import pytest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-import html_tools
-
-# test generation guide.
-# 1. Do not include encoding in the xml declaration if the test object is a str type.
-# 2. Always paraphrase test.
-
-hotels = """
-<hotel>
-  <branch location="California">
-    <staff>
-      <given_name>Christopher</given_name>
-      <surname>Anderson</surname>
-      <age>25</age>
-    </staff>
-    <staff>
-      <given_name>Christopher</given_name>
-      <surname>Carter</surname>
-      <age>30</age>
-    </staff>
-  </branch>
-  <branch location="Las Vegas">
-    <staff>
-      <given_name>Lisa</given_name>
-      <surname>Walker</surname>
-      <age>60</age>
-    </staff>
-    <staff>
-      <given_name>Jessica</given_name>
-      <surname>Walker</surname>
-      <age>32</age>
-    </staff>
-    <staff>
-      <given_name>Jennifer</given_name>
-      <surname>Roberts</surname>
-      <age>50</age>
-    </staff>
-  </branch>
-</hotel>"""
-
-@pytest.mark.parametrize("html_content", [hotels])
-@pytest.mark.parametrize("xpath, answer", [('(//staff/given_name, //staff/age)', '25'),
-                          ("xs:date('2023-10-10')", '2023-10-10'),
-                          ("if (/hotel/branch[@location = 'California']/staff[1]/age = 25) then 'is 25' else 'is not 25'", 'is 25'),
-                          ("if (//hotel/branch[@location = 'California']/staff[1]/age = 25) then 'is 25' else 'is not 25'", 'is 25'),
-                          ("if (count(/hotel/branch/staff) = 5) then true() else false()", 'true'),
-                          ("if (count(//hotel/branch/staff) = 5) then true() else false()", 'true'),
-                          ("for $i in /hotel/branch/staff return if ($i/age >= 40) then upper-case($i/surname) else lower-case($i/surname)", 'anderson'),
-                          ("given_name  =  'Christopher' and age  =  40", 'false'),
-                          ("//given_name  =  'Christopher' and //age  =  40", 'false'),
-                          #("(staff/given_name, staff/age)", 'Lisa'),
-                          ("(//staff/given_name, //staff/age)", 'Lisa'),
-                          #("hotel/branch[@location = 'California']/staff/age union hotel/branch[@location = 'Las Vegas']/staff/age", ''),
-                          ("(//hotel/branch[@location = 'California']/staff/age union //hotel/branch[@location = 'Las Vegas']/staff/age)", '60'),
-                          ("(200 to 210)", "205"),
-                          ("(//hotel/branch[@location = 'California']/staff/age union //hotel/branch[@location = 'Las Vegas']/staff/age)", "50"),
-                          ("(1, 9, 9, 5)", "5"),
-                          ("(3, (), (14, 15), 92, 653)", "653"),
-                          ("for $i in /hotel/branch/staff return $i/given_name", "Christopher"),
-                          ("for $i in //hotel/branch/staff return $i/given_name", "Christopher"),
-                          ("distinct-values(for $i in /hotel/branch/staff return $i/given_name)", "Jessica"),
-                          ("distinct-values(for $i in //hotel/branch/staff return $i/given_name)", "Jessica"),
-                          ("for $i in (7 to  15) return $i*10", "130"),
-                          ("some $i in /hotel/branch/staff satisfies $i/age < 20", "false"),
-                          ("some $i in //hotel/branch/staff satisfies $i/age < 20", "false"),
-                          ("every $i in /hotel/branch/staff satisfies $i/age > 20", "true"),
-                          ("every $i in //hotel/branch/staff satisfies $i/age > 20 ", "true"),
-                          ("let $x := branch[@location = 'California'], $y := branch[@location = 'Las Vegas'] return (avg($x/staff/age), avg($y/staff/age))", "27.5"),
-                          ("let $x := //branch[@location = 'California'], $y := //branch[@location = 'Las Vegas'] return (avg($x/staff/age), avg($y/staff/age))", "27.5"),
-                          ("let $nu := 1, $de := 1000 return  'probability = ' || $nu div $de * 100 || '%'", "0.1%"),
-                          ("let $nu := 2, $probability := function ($argument) { 'probability = ' ||  $nu div $argument  * 100 || '%'}, $de := 5 return $probability($de)", "40%"),
-                          ("'XPATH2.0-3.1 dissemination' instance of xs:string ", "true"),
-                          ("'new stackoverflow question incoming' instance of xs:integer ", "false"),
-                          ("'50000' cast as xs:integer", "50000"),
-                          ("//branch[@location = 'California']/staff[1]/surname eq 'Anderson'", "true"),
-                          ("fn:false()", "false")])
-def test_hotels(html_content, xpath, answer):
-    html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True)
-    assert type(html_content) == str
-    assert answer in html_content
-
-
-
-branches_to_visit = """<?xml version="1.0" ?>
-  <branches_to_visit>
-     <manager name="Godot" room_no="501">
-         <branch>Area 51</branch>
-         <branch>A place with no name</branch>
-         <branch>Stalsk12</branch>
-     </manager>
-      <manager name="Freya" room_no="305">
-         <branch>Stalsk12</branch>
-         <branch>Barcelona</branch>
-         <branch>Paris</branch>
-     </manager>
- </branches_to_visit>"""
-@pytest.mark.parametrize("html_content", [branches_to_visit])
-@pytest.mark.parametrize("xpath, answer", [
-    ("manager[@name = 'Godot']/branch union manager[@name = 'Freya']/branch", "Area 51"),
-    ("//manager[@name = 'Godot']/branch union //manager[@name = 'Freya']/branch", "Stalsk12"),
-    ("manager[@name = 'Godot']/branch | manager[@name = 'Freya']/branch", "Stalsk12"),
-    ("//manager[@name = 'Godot']/branch | //manager[@name = 'Freya']/branch", "Stalsk12"),
-    ("manager/branch intersect manager[@name = 'Godot']/branch", "A place with no name"),
-    ("//manager/branch intersect //manager[@name = 'Godot']/branch", "A place with no name"),
-    ("manager[@name = 'Godot']/branch intersect manager[@name = 'Freya']/branch", ""),
-    ("manager/branch except manager[@name = 'Godot']/branch", "Barcelona"),
-    ("manager[@name = 'Godot']/branch[1]  eq 'Area 51'", "true"),
-    ("//manager[@name = 'Godot']/branch[1]  eq 'Area 51'", "true"),
-    ("manager[@name = 'Godot']/branch[1]  eq 'Seoul'", "false"),
-    ("//manager[@name = 'Godot']/branch[1]  eq 'Seoul'", "false"),
-    ("manager[@name = 'Godot']/branch[2] eq manager[@name = 'Freya']/branch[2]", "false"),
-    ("//manager[@name = 'Godot']/branch[2] eq //manager[@name = 'Freya']/branch[2]", "false"),
-    ("manager[1]/@room_no lt manager[2]/@room_no", "false"),
-    ("//manager[1]/@room_no lt //manager[2]/@room_no", "false"),
-    ("manager[1]/@room_no gt manager[2]/@room_no", "true"),
-    ("//manager[1]/@room_no gt //manager[2]/@room_no", "true"),
-    ("manager[@name = 'Godot']/branch[1]  = 'Area 51'", "true"),
-    ("//manager[@name = 'Godot']/branch[1]  = 'Area 51'", "true"),
-    ("manager[@name = 'Godot']/branch[1]  = 'Seoul'", "false"),
-    ("//manager[@name = 'Godot']/branch[1]  = 'Seoul'", "false"),
-    ("manager[@name = 'Godot']/branch  = 'Area 51'", "true"),
-    ("//manager[@name = 'Godot']/branch  = 'Area 51'", "true"),
-    ("manager[@name = 'Godot']/branch  = 'Barcelona'", "false"),
-    ("//manager[@name = 'Godot']/branch  = 'Barcelona'", "false"),
-    ("manager[1]/@room_no > manager[2]/@room_no", "true"),
-    ("//manager[1]/@room_no > //manager[2]/@room_no", "true"),
-    ("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[1]/branch[1]", "false"),
-    ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[1]/branch[1]", "false"),
-    ("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[1]/branch[3]", "true"),
-    ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[1]/branch[3]", "true"),
-    ("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] <<  manager[1]/branch[1]", "false"),
-    ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] <<  //manager[1]/branch[1]", "false"),
-    ("manager[@name = 'Godot']/branch[ . = 'Stalsk12']  >>  manager[1]/branch[1]", "true"),
-    ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] >>  //manager[1]/branch[1]", "true"),
-    ("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[@name = 'Freya']/branch[ . = 'Stalsk12']", "false"),
-    ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[@name = 'Freya']/branch[ . = 'Stalsk12']", "false"),
-    ("manager[1]/@name || manager[2]/@name", "GodotFreya"),
-    ("//manager[1]/@name || //manager[2]/@name", "GodotFreya"),
-                          ])
-def test_branches_to_visit(html_content, xpath, answer):
-    html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True)
-    assert type(html_content) == str
-    assert answer in html_content
-
-trips = """
-<trips>
-   <trip reservation_number="10">
-       <depart>2023-10-06</depart>
-       <arrive>2023-10-10</arrive>
-       <traveler name="Christopher Anderson">
-           <duration>4</duration>
-           <price>2000.00</price>
-       </traveler>
-   </trip>
-   <trip reservation_number="12">
-       <depart>2023-10-06</depart>
-       <arrive>2023-10-12</arrive>
-       <traveler name="Frank Carter">
-           <duration>6</duration>
-           <price>3500.34</price>
-       </traveler>
-   </trip>
-</trips>"""
-@pytest.mark.parametrize("html_content", [trips])
-@pytest.mark.parametrize("xpath, answer", [
-    ("1 + 9 * 9 + 5 div 5", "83"),
-    ("(1 + 9 * 9 + 5) div 6", "14.5"),
-    ("23 idiv 3", "7"),
-    ("23 div 3", "7.66666666"),
-    ("for $i in ./trip return $i/traveler/duration * $i/traveler/price", "21002.04"),
-    ("for $i in ./trip return $i/traveler/duration ", "4"),
-    ("for $i in .//trip return $i/traveler/duration * $i/traveler/price", "21002.04"),
-    ("sum(for $i in ./trip return $i/traveler/duration * $i/traveler/price)", "29002.04"),
-    ("sum(for $i in .//trip return $i/traveler/duration * $i/traveler/price)", "29002.04"),
-    #("trip[1]/depart - trip[1]/arrive", "fail_to_get_answer"),
-    #("//trip[1]/depart - //trip[1]/arrive", "fail_to_get_answer"),
-    #("trip[1]/depart + trip[1]/arrive", "fail_to_get_answer"),
-    #("xs:date(trip[1]/depart) + xs:date(trip[1]/arrive)", "fail_to_get_answer"),
-    ("(//trip[1]/arrive cast as xs:date) - (//trip[1]/depart cast as xs:date)", "P4D"),
-    ("(//trip[1]/depart cast as xs:date) - (//trip[1]/arrive cast as xs:date)", "-P4D"),
-    ("(//trip[1]/depart cast as xs:date) + xs:dayTimeDuration('P3D')", "2023-10-09"),
-    ("(//trip[1]/depart cast as xs:date) - xs:dayTimeDuration('P3D')", "2023-10-03"),
-    ("(456, 623) instance of xs:integer", "false"),
-    ("(456, 623) instance of xs:integer*", "true"),
-    ("/trips/trip instance of element()", "false"),
-    ("/trips/trip instance of element()*", "true"),
-    ("/trips/trip[1]/arrive instance of xs:date", "false"),
-    ("date(/trips/trip[1]/arrive) instance of xs:date", "true"),
-    ("'8' cast as xs:integer", "8"),
-    ("'11.1E3' cast as xs:double", "11100"),
-    ("6.5 cast as xs:integer", "6"),
-    #("/trips/trip[1]/arrive cast as xs:dateTime", "fail_to_get_answer"),
-    ("/trips/trip[1]/arrive cast as xs:date", "2023-10-10"),
-    ("('2023-10-12') cast as xs:date", "2023-10-12"),
-    ("for $i in //trip return concat($i/depart, '  ', $i/arrive)", "2023-10-06  2023-10-10"),
-                          ])
-def test_trips(html_content, xpath, answer):
-    html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True)
-    assert type(html_content) == str
-    assert answer in html_content

changedetectionio/tests/unit/test_watch_model.py

@@ -1,54 +0,0 @@
-#!/usr/bin/python3
-
-# run from dir above changedetectionio/ dir
-# python3 -m unittest changedetectionio.tests.unit.test_notification_diff
-
-import unittest
-import os
-
-from changedetectionio.model import Watch
-
-# mostly
-class TestDiffBuilder(unittest.TestCase):
-
-    def test_watch_get_suggested_from_diff_timestamp(self):
-        import uuid as uuid_builder
-        watch = Watch.model(datastore_path='/tmp', default={})
-        watch.ensure_data_dir_exists()
-
-        watch['last_viewed'] = 110
-
-        watch.save_history_text(contents=b"hello world", timestamp=100, snapshot_id=str(uuid_builder.uuid4()))
-        watch.save_history_text(contents=b"hello world", timestamp=105, snapshot_id=str(uuid_builder.uuid4()))
-        watch.save_history_text(contents=b"hello world", timestamp=109, snapshot_id=str(uuid_builder.uuid4()))
-        watch.save_history_text(contents=b"hello world", timestamp=112, snapshot_id=str(uuid_builder.uuid4()))
-        watch.save_history_text(contents=b"hello world", timestamp=115, snapshot_id=str(uuid_builder.uuid4()))
-        watch.save_history_text(contents=b"hello world", timestamp=117, snapshot_id=str(uuid_builder.uuid4()))
-
-        p = watch.get_next_snapshot_key_to_last_viewed
-        assert p == "112", "Correct last-viewed timestamp was detected"
-
-        # When there is only one step of difference from the end of the list, it should return second-last change
-        watch['last_viewed'] = 116
-        p = watch.get_next_snapshot_key_to_last_viewed
-        assert p == "115", "Correct 'second last' last-viewed timestamp was detected when using the last timestamp"
-
-        watch['last_viewed'] = 99
-        p = watch.get_next_snapshot_key_to_last_viewed
-        assert p == "100"
-
-        watch['last_viewed'] = 200
-        p = watch.get_next_snapshot_key_to_last_viewed
-        assert p == "115", "When the 'last viewed' timestamp is greater than the newest snapshot, return second last "
-
-        watch['last_viewed'] = 109
-        p = watch.get_next_snapshot_key_to_last_viewed
-        assert p == "109", "Correct when its the same time"
-
-        # new empty one
-        watch = Watch.model(datastore_path='/tmp', default={})
-        p = watch.get_next_snapshot_key_to_last_viewed
-        assert p == None, "None when no history available"
-
-if __name__ == '__main__':
-    unittest.main()

changedetectionio/tests/util.py

@@ -205,9 +205,6 @@ def live_server_setup(live_server):
         with open("test-datastore/notification-url.txt", "w") as f:
             f.write(request.url)
 
-        with open("test-datastore/notification-headers.txt", "w") as f:
-            f.write(str(request.headers))
-
         if request.content_type:
             with open("test-datastore/notification-content-type.txt", "w") as f:
                 f.write(request.content_type)

docker-compose.yml

@@ -90,7 +90,7 @@ services:
 #
 
      # Used for fetching pages via Playwright+Chrome where you need Javascript support.
-     # Note: works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector) and other issues
+     # Note: works well but is deprecated, doesnt fetch full page screenshots and other issues
 #    browser-chrome:
 #        hostname: browser-chrome
 #        image: selenium/standalone-chrome:4

docs/api_v1/index.html

@@ -5,13 +5,13 @@
   <meta name="description" content="Manage your changedetection.io watches via API, requires the `x-api-key` header which is found in the settings UI.">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-  <link href="assets/bootstrap.min.css?v=1701595483622" rel="stylesheet" media="screen">
-  <link href="assets/prism.css?v=1701595483622" rel="stylesheet" />
-  <link href="assets/main.css?v=1701595483622" rel="stylesheet" media="screen, print">
-  <link href="assets/favicon.ico?v=1701595483622" rel="icon" type="image/x-icon">
-  <link href="assets/apple-touch-icon.png?v=1701595483622" rel="apple-touch-icon" sizes="180x180">
-  <link href="assets/favicon-32x32.png?v=1701595483622" rel="icon" type="image/png" sizes="32x32">
-  <link href="assets/favicon-16x16.png?v=1701595483622" rel="icon" type="image/png" sizes="16x16">
+  <link href="assets/bootstrap.min.css?v=1677105736053" rel="stylesheet" media="screen">
+  <link href="assets/prism.css?v=1677105736053" rel="stylesheet" />
+  <link href="assets/main.css?v=1677105736053" rel="stylesheet" media="screen, print">
+  <link href="assets/favicon.ico?v=1677105736053" rel="icon" type="image/x-icon">
+  <link href="assets/apple-touch-icon.png?v=1677105736053" rel="apple-touch-icon" sizes="180x180">
+  <link href="assets/favicon-32x32.png?v=1677105736053" rel="icon" type="image/png" sizes="32x32">
+  <link href="assets/favicon-16x16.png?v=1677105736053" rel="icon" type="image/png" sizes="16x16">
 </head>
 
 <body class="container-fluid">
@@ -928,6 +928,6 @@
   </div>
 </div>
 
-<script src="assets/main.bundle.js?v=1701595483622"></script>
+<script src="assets/main.bundle.js?v=1677105736053"></script>
 </body>
 </html>

requirements.txt

@@ -46,9 +46,6 @@ beautifulsoup4
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 lxml
 
-# XPath 2.0-3.1 support
-elementpath
-
 selenium~=4.14.0
 
 werkzeug~=3.0

setup.py

@@ -32,7 +32,7 @@ setup(
     long_description_content_type='text/markdown',
     keywords='website change monitor for changes notification change detection '
              'alerts tracking website tracker change alert website and monitoring',
-    entry_points={"console_scripts": ["changedetection.io=changedetectionio:main"]},
+    entry_points={"console_scripts": ["changedetection.io=changedetectionio.changedetection:main"]},
     zip_safe=True,
     scripts=["changedetection.py"],
     author='dgtlmoon',