Sync with https://github.com/linuxserver/docker-changedetection.io/blob/main/Dockerfile

.github/test/Dockerfile-alpine

@@ -8,27 +8,26 @@ ENV PYTHONUNBUFFERED=1
 COPY requirements.txt /requirements.txt
 
 RUN \
- apk add --update --no-cache --virtual=build-dependencies \
+  apk add --update --no-cache --virtual=build-dependencies \
-    build-base \
     cargo \
-    git \
+    g++ \
+    gcc \
     jpeg-dev \
     libc-dev \
     libffi-dev \
+    libjpeg \
     libxslt-dev \
+    make \
     openssl-dev \
+    py3-wheel \
     python3-dev \
-    zip \
     zlib-dev && \
   apk add --update --no-cache \
-    libjpeg \
     libxslt \
-    nodejs \
+    python3 \
-    poppler-utils \
+    py3-pip && \
-    python3 && \
   echo "**** pip3 install test of changedetection.io ****" && \
-  python3 -m venv /lsiopy  && \
+  pip3 install -U pip wheel setuptools && \
-  pip install -U pip wheel setuptools && \
+  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.21/ -r /requirements.txt && \
-  pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.21/ -r /requirements.txt && \
   apk del --purge \
     build-dependencies

.github/workflows/containers.yml

@@ -103,19 +103,6 @@ jobs:
 #          provenance: false
 
       # A new tagged release is required, which builds :tag and :latest
-      - name: Docker meta :tag
-        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
-        uses: docker/metadata-action@v5
-        id: meta
-        with:
-            images: |
-                ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
-                ghcr.io/dgtlmoon/changedetection.io
-            tags: |
-                type=semver,pattern={{version}}
-                type=semver,pattern={{major}}.{{minor}}
-                type=semver,pattern={{major}}
-
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
@@ -124,7 +111,11 @@ jobs:
           context: ./
           file: ./Dockerfile
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
+            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
+            ghcr.io/dgtlmoon/changedetection.io:latest
           platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/pypi-release.yml

@@ -45,6 +45,7 @@ jobs:
     - name: Test that the basic pip built package runs without error
       run: |
         set -ex
+        sudo pip3 install --upgrade pip 
         pip3 install dist/changedetection.io*.whl
         changedetection.io -d /tmp -p 10000 &
         sleep 3

.github/workflows/test-stack-reusable-workflow.yml

@@ -64,16 +64,14 @@ jobs:
           echo "Running processes in docker..."
           docker ps
 
-      - name: Run Unit Tests
+      - name: Test built container with Pytest (generally as requests/plaintext fetching)
         run: |
           # Unit tests
+          echo "run test with unittest"
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
+          
-
-      - name: Test built container with Pytest (generally as requests/plaintext fetching)
-        run: |
           # All tests
           echo "run test with pytest"
           # The default pytest logger_level is TRACE

README.md

@@ -120,7 +120,7 @@ Easily add the current web page to your changedetection.io tool, simply install
 
 [<img src="./docs/chrome-extension-screenshot.png" style="max-width:80%;" alt="Chrome Extension to easily add the current web-page to detect a change."  title="Chrome Extension to easily add the current web-page to detect a change."  />](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
 
-[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) ( Or check out the [GitHub repo](https://github.com/dgtlmoon/changedetection.io-browser-extension) ) 
+[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)
 
 ## Installation
 

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.49.1'
+__version__ = '0.48.05'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -24,9 +24,6 @@ from loguru import logger
 app = None
 datastore = None
 
-def get_version():
-    return __version__
-
 # Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown
 def sigshutdown_handler(_signo, _stack_frame):
     global app

changedetectionio/api/api_schema.py

@@ -112,35 +112,6 @@ def build_watch_json_schema(d):
 
     schema['properties']['time_between_check'] = build_time_between_check_json_schema()
 
-    schema['properties']['browser_steps'] = {
-        "anyOf": [
-            {
-                "type": "array",
-                "items": {
-                    "type": "object",
-                    "properties": {
-                        "operation": {
-                            "type": ["string", "null"],
-                            "maxLength": 5000  # Allows null and any string up to 5000 chars (including "")
-                        },
-                        "selector": {
-                            "type": ["string", "null"],
-                            "maxLength": 5000
-                        },
-                        "optional_value": {
-                            "type": ["string", "null"],
-                            "maxLength": 5000
-                        }
-                    },
-                    "required": ["operation", "selector", "optional_value"],
-                    "additionalProperties": False  # No extra keys allowed
-                }
-            },
-            {"type": "null"},  # Allows null for `browser_steps`
-            {"type": "array", "maxItems": 0}  # Allows empty array []
-        ]
-    }
-
     # headers ?
     return schema
 

changedetectionio/api/api_v1.py

@@ -76,7 +76,6 @@ class Watch(Resource):
         # Return without history, get that via another API call
         # Properties are not returned as a JSON, so add the required props manually
         watch['history_n'] = watch.history_n
-        # attr .last_changed will check for the last written text snapshot on change
         watch['last_changed'] = watch.last_changed
         watch['viewed'] = watch.viewed
         return watch

changedetectionio/apprise_asset.py

@@ -1,4 +1,3 @@
-from changedetectionio import apprise_plugin
 import apprise
 
 # Create our AppriseAsset and populate it with some of our new values:

changedetectionio/apprise_plugin/__init__.py

@@ -1,8 +1,6 @@
 # include the decorator
 from apprise.decorators import notify
 from loguru import logger
-from requests.structures import CaseInsensitiveDict
-
 
 @notify(on="delete")
 @notify(on="deletes")
@@ -15,84 +13,70 @@ from requests.structures import CaseInsensitiveDict
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
     import json
-    import re
-
     from urllib.parse import unquote_plus
-    from apprise.utils.parse import parse_url as apprise_parse_url
+    from apprise.utils import parse_url as apprise_parse_url
+    from apprise import URLBase
 
     url = kwargs['meta'].get('url')
-    schema = kwargs['meta'].get('schema').lower().strip()
 
-    # Choose POST, GET etc from requests
+    if url.startswith('post'):
-    method =  re.sub(rf's$', '', schema)
+        r = requests.post
-    requests_method = getattr(requests, method)
+    elif url.startswith('get'):
+        r = requests.get
+    elif url.startswith('put'):
+        r = requests.put
+    elif url.startswith('delete'):
+        r = requests.delete
 
-    params = CaseInsensitiveDict({}) # Added to requests
+    url = url.replace('post://', 'http://')
+    url = url.replace('posts://', 'https://')
+    url = url.replace('put://', 'http://')
+    url = url.replace('puts://', 'https://')
+    url = url.replace('get://', 'http://')
+    url = url.replace('gets://', 'https://')
+    url = url.replace('put://', 'http://')
+    url = url.replace('puts://', 'https://')
+    url = url.replace('delete://', 'http://')
+    url = url.replace('deletes://', 'https://')
+
+    headers = {}
+    params = {}
     auth = None
-    has_error = False
 
     # Convert /foobar?+some-header=hello to proper header dictionary
     results = apprise_parse_url(url)
+    if results:
+        # Add our headers that the user can potentially over-ride if they wish
+        # to to our returned result set and tidy entries by unquoting them
+        headers = {unquote_plus(x): unquote_plus(y)
+                   for x, y in results['qsd+'].items()}
 
-    # Add our headers that the user can potentially over-ride if they wish
+        # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-    # to to our returned result set and tidy entries by unquoting them
+        # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
-    headers = CaseInsensitiveDict({unquote_plus(x): unquote_plus(y)
+        # but here we are making straight requests, so we need todo convert this against apprise's logic
-               for x, y in results['qsd+'].items()})
+        for k, v in results['qsd'].items():
+            if not k.strip('+-') in results['qsd+'].keys():
+                params[unquote_plus(k)] = unquote_plus(v)
 
-    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+        # Determine Authentication
-    # In Apprise, it relies on prefixing each request arg with "-", because it uses say &method=update as a flag for apprise
+        auth = ''
-    # but here we are making straight requests, so we need todo convert this against apprise's logic
+        if results.get('user') and results.get('password'):
-    for k, v in results['qsd'].items():
+            auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
-        if not k.strip('+-') in results['qsd+'].keys():
+        elif results.get('user'):
-            params[unquote_plus(k)] = unquote_plus(v)
+            auth = (unquote_plus(results.get('user')))
 
-    # Determine Authentication
+    # Try to auto-guess if it's JSON
-    auth = ''
+    h = 'application/json; charset=utf-8'
-    if results.get('user') and results.get('password'):
-        auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
-    elif results.get('user'):
-        auth = (unquote_plus(results.get('user')))
-
-    # If it smells like it could be JSON and no content-type was already set, offer a default content type.
-    if body and '{' in body[:100] and not headers.get('Content-Type'):
-        json_header = 'application/json; charset=utf-8'
-        try:
-            # Try if it's JSON
-            json.loads(body)
-            headers['Content-Type'] = json_header
-        except ValueError as e:
-            logger.warning(f"Could not automatically add '{json_header}' header to the notification because the document failed to parse as JSON: {e}")
-            pass
-
-    # POSTS -> HTTPS etc
-    if schema.lower().endswith('s'):
-        url = re.sub(rf'^{schema}', 'https', results.get('url'))
-    else:
-        url = re.sub(rf'^{schema}', 'http', results.get('url'))
-
-    status_str = ''
     try:
-        r = requests_method(url,
+        json.loads(body)
-          auth=auth,
+        headers['Content-Type'] = h
-          data=body.encode('utf-8') if type(body) is str else body,
+    except ValueError as e:
-          headers=headers,
+        logger.warning(f"Could not automatically add '{h}' header to the {kwargs['meta'].get('schema')}:// notification because the document failed to parse as JSON: {e}")
-          params=params
+        pass
-        )
 
-        if not (200 <= r.status_code < 300):
+    r(results.get('url'),
-            status_str = f"Error sending '{method.upper()}' request to {url} - Status: {r.status_code}: '{r.reason}'"
+      auth=auth,
-            logger.error(status_str)
+      data=body.encode('utf-8') if type(body) is str else body,
-            has_error = True
+      headers=headers,
-        else:
+      params=params
-            logger.info(f"Sent '{method.upper()}' request to {url}")
+      )
-            has_error = False
-
-    except requests.RequestException as e:
-        status_str = f"Error sending '{method.upper()}' request to {url} - {str(e)}"
-        logger.error(status_str)
-        has_error = True
-
-    if has_error:
-        raise TypeError(status_str)
-
-    return True

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -52,8 +52,6 @@ class steppable_browser_interface():
     page = None
     start_url = None
 
-    action_timeout = 10 * 1000
-
     def __init__(self, start_url):
         self.start_url = start_url
 
@@ -104,7 +102,7 @@ class steppable_browser_interface():
             return
         elem = self.page.get_by_text(value)
         if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
+            elem.first.click(delay=randint(200, 500), timeout=3000)
 
     def action_click_element_containing_text_if_exists(self, selector=None, value=''):
         logger.debug("Clicking element containing text if exists")
@@ -113,7 +111,7 @@ class steppable_browser_interface():
         elem = self.page.get_by_text(value)
         logger.debug(f"Clicking element containing text - {elem.count()} elements found")
         if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=self.action_timeout)
+            elem.first.click(delay=randint(200, 500), timeout=3000)
         else:
             return
 
@@ -121,7 +119,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
 
-        self.page.fill(selector, value, timeout=self.action_timeout)
+        self.page.fill(selector, value, timeout=10 * 1000)
 
     def action_execute_js(self, selector, value):
         response = self.page.evaluate(value)
@@ -132,7 +130,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
 
-        self.page.click(selector=selector, timeout=self.action_timeout + 20 * 1000, delay=randint(200, 500))
+        self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
 
     def action_click_element_if_exists(self, selector, value):
         import playwright._impl._errors as _api_types
@@ -140,7 +138,7 @@ class steppable_browser_interface():
         if not len(selector.strip()):
             return
         try:
-            self.page.click(selector, timeout=self.action_timeout, delay=randint(200, 500))
+            self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
         except _api_types.TimeoutError as e:
             return
         except _api_types.Error as e:
@@ -187,10 +185,10 @@ class steppable_browser_interface():
         self.page.keyboard.press("PageDown", delay=randint(200, 500))
 
     def action_check_checkbox(self, selector, value):
-        self.page.locator(selector).check(timeout=self.action_timeout)
+        self.page.locator(selector).check(timeout=1000)
 
     def action_uncheck_checkbox(self, selector, value):
-        self.page.locator(selector).uncheck(timeout=self.action_timeout)
+        self.page.locator(selector, timeout=1000).uncheck(timeout=1000)
 
 
 # Responsible for maintaining a live 'context' with the chrome CDP

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -52,7 +52,6 @@ function isItemInStock() {
         'niet leverbaar',
         'niet op voorraad',
         'no disponible',
-        'non disponibile',
         'no longer in stock',
         'no tickets available',
         'not available',

changedetectionio/flask_app.py

@@ -598,31 +598,17 @@ def changedetection_app(config=None, datastore_o=None):
 
             if 'notification_title' in request.form and request.form['notification_title'].strip():
                 n_object['notification_title'] = request.form.get('notification_title', '').strip()
-            elif datastore.data['settings']['application'].get('notification_title'):
-                n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
-            else:
-                n_object['notification_title'] = "Test title"
 
             if 'notification_body' in request.form and request.form['notification_body'].strip():
                 n_object['notification_body'] = request.form.get('notification_body', '').strip()
-            elif datastore.data['settings']['application'].get('notification_body'):
-                n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
-            else:
-                n_object['notification_body'] = "Test body"
 
-            n_object['as_async'] = False
             n_object.update(watch.extra_notification_token_values())
-            from .notification import process_notification
-            sent_obj = process_notification(n_object, datastore)
 
+            from . import update_worker
+            new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
+            new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch)
         except Exception as e:
-            e_str = str(e)
+            return make_response(f"Error: str(e)", 400)
-            # Remove this text which is not important and floods the container
-            e_str = e_str.replace(
-                "DEBUG - <class 'apprise.decorators.base.CustomNotifyPlugin.instantiate_plugin.<locals>.CustomNotifyPluginWrapper'>",
-                '')
-
-            return make_response(e_str, 400)
 
         return 'OK - Sent test notifications'
 

changedetectionio/html_tools.py

@@ -1,6 +1,5 @@
-from loguru import logger
-from lxml import etree
 from typing import List
+from lxml import etree
 import json
 import re
 
@@ -299,10 +298,8 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 # https://github.com/dgtlmoon/changedetection.io/pull/2041#issuecomment-1848397161w
     # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags
     try:
-        # .lstrip("\ufeff") strings ByteOrderMark from UTF8 and still lets the UTF work
+        stripped_text_from_html = _parse_json(json.loads(content), json_filter)
-        stripped_text_from_html = _parse_json(json.loads(content.lstrip("\ufeff") ), json_filter)
+    except json.JSONDecodeError:
-    except json.JSONDecodeError as e:
-        logger.warning(str(e))
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         # As a last resort, try to parse the whole <body>

changedetectionio/model/App.py

@@ -69,7 +69,7 @@ def parse_headers_from_text_file(filepath):
         for l in f.readlines():
             l = l.strip()
             if not l.startswith('#') and ':' in l:
-                (k, v) = l.split(':', 1)  # Split only on the first colon
+                (k, v) = l.split(':')
                 headers[k.strip()] = v.strip()
 
     return headers

changedetectionio/model/Watch.py

@@ -352,7 +352,7 @@ class model(watch_base):
     # Iterate over all history texts and see if something new exists
     # Always applying .strip() to start/end but optionally replace any other whitespace
     def lines_contain_something_unique_compared_to_history(self, lines: list, ignore_whitespace=False):
-        local_lines = set([])
+        local_lines = []
         if lines:
             if ignore_whitespace:
                 if isinstance(lines[0], str): # Can be either str or bytes depending on what was on the disk

changedetectionio/notification.py

@@ -67,10 +67,6 @@ def process_notification(n_object, datastore):
 
     sent_objs = []
     from .apprise_asset import asset
-
-    if 'as_async' in n_object:
-        asset.async_mode = n_object.get('as_async')
-
     apobj = apprise.Apprise(debug=True, asset=asset)
 
     if not n_object.get('notification_urls'):
@@ -161,6 +157,8 @@ def process_notification(n_object, datastore):
             attach=n_object.get('screenshot', None)
         )
 
+        # Give apprise time to register an error
+        time.sleep(3)
 
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()

changedetectionio/static/js/notifications.js

@@ -1,52 +1,42 @@
-$(document).ready(function () {
+$(document).ready(function() {
 
-    $('#add-email-helper').click(function (e) {
+  $('#add-email-helper').click(function (e) {
-        e.preventDefault();
+    e.preventDefault();
-        email = prompt("Destination email");
+    email = prompt("Destination email");
-        if (email) {
+    if(email) {
-            var n = $(".notification-urls");
+      var n = $(".notification-urls");
-            var p = email_notification_prefix;
+      var p=email_notification_prefix;
-            $(n).val($.trim($(n).val()) + "\n" + email_notification_prefix + email);
+      $(n).val( $.trim( $(n).val() )+"\n"+email_notification_prefix+email );
+    }
+  });
+
+  $('#send-test-notification').click(function (e) {
+    e.preventDefault();
+
+    data = {
+      notification_body: $('#notification_body').val(),
+      notification_format: $('#notification_format').val(),
+      notification_title: $('#notification_title').val(),
+      notification_urls: $('.notification-urls').val(),
+      tags: $('#tags').val(),
+      window_url: window.location.href,
+    }
+
+
+    $.ajax({
+      type: "POST",
+      url: notification_base_url,
+      data : data,
+        statusCode: {
+        400: function(data) {
+          // More than likely the CSRF token was lost when the server restarted
+          alert(data.responseText);
         }
-    });
+      }
-
+    }).done(function(data){
-    $('#send-test-notification').click(function (e) {
+      console.log(data);
-        e.preventDefault();
+      alert(data);
-
+    })
-        data = {
+  });
-            notification_body: $('#notification_body').val(),
-            notification_format: $('#notification_format').val(),
-            notification_title: $('#notification_title').val(),
-            notification_urls: $('.notification-urls').val(),
-            tags: $('#tags').val(),
-            window_url: window.location.href,
-        }
-
-        $('.notifications-wrapper .spinner').fadeIn();
-        $('#notification-test-log').show();
-        $.ajax({
-            type: "POST",
-            url: notification_base_url,
-            data: data,
-            statusCode: {
-                400: function (data) {
-                    $("#notification-test-log>span").text(data.responseText);
-                },
-            }
-        }).done(function (data) {
-            $("#notification-test-log>span").text(data);
-        }).fail(function (jqXHR, textStatus, errorThrown) {
-            // Handle connection refused or other errors
-            if (textStatus === "error" && errorThrown === "") {
-                console.error("Connection refused or server unreachable");
-                $("#notification-test-log>span").text("Error: Connection refused or server is unreachable.");
-            } else {
-                console.error("Error:", textStatus, errorThrown);
-                $("#notification-test-log>span").text("An error occurred: " + textStatus);
-            }
-        }).always(function () {
-            $('.notifications-wrapper .spinner').hide();
-        })
-    });
 });
 

changedetectionio/static/js/tabs.js

@@ -1,66 +1,48 @@
-(function ($) {
+// Rewrite this is a plugin.. is all this JS really 'worth it?'
-    $.fn.hashTabs = function (options) {
-        var settings = $.extend({
-            tabContainer: ".tabs ul",
-            tabSelector: "li a",
-            tabContent: ".tab-pane-inner",
-            activeClass: "active",
-            errorClass: ".messages .error",
-            bodyClassToggle: "full-width"
-        }, options);
 
-        var $tabs = $(settings.tabContainer).find(settings.tabSelector);
+window.addEventListener('hashchange', function () {
+    var tabs = document.getElementsByClassName('active');
+    while (tabs[0]) {
+        tabs[0].classList.remove('active');
+        document.body.classList.remove('full-width');
+    }
+    set_active_tab();
+}, false);
 
-        function setActiveTab() {
+var has_errors = document.querySelectorAll(".messages .error");
-            var hash = window.location.hash;
+if (!has_errors.length) {
-            var $activeTab = $tabs.filter("[href='" + hash + "']");
+    if (document.location.hash == "") {
+        location.replace(document.querySelector(".tabs ul li:first-child a").hash);
+    } else {
+        set_active_tab();
+    }
+} else {
+    focus_error_tab();
+}
 
-            // Remove active class from all tabs
+function set_active_tab() {
-            $(settings.tabContainer).find("li").removeClass(settings.activeClass);
+    document.body.classList.remove('full-width');
+    var tab = document.querySelectorAll("a[href='" + location.hash + "']");
+    if (tab.length) {
+        tab[0].parentElement.className = "active";
+    }
 
-            // Add active class to selected tab
+}
-            if ($activeTab.length) {
-                $activeTab.parent().addClass(settings.activeClass);
-            }
 
-            // Show the correct content
+function focus_error_tab() {
-            $(settings.tabContent).hide();
+    // time to use jquery or vuejs really,
-            if (hash) {
+    // activate the tab with the error
-                $(hash).show();
+    var tabs = document.querySelectorAll('.tabs li a'), i;
-            }
+    for (i = 0; i < tabs.length; ++i) {
+        var tab_name = tabs[i].hash.replace('#', '');
+        var pane_errors = document.querySelectorAll('#' + tab_name + ' .error')
+        if (pane_errors.length) {
+            document.location.hash = '#' + tab_name;
+            return true;
         }
-
+    }
-        function focusErrorTab() {
+    return false;
-            $tabs.each(function () {
+}
-                var tabName = this.hash.replace("#", "");
+
-                if ($("#" + tabName).find(settings.errorClass).length) {
-                    window.location.hash = "#" + tabName;
-                    return false; // Stop loop on first error tab
-                }
-            });
-        }
-
-        function initializeTabs() {
-            if ($(settings.errorClass).length) {
-                focusErrorTab();
-            } else if (!window.location.hash) {
-                window.location.replace($tabs.first().attr("href"));
-            } else {
-                setActiveTab();
-            }
-        }
-
-        // Listen for hash changes
-        $(window).on("hashchange", setActiveTab);
-
-        // Initialize on page load
-        initializeTabs();
-
-        return this; // Enable jQuery chaining
-    };
-})(jQuery);
 
 
-$(document).ready(function () {
-    $(".tabs").hashTabs();
-});

changedetectionio/static/styles/scss/styles.scss

@@ -380,15 +380,7 @@ a.pure-button-selected {
 }
 
 .notifications-wrapper {
-  padding-top: 0.5rem;
+  padding: 0.5rem 0 1rem 0;
-  #notification-test-log {
-    padding-top: 1rem;
-    white-space: pre-wrap;
-    word-break: break-word;
-    overflow-wrap: break-word;
-    max-width: 100%;
-    box-sizing: border-box;
-  }
 }
 
 label {
@@ -945,7 +937,15 @@ $form-edge-padding: 20px;
 }
 
 .tab-pane-inner {
-  display: none;
+
+  &:not(:target) {
+    display: none;
+  }
+
+  &:target {
+    display: block;
+  }
+
   // doesnt need padding because theres another row of buttons/activity
   padding: 0px;
 }

changedetectionio/static/styles/styles.css

@@ -780,14 +780,7 @@ a.pure-button-selected {
   cursor: pointer; }
 
 .notifications-wrapper {
-  padding-top: 0.5rem; }
+  padding: 0.5rem 0 1rem 0; }
-  .notifications-wrapper #notification-test-log {
-    padding-top: 1rem;
-    white-space: pre-wrap;
-    word-break: break-word;
-    overflow-wrap: break-word;
-    max-width: 100%;
-    box-sizing: border-box; }
 
 label:hover {
   cursor: pointer; }
@@ -1159,8 +1152,11 @@ textarea::placeholder {
   border-radius: 5px; }
 
 .tab-pane-inner {
-  display: none;
   padding: 0px; }
+  .tab-pane-inner:not(:target) {
+    display: none; }
+  .tab-pane-inner:target {
+    display: block; }
 
 .beta-logo {
   height: 50px;

changedetectionio/templates/_common_fields.html

@@ -12,25 +12,23 @@
                             }}
                             <div class="pure-form-message-inline">
                                 <p>
-                                <strong>Tip:</strong> Use <a target="newwindow" href="https://github.com/caronc/apprise">AppRise Notification URLs</a> for notification to just about any service! <i><a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.<br>
+                                <strong>Tip:</strong> Use <a target=_new href="https://github.com/caronc/apprise">AppRise Notification URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.<br>
 </p>
                                 <div data-target="#advanced-help-notifications" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</div>
                                 <ul style="display: none" id="advanced-help-notifications">
-                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
-                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
-                                <li><code><a target="newwindow" href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
                                 <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>) <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts">more help here</a></li>
                                   <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
                             </div>
                             <div class="notifications-wrapper">
-                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a> <div class="spinner"  style="display: none;"></div>
+                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a>
                             {% if emailprefix %}
                               <a id="add-email-helper" class="pure-button button-secondary button-xsmall" >Add email <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='email.svg')}}" alt="Add an email address"> </a>
                             {% endif %}
                               <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" >Notification debug logs</a>
-                              <br>
-                                <div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
                             </div>
                         </div>
                         <div id="notification-customisation" class="pure-control-group">
@@ -126,7 +124,7 @@
                                 <div class="pure-form-message-inline">
                                     <p>
 									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
-                                    For example, an addition or removal could be perceived as a change in some cases. <a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
+                                    For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
                                     </p>
                                     <p>
                                         For JSON payloads, use <strong>|tojson</strong> without quotes for automatic escaping, for example - <code>{ "name": {{ '{{ watch_title|tojson }}' }} }</code>

changedetectionio/templates/base.html

@@ -159,7 +159,7 @@
                     <a id="chrome-extension-link"
                        title="Try our new Chrome Extension!"
                        href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                        <img alt="Chrome store icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}">
+                        <img src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}">
                         Chrome Webstore
                     </a>
                 </p>

changedetectionio/templates/edit.html

@@ -45,8 +45,9 @@
             {% if extra_tab_content %}
             <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
             {% endif %}
+            {% if playwright_enabled %}
             <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
-        <!-- should goto extra forms? -->
+            {% endif %}
             {% if watch['processor'] == 'text_json_diff' %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
             <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
@@ -198,9 +199,8 @@ Math: {{ 1 + 1 }}") }}
                     </div>
             </fieldset>
             </div>
-
-            <div class="tab-pane-inner" id="browser-steps">
             {% if playwright_enabled %}
+            <div class="tab-pane-inner" id="browser-steps">
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
                 <fieldset>
                     <div class="pure-control-group">
@@ -234,22 +234,14 @@ Math: {{ 1 + 1 }}") }}
                                 </div>
                             </div>
                             <div id="browser-steps-fieldlist" >
-                                <span id="browser-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
+                                <span id="browser-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target=_new href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
                                 {{ render_field(form.browser_steps) }}
                             </div>
                         </div>
                     </div>
                 </fieldset>
-                {% else %}
-                    <span class="pure-form-message-inline">
-                        <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
-                        <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
-                        <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
-                        <p>You may need to <a href="https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31">Enable playwright environment variable</a> and uncomment the <strong>sockpuppetbrowser</strong> from the docker-compose.yml file.</p>
-                    </span>
-                {% endif %}
             </div>
-
+            {% endif %}
 
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
@@ -306,7 +298,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                           <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
                         {% endif %}
                         <span class="pure-form-message-inline">One CSS, xPath, JSON Path/JQ selector per line, <i>any</i> rules that matches will be used.<br>
-                        <span data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</span><br>
+<p><div data-target="#advanced-help-selectors" class="toggle-show pure-button button-tag button-xsmall">Show advanced help and tips</div><br></p>
                     <ul id="advanced-help-selectors" style="display: none;">
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
                         <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
@@ -501,7 +493,6 @@ keyword") }}
                                 <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
                                 <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
                                 <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
-                                <p>You may need to <a href="https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31">Enable playwright environment variable</a> and uncomment the <strong>sockpuppetbrowser</strong> from the docker-compose.yml file.</p>
                             </span>
                         {% endif %}
                     </div>

changedetectionio/templates/settings.html

@@ -214,7 +214,7 @@ nav
                         <a id="chrome-extension-link"
                            title="Try our new Chrome Extension!"
                            href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome">
+                            <img src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome">
                             Chrome Webstore
                         </a>
                     </p>
@@ -280,7 +280,9 @@ nav
                         
                     </div>
                 </div>
-
+                <p>
+                    Your proxy provider may need to whitelist our IP of <code>204.15.192.195</code>
+                </p>
                <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
 
                 <div class="pure-control-group" id="extra-proxies-setting">

changedetectionio/templates/watch-overview.html

@@ -108,8 +108,7 @@
                     {% else %}
                     <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
                     {% endif %}
-                    {% set mute_label = 'UnMute notification' if watch.notification_muted else 'Mute notification' %}
+                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
-                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag_uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="{{ mute_label }}" title="{{ mute_label }}" class="icon icon-mute" ></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
@@ -119,7 +118,7 @@
                          or (  watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
                          or "extra_browser_" in watch.get_fetch_backend
                     %}
-                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" alt="Using a Chrome browser" title="Using a Chrome browser" >
+                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a Chrome browser" >
                     {% endif %}
 
                     {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" >{% endif %}

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -34,7 +34,7 @@ def test_execute_custom_js(client, live_server, measure_memory_usage):
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
 
     assert b"This text should be removed" not in res.data

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -48,7 +48,7 @@ def test_noproxy_option(client, live_server, measure_memory_usage):
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.get(
         url_for("edit_page", uuid=uuid, unpause_on_save=1))
     assert b'No proxy' in res.data

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -81,7 +81,7 @@ def test_socks5(client, live_server, measure_memory_usage):
     assert "Awesome, you made it".encode('utf-8') in res.data
 
     # PROXY CHECKER WIDGET CHECK - this needs more checking
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     res = client.get(
         url_for("check_proxies.start_check", uuid=uuid),

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
-
 import os.path
-
+import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output
+from changedetectionio import html_tools
 
 
 def set_original(excluding=None, add_line=None):

changedetectionio/tests/test_api.py

@@ -44,6 +44,7 @@ def set_modified_response():
 
     return None
 
+
 def is_valid_uuid(val):
     try:
         uuid.UUID(str(val))
@@ -55,9 +56,8 @@ def is_valid_uuid(val):
 def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
-
 def test_api_simple(client, live_server, measure_memory_usage):
-#    live_server_setup(live_server)
+    #live_server_setup(live_server)
 
     api_key = extract_api_key_from_UI(client)
 
@@ -129,9 +129,6 @@ def test_api_simple(client, live_server, measure_memory_usage):
     assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']
     assert after_recheck_info['last_changed'] != 0
 
-    # #2877 When run in a slow fetcher like playwright etc
-    assert after_recheck_info['last_changed'] ==  after_recheck_info['last_checked']
-
     # Check history index list
     res = client.get(
         url_for("watchhistory", uuid=watch_uuid),

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -99,7 +99,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
     assert b'ldjson-price-track-offer' in res.data
 
     # Accept it
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     #time.sleep(1)
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
     client.get(url_for("form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_backend.py

@@ -2,6 +2,7 @@
 
 import time
 from flask import url_for
+from urllib.request import urlopen
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
     extract_UUID_from_client
 
@@ -68,7 +69,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     # Check the 'get latest snapshot works'
     res = client.get(url_for("watch_get_latest_html", uuid=uuid))

changedetectionio/tests/test_encoding.py

@@ -40,7 +40,7 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage):
 
 
     # Content type recording worked
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['content-type'] == "text/html"
 
     res = client.get(

changedetectionio/tests/test_filter_failure_notification.py

@@ -51,7 +51,7 @@ def run_filter_test(client, live_server, content_filter):
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 0, "No filter = No filter failure"
 

changedetectionio/tests/test_group.py

@@ -288,7 +288,7 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
-    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    watch_uuid = extract_UUID_from_client(client)
     res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data
@@ -315,7 +315,7 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa
     assert b'test-tag' in res.data
     assert b'another-tag' in res.data
 
-    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    watch_uuid = extract_UUID_from_client(client)
     res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
 
     assert b'Cloned' in res.data

changedetectionio/tests/test_ignore.py

@@ -36,7 +36,7 @@ def test_ignore(client, live_server, measure_memory_usage):
 
     # Give the thread time to pick it up
     wait_for_all_checks(client)
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     # use the highlighter endpoint
     res = client.post(
         url_for("highlight_submit_ignore_url", uuid=uuid),

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -514,15 +514,3 @@ def test_check_jq_ext_filter(client, live_server, measure_memory_usage):
 def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage):
     if jq_support:
         check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
-
-def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage):
-    from .. import html_tools
-
-    # JSON string with BOM and correct double-quoted keys
-    json_str = '\ufeff{"name": "José", "emoji": "😊", "language": "中文", "greeting": "Привет"}'
-
-    # See that we can find the second <script> one, which is not broken, and matches our filter
-    text = html_tools.extract_json_as_string(json_str, "json:$.name")
-    assert text == '"José"'
-
-    

changedetectionio/tests/test_live_preview.py

@@ -29,7 +29,7 @@ def test_content_filter_live_preview(client, live_server, measure_memory_usage):
         data={"url": test_url, "tags": ''},
         follow_redirects=True
     )
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(
         url_for("edit_page", uuid=uuid),
         data={

changedetectionio/tests/test_nonrenderable_pages.py

@@ -48,7 +48,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     #####################
     client.post(
         url_for("settings_page"),
-        data={"application-empty_pages_are_a_change": "", # default, OFF, they are NOT a change
+        data={"application-empty_pages_are_a_change": "",
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
@@ -66,14 +66,6 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
-
-    assert watch.last_changed == 0
-    assert watch['last_checked'] != 0
-
-
-
 
     # ok now do the opposite
 
@@ -100,10 +92,6 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # A totally zero byte (#2528) response should also not trigger an error
     set_zero_byte_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
-    # 2877
-    assert watch.last_changed == watch['last_checked']
-
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data # A change should have registered because empty_pages_are_a_change is ON

changedetectionio/tests/test_notification.py

@@ -6,7 +6,7 @@ from flask import url_for
 from loguru import logger
 
 from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
-    set_longer_modified_response, get_index
+    set_longer_modified_response
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -29,7 +29,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     # Re 360 - new install should have defaults set
     res = client.get(url_for("settings_page"))
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')+"?status_code=204"
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
 
     assert default_notification_body.encode() in res.data
     assert default_notification_title.encode() in res.data
@@ -76,7 +76,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
     testimage_png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='
 
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     datastore = 'test-datastore'
     with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f:
         f.write(base64.b64decode(testimage_png))
@@ -135,14 +135,7 @@ def test_check_notification(client, live_server, measure_memory_usage):
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
     time.sleep(3)
-
-    # Check no errors were recorded
-    res = client.get(url_for("index"))
-    assert b'notification-error' not in res.data
-
-
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
@@ -291,7 +284,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
 
     res = client.post(
         url_for("settings_page"),
@@ -326,11 +319,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
 
     time.sleep(2) # plus extra delay for notifications to fire
 
-
-    # Check no errors were recorded, because we asked for 204 which is slightly uncommon but is still OK
-    res = get_index(client)
-    assert b'notification-error' not in res.data
-
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
         j = json.loads(x)
@@ -372,10 +360,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     #live_server_setup(live_server)
     set_original_response()
     if os.path.isfile("test-datastore/notification.txt"):
-        os.unlink("test-datastore/notification.txt") \
+        os.unlink("test-datastore/notification.txt")
-
-    # 1995 UTF-8 content should be encoded
-    test_body = 'change detection is cool 网站监测 内容更新了'
 
     # otherwise other settings would have already existed from previous tests in this file
     res = client.post(
@@ -383,7 +368,8 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
         data={
             "application-fetch_backend": "html_requests",
             "application-minutes_between_check": 180,
-            "application-notification_body": test_body,
+            #1995 UTF-8 content should be encoded
+            "application-notification_body": 'change detection is cool 网站监测 内容更新了',
             "application-notification_format": default_notification_format,
             "application-notification_urls": "",
             "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
@@ -413,10 +399,12 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert res.status_code != 400
     assert res.status_code != 500
 
+    # Give apprise time to fire
+    time.sleep(4)
 
     with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
-        assert test_body in x
+        assert 'change detection is cool 网站监测 内容更新了' in x
 
     os.unlink("test-datastore/notification.txt")
 

changedetectionio/tests/test_request.py

@@ -373,14 +373,13 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     with open('test-datastore/headers-testtag.txt', 'w') as f:
-        f.write("tag-header: test\r\nurl-header: http://example.com")
+        f.write("tag-header: test")
 
     with open('test-datastore/headers.txt', 'w') as f:
-        f.write("global-header: nice\r\nnext-global-header: nice\r\nurl-header-global: http://example.com/global")
+        f.write("global-header: nice\r\nnext-global-header: nice")
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    with open('test-datastore/' + extract_UUID_from_client(client) + '/headers.txt', 'w') as f:
-    with open(f'test-datastore/{uuid}/headers.txt', 'w') as f:
+        f.write("watch-header: nice")
-        f.write("watch-header: nice\r\nurl-header-watch: http://example.com/watch")
 
     wait_for_all_checks(client)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -411,9 +410,6 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
     assert b"Xxx:ooo" in res.data
     assert b"Watch-Header:nice" in res.data
     assert b"Tag-Header:test" in res.data
-    assert b"Url-Header:http://example.com" in res.data
-    assert b"Url-Header-Global:http://example.com/global" in res.data
-    assert b"Url-Header-Watch:http://example.com/watch" in res.data
 
     # Check the custom UA from system settings page made it through
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):

changedetectionio/tests/test_restock_itemprop.py

@@ -189,17 +189,6 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
     client.get(url_for("mark_all_viewed"))
 
-
-    # 2715 - Price detection (once it crosses the "lower" threshold) again with a lower price - should trigger again!
-    set_original_response(props_markup=instock_props[0], price='820.45')
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'820.45' in res.data
-    assert b'unviewed' in res.data
-    client.get(url_for("mark_all_viewed"))
-
     # price changed to something MORE than max (1100.10), SHOULD be a change
     set_original_response(props_markup=instock_props[0], price='1890.45')
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -214,7 +203,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
 
 def test_restock_itemprop_minmax(client, live_server):
-    #live_server_setup(live_server)
+#    live_server_setup(live_server)
     extras = {
         "restock_settings-follow_price_changes": "y",
         "restock_settings-price_change_min": 900.0,
@@ -380,7 +369,7 @@ def test_change_with_notification_values(client, live_server):
 
     ## Now test the "SEND TEST NOTIFICATION" is working
     os.unlink("test-datastore/notification.txt")
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(url_for("ajax_callback_send_notification_test", watch_uuid=uuid), data={}, follow_redirects=True)
     time.sleep(5)
     assert os.path.isfile("test-datastore/notification.txt"), "Notification received"

changedetectionio/tests/test_rss.py

@@ -132,7 +132,7 @@ def test_rss_xpath_filtering(client, live_server, measure_memory_usage):
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={

changedetectionio/tests/test_scheduler.py

@@ -39,7 +39,7 @@ def test_check_basic_scheduler_functionality(client, live_server, measure_memory
 
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
 
@@ -104,7 +104,7 @@ def test_check_basic_global_scheduler_functionality(client, live_server, measure
 
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
 
     # Setup all the days of the weeks using XXX as the placeholder for monday/tuesday/etc
 

changedetectionio/tests/unit/test_semver.py

@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-
-# run from dir above changedetectionio/ dir
-# python3 -m unittest changedetectionio.tests.unit.test_semver
-
-import re
-import unittest
-
-
-# The SEMVER regex
-SEMVER_REGEX = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
-
-# Compile the regex
-semver_pattern = re.compile(SEMVER_REGEX)
-
-class TestSemver(unittest.TestCase):
-    def test_valid_versions(self):
-        """Test valid semantic version strings"""
-        valid_versions = [
-            "1.0.0",
-            "0.1.0",
-            "0.0.1",
-            "1.0.0-alpha",
-            "1.0.0-alpha.1",
-            "1.0.0-0.3.7",
-            "1.0.0-x.7.z.92",
-            "1.0.0-alpha+001",
-            "1.0.0+20130313144700",
-            "1.0.0-beta+exp.sha.5114f85"
-        ]
-        for version in valid_versions:
-            with self.subTest(version=version):
-                self.assertIsNotNone(semver_pattern.match(version), f"Version {version} should be valid")
-
-    def test_invalid_versions(self):
-        """Test invalid semantic version strings"""
-        invalid_versions = [
-            "0.48.06",
-            "1.0",
-            "1.0.0-",
-# Seems to pass the semver.org regex?
-#            "1.0.0-alpha-",
-            "1.0.0+",
-            "1.0.0-alpha+",
-            "1.0.0-",
-            "01.0.0",
-            "1.01.0",
-            "1.0.01",
-            ".1.0.0",
-            "1..0.0"
-        ]
-        for version in invalid_versions:
-            with self.subTest(version=version):
-                res = semver_pattern.match(version)
-                self.assertIsNone(res, f"Version '{version}' should be invalid")
-
-    def test_our_version(self):
-        from changedetectionio import get_version
-        our_version = get_version()
-        self.assertIsNotNone(semver_pattern.match(our_version), f"Our version '{our_version}' should be a valid SEMVER string")
-
-
-if __name__ == '__main__':
-    unittest.main()

changedetectionio/tests/util.py

@@ -76,14 +76,6 @@ def set_more_modified_response():
     return None
 
 
-def set_empty_text_response():
-    test_return_data = """<html><body></body></html>"""
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-    return None
-
 def wait_for_notification_endpoint_output():
     '''Apprise can take a few seconds to fire'''
     #@todo - could check the apprise object directly instead of looking for this file
@@ -223,10 +215,9 @@ def live_server_setup(live_server):
     def test_method():
         return request.method
 
-    # Where we POST to as a notification, also use a space here to test URL escaping is OK across all tests that use this. ( #2868 )
+    # Where we POST to as a notification
-    @live_server.app.route('/test_notification endpoint', methods=['POST', 'GET'])
+    @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET'])
     def test_notification_endpoint():
-
         with open("test-datastore/notification.txt", "wb") as f:
             # Debug method, dump all POST to file also, used to prove #65
             data = request.stream.read()
@@ -244,11 +235,8 @@ def live_server_setup(live_server):
                 f.write(request.content_type)
 
         print("\n>> Test notification endpoint was hit.\n", data)
+        return "Text was set"
 
-        content = "Text was set"
-        status_code = request.args.get('status_code',200)
-        resp = make_response(content, status_code)
-        return resp
 
     # Just return the verb in the request
     @live_server.app.route('/test-basicauth', methods=['GET'])
@@ -285,43 +273,15 @@ def live_server_setup(live_server):
             <p id="remove">This text should be removed</p>
               <form onsubmit="event.preventDefault();">
             <!-- obfuscated text so that we dont accidentally get a false positive due to conversion of the source :) --->
-                <button name="test-button" onclick="
+                <button name="test-button" onclick="getElementById('remove').remove();getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=')">Click here</button>
-                getElementById('remove').remove();
+                <div id=some-content></div>
-                getElementById('some-content').innerHTML = atob('SSBzbWVsbCBKYXZhU2NyaXB0IGJlY2F1c2UgdGhlIGJ1dHRvbiB3YXMgcHJlc3NlZCE=');
-                getElementById('reflect-text').innerHTML = getElementById('test-input-text').value;
-                ">Click here</button>
-                
-                <div id="some-content"></div>
-                
                 <pre>
                 {header_text.lower()}
                 </pre>
-                
+              </body>
-                <br>
-                <!-- used for testing that the jinja2 compiled here --->
-                <input type="text" value="" id="test-input-text" /><br>
-                <div id="reflect-text">Waiting to reflect text from #test-input-text here</div>
-              </form>
-                
-           </body>
          </html>""", 200)
         resp.headers['Content-Type'] = 'text/html'
         return resp
 
     live_server.start()
 
-def get_index(client):
-    import inspect
-    # Get the caller's frame (parent function)
-    frame = inspect.currentframe()
-    caller_frame = frame.f_back  # Go back to the caller's frame
-    caller_name = caller_frame.f_code.co_name
-    caller_line = caller_frame.f_lineno
-
-    print(f"Called by: {caller_name}, Line: {caller_line}")
-
-    res = client.get(url_for("index"))
-    with open(f"test-datastore/index-{caller_name}-{caller_line}.html", 'wb') as f:
-        f.write(res.data)
-
-    return res

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -2,16 +2,14 @@
 
 import os
 from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks, get_index
+from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
-def test_setup(client, live_server):
+def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server, measure_memory_usage):
-    live_server.stop()
-    live_server.start()
 
     import os
     import json
@@ -29,7 +27,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
         follow_redirects=True
     )
     assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
         data={
@@ -89,9 +87,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage
 
 def test_basic_browserstep(client, live_server, measure_memory_usage):
 
-    live_server.stop()
+    #live_server_setup(live_server)
-    live_server.start()
-
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
 
     test_url = url_for('test_interactive_html_endpoint', _external=True)
@@ -112,13 +108,9 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
             "url": test_url,
             "tags": "",
             'fetch_backend': "html_webdriver",
-            'browser_steps-0-operation': 'Enter text in field',
+            'browser_steps-0-operation': 'Click element',
-            'browser_steps-0-selector': '#test-input-text',
+            'browser_steps-0-selector': 'button[name=test-button]',
-            # Should get set to the actual text (jinja2 rendered)
+            'browser_steps-0-optional_value': '',
-            'browser_steps-0-optional_value': "Hello-Jinja2-{% now  'Europe/Berlin', '%Y-%m-%d' %}",
-            'browser_steps-1-operation': 'Click element',
-            'browser_steps-1-selector': 'button[name=test-button]',
-            'browser_steps-1-optional_value': '',
             # For now, cookies doesnt work in headers because it must be a full cookiejar object
             'headers': "testheader: yes\buser-agent: MyCustomAgent",
         },
@@ -127,7 +119,7 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
 
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
+    uuid = extract_UUID_from_client(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, "Watch history had atleast 1 (everything fetched OK)"
 
     assert b"This text should be removed" not in res.data
@@ -140,32 +132,13 @@ def test_basic_browserstep(client, live_server, measure_memory_usage):
     assert b"This text should be removed" not in res.data
     assert b"I smell JavaScript because the button was pressed" in res.data
 
-    assert b'Hello-Jinja2-20' in res.data
-
     assert b"testheader: yes" in res.data
     assert b"user-agent: mycustomagent" in res.data
-    live_server.stop()
-
-def test_non_200_errors_report_browsersteps(client, live_server):
-
-    live_server.stop()
-    live_server.start()
 
     four_o_four_url =  url_for('test_endpoint', status_code=404, _external=True)
     four_o_four_url = four_o_four_url.replace('localhost.localdomain', 'cdio')
     four_o_four_url = four_o_four_url.replace('localhost', 'cdio')
 
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": four_o_four_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added in Paused state, saving will unpause" in res.data
-    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
-
-    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-
     # now test for 404 errors
     res = client.post(
         url_for("edit_page", uuid=uuid, unpause_on_save=1),
@@ -180,14 +153,12 @@ def test_non_200_errors_report_browsersteps(client, live_server):
         follow_redirects=True
     )
     assert b"unpaused" in res.data
-
     wait_for_all_checks(client)
 
-    res = get_index(client)
+    res = client.get(url_for("index"))
-
     assert b'Error - 404' in res.data
 
     client.get(
         url_for("form_delete", uuid="all"),
         follow_redirects=True
-    )
+    )

changedetectionio/update_worker.py

@@ -243,6 +243,7 @@ class update_worker(threading.Thread):
                 os.unlink(full_path)
 
     def run(self):
+        now = time.time()
         
         while not self.app.config.exit.is_set():
             update_handler = None
@@ -253,7 +254,6 @@ class update_worker(threading.Thread):
                 pass
 
             else:
-                fetch_start_time = time.time()
                 uuid = queued_item_data.item.get('uuid')
                 self.current_uuid = uuid
                 if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
@@ -268,6 +268,7 @@ class update_worker(threading.Thread):
                     watch = self.datastore.data['watching'].get(uuid)
 
                     logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
+                    now = time.time()
 
                     try:
                         # Processor is what we are using for detecting the "Change"
@@ -287,10 +288,6 @@ class update_worker(threading.Thread):
 
                         update_handler.call_browser()
 
-                        # In reality, the actual time of when the change was detected could be a few seconds after this
-                        # For example it should include when the page stopped rendering if using a playwright/chrome type fetch
-                        fetch_start_time = time.time()
-
                         changed_detected, update_obj, contents = update_handler.run_changedetection(watch=watch)
 
                         # Re #342
@@ -515,7 +512,7 @@ class update_worker(threading.Thread):
 
                     if not self.datastore.data['watching'].get(uuid):
                         continue
-
+                    #
                     # Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
                     if process_changedetection_results:
 
@@ -528,6 +525,8 @@ class update_worker(threading.Thread):
                                 except Exception as e:
                                     logger.warning(f"UUID: {uuid} Extract <title> as watch title was enabled, but couldn't find a <title>.")
 
+                        # Now update after running everything
+                        timestamp = round(time.time())
                         try:
                             self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
@@ -543,28 +542,24 @@ class update_worker(threading.Thread):
 
                                 # Small hack so that we sleep just enough to allow 1 second  between history snapshots
                                 # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
-                                # @also - the keys are one per second at the most (for now)
+
-                                if watch.newest_history_key and int(fetch_start_time) == int(watch.newest_history_key):
+                                if watch.newest_history_key and int(timestamp) == int(watch.newest_history_key):
                                     logger.warning(
-                                        f"Timestamp {fetch_start_time} already exists, waiting 1 seconds so we have a unique key in history.txt")
+                                        f"Timestamp {timestamp} already exists, waiting 1 seconds so we have a unique key in history.txt")
-                                    fetch_start_time += 1
+                                    timestamp = str(int(timestamp) + 1)
                                     time.sleep(1)
 
                                 watch.save_history_text(contents=contents,
-                                                        timestamp=int(fetch_start_time),
+                                                        timestamp=timestamp,
                                                         snapshot_id=update_obj.get('previous_md5', 'none'))
 
-
+                                if update_handler.fetcher.content:
-                                empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
+                                    watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=timestamp)
-                                if update_handler.fetcher.content or (not update_handler.fetcher.content and empty_pages_are_a_change):
-                                    # attribute .last_changed is then based on this data
-                                    watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=int(fetch_start_time))
 
                                 # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                 if watch.history_n >= 2:
                                     logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
                                     if not watch.get('notification_muted'):
-                                        # @todo only run this if notifications exist
                                         self.send_content_changed_notification(watch_uuid=uuid)
 
                         except Exception as e:
@@ -586,15 +581,15 @@ class update_worker(threading.Thread):
                     except Exception as e:
                         pass
 
-                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - fetch_start_time, 3),
+                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
-                                                                       'last_checked': int(fetch_start_time),
+                                                                       'last_checked': round(time.time()),
                                                                        'check_count': count
                                                                        })
 
 
                 self.current_uuid = None  # Done
                 self.q.task_done()
-                logger.debug(f"Watch {uuid} done in {time.time()-fetch_start_time:.2f}s")
+                logger.debug(f"Watch {uuid} done in {time.time()-now:.2f}s")
 
                 # Give the CPU time to interrupt
                 time.sleep(0.1)

docker-compose.yml

@@ -1,3 +1,4 @@
+version: '3.2'
 services:
     changedetection:
       image: ghcr.io/dgtlmoon/changedetection.io

requirements.txt

@@ -35,7 +35,7 @@ dnspython==2.6.1 # related to eventlet fixes
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise==1.9.2
+apprise==1.9.0
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 # use any version other than 2.0.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
@@ -95,8 +95,5 @@ babel
 # Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
 greenlet >= 3.0.3
 
-# Pinned or it causes problems with flask_expects_json which seems unmaintained
-referencing==0.35.1
-
 # Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !)
 tzdata