Use soft delays instead of blocking time sleeps

This reverts commit e2b407c6f3.

.github/actions/extract-memory-report/action.yml

@@ -23,26 +23,29 @@ runs:
     - name: Dump container log
       shell: bash
       run: |
-        docker logs ${{ inputs.container-name }} > ${{ inputs.output-dir }}/${{ inputs.container-name }}-stdout-${{ inputs.python-version }}.txt 2>&1 || echo "Could not get stdout"
-        docker logs ${{ inputs.container-name }} 2> ${{ inputs.output-dir }}/${{ inputs.container-name }}-stderr-${{ inputs.python-version }}.txt || echo "Could not get stderr"
+        echo "Disabled for now"
+#        return
+#        docker logs ${{ inputs.container-name }} > ${{ inputs.output-dir }}/${{ inputs.container-name }}-stdout-${{ inputs.python-version }}.txt 2>&1 || echo "Could not get stdout"
+#        docker logs ${{ inputs.container-name }} 2> ${{ inputs.output-dir }}/${{ inputs.container-name }}-stderr-${{ inputs.python-version }}.txt || echo "Could not get stderr"
 
     - name: Extract and display memory test report
       shell: bash
       run: |
-        echo "Extracting test-memory.log from container..."
-        docker cp ${{ inputs.container-name }}:/app/changedetectionio/test-memory.log ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log || echo "test-memory.log not found in container"
-
-        echo "=== Top 10 Highest Peak Memory Tests ==="
-        if [ -f ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log ]; then
-          grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log | \
-            sed 's/.*Peak memory: //' | \
-            paste -d'|' - <(grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log) | \
-            sort -t'|' -k1 -nr | \
-            cut -d'|' -f2 | \
-            head -10
-          echo ""
-          echo "=== Full Memory Test Report ==="
-          cat ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log
-        else
-          echo "No memory log available"
-        fi
+        echo "Disabled for now"
+#        echo "Extracting test-memory.log from container..."
+#        docker cp ${{ inputs.container-name }}:/app/changedetectionio/test-memory.log ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log || echo "test-memory.log not found in container"
+#
+#        echo "=== Top 10 Highest Peak Memory Tests ==="
+#        if [ -f ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log ]; then
+#          grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log | \
+#            sed 's/.*Peak memory: //' | \
+#            paste -d'|' - <(grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log) | \
+#            sort -t'|' -k1 -nr | \
+#            cut -d'|' -f2 | \
+#            head -10
+#          echo ""
+#          echo "=== Full Memory Test Report ==="
+#          cat ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log
+#        else
+#          echo "No memory log available"
+#        fi

.github/workflows/codeql-analysis.yml

@@ -30,7 +30,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v5
+      uses: actions/checkout@v6
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/containers.yml

@@ -39,12 +39,20 @@ jobs:
     # Or if we are in a tagged release scenario.
     if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Set up Python 3.11
         uses: actions/setup-python@v6
         with:
           python-version: 3.11
 
+      - name: Cache pip packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/pypi-release.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6
     - name: Set up Python
       uses: actions/setup-python@v6
       with:

.github/workflows/test-container-build.yml

@@ -44,12 +44,20 @@ jobs:
           - platform: linux/arm64
             dockerfile: ./.github/test/Dockerfile-alpine
     steps:
-        - uses: actions/checkout@v5
+        - uses: actions/checkout@v6
         - name: Set up Python 3.11
           uses: actions/setup-python@v6
           with:
             python-version: 3.11
 
+        - name: Cache pip packages
+          uses: actions/cache@v4
+          with:
+            path: ~/.cache/pip
+            key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+            restore-keys: |
+              ${{ runner.os }}-pip-
+
         # Just test that the build works, some libraries won't compile on ARM/rPi etc
         - name: Set up QEMU
           uses: docker/setup-qemu-action@v3
@@ -74,5 +82,5 @@ jobs:
             file: ${{ matrix.dockerfile }}
             platforms: ${{ matrix.platform }}
             cache-from: type=gha
-            cache-to: type=gha,mode=min
+            cache-to: type=gha,mode=max
 

.github/workflows/test-only.yml

@@ -7,7 +7,7 @@ jobs:
   lint-code:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Lint with Ruff
         run: |
           pip install ruff
@@ -21,6 +21,8 @@ jobs:
           python3 -c "from openapi_spec_validator import validate_spec; import yaml; validate_spec(yaml.safe_load(open('docs/api-spec.yaml')))"
 
   test-application-3-10:
+    # Only run on push to master (including PR merges)
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
     needs: lint-code
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:
@@ -28,12 +30,15 @@ jobs:
 
 
   test-application-3-11:
+    # Always run
     needs: lint-code
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:
       python-version: '3.11'
 
   test-application-3-12:
+    # Only run on push to master (including PR merges)
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
     needs: lint-code
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:
@@ -41,6 +46,8 @@ jobs:
       skip-pypuppeteer: true
 
   test-application-3-13:
+    # Only run on push to master (including PR merges)
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
     needs: lint-code
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:

.github/workflows/test-stack-reusable-workflow.yml

@@ -21,13 +21,22 @@ jobs:
     env:
       PYTHON_VERSION: ${{ inputs.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
+      - name: Cache pip packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}-
+            ${{ runner.os }}-pip-
+
       - name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}
         run: |
           echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----"
@@ -57,10 +66,10 @@ jobs:
     env:
       PYTHON_VERSION: ${{ inputs.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -84,10 +93,10 @@ jobs:
     env:
       PYTHON_VERSION: ${{ inputs.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -123,10 +132,10 @@ jobs:
     env:
       PYTHON_VERSION: ${{ inputs.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -165,10 +174,10 @@ jobs:
     env:
       PYTHON_VERSION: ${{ inputs.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -205,10 +214,10 @@ jobs:
     env:
       PYTHON_VERSION: ${{ inputs.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -241,10 +250,10 @@ jobs:
     env:
       PYTHON_VERSION: ${{ inputs.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -270,10 +279,10 @@ jobs:
     env:
       PYTHON_VERSION: ${{ inputs.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -289,15 +298,18 @@ jobs:
           docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
           docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
 
-      - name: Test proxy squid style interaction
+      - name: Test proxy Squid style interaction
         run: |
           cd changedetectionio
           ./run_proxy_tests.sh
+          docker ps
+          cd ..
 
       - name: Test proxy SOCKS5 style interaction
         run: |
           cd changedetectionio
           ./run_socks_proxy_tests.sh
+          cd ..
 
   # Custom browser URL tests
   custom-browser-tests:
@@ -307,10 +319,10 @@ jobs:
     env:
       PYTHON_VERSION: ${{ inputs.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -338,10 +350,10 @@ jobs:
     env:
       PYTHON_VERSION: ${{ inputs.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -383,10 +395,10 @@ jobs:
     env:
       PYTHON_VERSION: ${{ inputs.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp

.gitignore

@@ -21,6 +21,7 @@ venv/
 # IDEs
 .idea
 .vscode/settings.json
+*~
 
 # Datastore files
 datastore/

Dockerfile

@@ -34,23 +34,27 @@ ENV OPENSSL_LIB_DIR="/usr/lib/arm-linux-gnueabihf"
 ENV OPENSSL_INCLUDE_DIR="/usr/include/openssl"
 # Additional environment variables for cryptography Rust build
 ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
-RUN --mount=type=cache,target=/tmp/pip-cache \
-    pip install \
-    --extra-index-url https://www.piwheels.org/simple \
-    --extra-index-url https://pypi.anaconda.org/ARM-software/simple \
-    --cache-dir=/tmp/pip-cache \
-    --target=/dependencies \
-    -r /requirements.txt
+RUN --mount=type=cache,id=pip,sharing=locked,target=/tmp/pip-cache \
+  pip install \
+  --prefer-binary \
+  --extra-index-url https://www.piwheels.org/simple \
+  --extra-index-url https://pypi.anaconda.org/ARM-software/simple \
+  --cache-dir=/tmp/pip-cache \
+  --target=/dependencies \
+  -r /requirements.txt
+
 
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN --mount=type=cache,target=/tmp/pip-cache \
-    pip install \
-    --cache-dir=/tmp/pip-cache \
-    --target=/dependencies \
-    playwright~=1.48.0 \
-    || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
+RUN --mount=type=cache,id=pip,sharing=locked,target=/tmp/pip-cache \
+  pip install \
+  --prefer-binary \
+  --cache-dir=/tmp/pip-cache \
+  --target=/dependencies \
+  playwright~=1.56.0 \
+  || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
+
 
 # Final image stage
 FROM python:${PYTHON_VERSION}-slim-bookworm

README-pip.md

@@ -14,7 +14,7 @@ Ideal for monitoring price changes, content edits, conditional changes and more.
 [<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring, list of websites with changes"  title="Self-hosted web page change monitoring, list of websites with changes"  />](https://changedetection.io)
 
 
-[**Don't have time? Try our extremely affordable subscription use our proxies and support!**](https://changedetection.io) 
+[**Don't have time? Try our extremely affordable subscription use our proxies and support!**](https://changedetection.io)
 
 
 
@@ -31,7 +31,7 @@ Available when connected to a <a href="https://github.com/dgtlmoon/changedetecti
 
 ### Perform interactive browser steps
 
-Fill in text boxes, click buttons and more, setup your changedetection scenario. 
+Fill in text boxes, click buttons and more, setup your changedetection scenario.
 
 Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
 
@@ -54,7 +54,7 @@ Requires Playwright to be enabled.
 - Know when your favourite whiskey is on sale, or other special deals are announced before anyone else
 - COVID related news from government websites
 - University/organisation news from their website
-- Detect and monitor changes in JSON API responses 
+- Detect and monitor changes in JSON API responses
 - JSON API monitoring and alerting
 - Changes in legal and other documents
 - Trigger API calls via notifications when text appears on a website
@@ -86,7 +86,7 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 
 We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
 
-[Oxylabs](https://oxylabs.go2cloud.org/SH2d) is also an excellent proxy provider and well worth using, they offer Residental, ISP, Rotating and many other proxy types to suit your project. 
+[Oxylabs](https://oxylabs.go2cloud.org/SH2d) is also an excellent proxy provider and well worth using, they offer Residential, ISP, Rotating and many other proxy types to suit your project.
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
@@ -106,4 +106,3 @@ $ changedetection.io -d /path/to/empty/data/dir -p 5000
 Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
 
 See https://changedetection.io for more information.
-

changedetectionio/PLUGIN_README.md

@@ -64,7 +64,7 @@ def count_words_in_history(watch):
             return 0
             
         latest_key = list(watch.history.keys())[-1]
-        latest_content = watch.get_history_snapshot(latest_key)
+        latest_content = watch.get_history_snapshot(timestamp=latest_key)
         return len(latest_content.split())
     except Exception as e:
         logger.error(f"Error counting words: {str(e)}")

changedetectionio/__init__.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
-
-__version__ = '0.50.33'
+# Semver means never use .01, or 00. Should be .1.
+__version__ = '0.51.4'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -74,6 +74,12 @@ def main():
 
     datastore_path = None
     do_cleanup = False
+    # Optional URL to watch since start
+    default_url = None
+    # Set a default logger level
+    logger_level = 'DEBUG'
+    include_default_watches = True
+
     host = os.environ.get("LISTEN_HOST", "0.0.0.0").strip()
     port = int(os.environ.get('PORT', 5000))
     ssl_mode = False
@@ -87,15 +93,13 @@ def main():
         datastore_path = os.path.join(os.getcwd(), "../datastore")
 
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:l:", "port")
+        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:l:u:", "port")
     except getopt.GetoptError:
-        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path] -l [debug level - TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL]')
+        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path] -u [default URL to watch] -l [debug level - TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL]')
         sys.exit(2)
 
     create_datastore_dir = False
 
-    # Set a default logger level
-    logger_level = 'DEBUG'
     # Set a logger level via shell env variable
     # Used: Dockerfile for CICD
     # To set logger level for pytest, see the app function in tests/conftest.py
@@ -116,6 +120,10 @@ def main():
         if opt == '-d':
             datastore_path = arg
 
+        if opt == '-u':
+            default_url = arg
+            include_default_watches = False
+
         # Cleanup (remove text files that arent in the index)
         if opt == '-c':
             do_cleanup = True
@@ -172,13 +180,16 @@ def main():
             sys.exit(2)
 
     try:
-        datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
+        datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__, include_default_watches=include_default_watches)
     except JSONDecodeError as e:
         # Dont' start if the JSON DB looks corrupt
         logger.critical(f"ERROR: JSON DB or Proxy List JSON at '{app_config['datastore_path']}' appears to be corrupt, aborting.")
         logger.critical(str(e))
         return
 
+    if default_url:
+        datastore.add_watch(url = default_url)
+
     app = changedetection_app(app_config, datastore)
 
     # Get the SocketIO instance from the Flask app (created in flask_app.py)

changedetectionio/api/Watch.py

@@ -175,7 +175,7 @@ class WatchSingleHistory(Resource):
                 response = make_response("No content found", 404)
                 response.mimetype = "text/plain"
         else:
-            content = watch.get_history_snapshot(timestamp)
+            content = watch.get_history_snapshot(timestamp=timestamp)
             response = make_response(content, 200)
             response.mimetype = "text/plain"
 

changedetectionio/api/__init__.py

@@ -41,7 +41,7 @@ def get_openapi_spec():
         # Possibly for pip3 packages
         spec_path = os.path.join(os.path.dirname(__file__), '../docs/api-spec.yaml')
 
-    with open(spec_path, 'r') as f:
+    with open(spec_path, 'r', encoding='utf-8') as f:
         spec_dict = yaml.safe_load(f)
     _openapi_spec = OpenAPI.from_dict(spec_dict)
     return _openapi_spec

changedetectionio/api/api_schema.py

@@ -96,7 +96,10 @@ def build_watch_json_schema(d):
                                                            "enum": ["html_requests", "html_webdriver"]
                                                            })
 
-
+    schema['properties']['processor'] = {"anyOf": [
+        {"type": "string", "enum": ["restock_diff", "text_json_diff"]},
+        {"type": "null"}
+    ]}
 
     # All headers must be key/value type dict
     schema['properties']['headers'] = {

changedetectionio/async_update_worker.py

@@ -353,12 +353,15 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
                 count = watch.get('check_count', 0) + 1
 
                 # Always record page title (used in notifications, and can change even when the content is the same)
-                try:
-                    page_title = html_tools.extract_title(data=update_handler.fetcher.content)
-                    logger.debug(f"UUID: {uuid} Page <title> is '{page_title}'")
-                    datastore.update_watch(uuid=uuid, update_obj={'page_title': page_title})
-                except Exception as e:
-                    logger.warning(f"UUID: {uuid} Exception when extracting <title> - {str(e)}")
+                if update_obj.get('content-type') and 'html' in update_obj.get('content-type'):
+                    try:
+                        page_title = html_tools.extract_title(data=update_handler.fetcher.content)
+                        if page_title:
+                            page_title = page_title.strip()[:2000]
+                            logger.debug(f"UUID: {uuid} Page <title> is '{page_title}'")
+                            datastore.update_watch(uuid=uuid, update_obj={'page_title': page_title})
+                    except Exception as e:
+                        logger.warning(f"UUID: {uuid} Exception when extracting <title> - {str(e)}")
 
                 # Record server header
                 try:

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -439,7 +439,7 @@ class browsersteps_live_ui(steppable_browser_interface):
             logger.warning("Attempted to get current state after cleanup")
             return (None, None)
 
-        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
+        xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text(encoding="utf-8")
 
         now = time.time()
         await self.page.wait_for_timeout(1 * 1000)

changedetectionio/blueprint/rss/__init__.py

@@ -1 +1,27 @@
-RSS_FORMAT_TYPES = [('plaintext', 'Plain text'), ('html', 'HTML Color')]
+from copy import deepcopy
+from loguru import logger
+
+from changedetectionio.model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
+from changedetectionio.notification import valid_notification_formats
+
+RSS_CONTENT_FORMAT_DEFAULT = 'text'
+
+# Some stuff not related
+RSS_FORMAT_TYPES = deepcopy(valid_notification_formats)
+if RSS_FORMAT_TYPES.get('markdown'):
+    del RSS_FORMAT_TYPES['markdown']
+
+if RSS_FORMAT_TYPES.get(USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH):
+    del RSS_FORMAT_TYPES[USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH]
+
+if not RSS_FORMAT_TYPES.get(RSS_CONTENT_FORMAT_DEFAULT):
+    logger.critical(f"RSS_CONTENT_FORMAT_DEFAULT not in the acceptable list {RSS_CONTENT_FORMAT_DEFAULT}")
+
+RSS_TEMPLATE_TYPE_OPTIONS = {'system_default': 'System default', 'notification_body': 'Notification body'}
+
+# @note: We use <pre> because nearly all RSS readers render only HTML (Thunderbird for example cant do just plaintext)
+RSS_TEMPLATE_PLAINTEXT_DEFAULT = "<pre>{{watch_label}} had a change.\n\n{{diff}}\n</pre>"
+
+# @todo add some [edit]/[history]/[goto] etc links
+# @todo need {{watch_edit_link}} + delete + history link token
+RSS_TEMPLATE_HTML_DEFAULT = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_label}}</a></h4>\n<p>{{diff}}</p>\n</body></html>\n"

changedetectionio/blueprint/rss/_util.py

@@ -0,0 +1,156 @@
+"""
+Utility functions for RSS feed generation.
+"""
+
+from changedetectionio.notification.handler import process_notification
+from changedetectionio.notification_service import NotificationContextData, _check_cascading_vars
+from loguru import logger
+import datetime
+import pytz
+import re
+
+
+BAD_CHARS_REGEX = r'[\x00-\x08\x0B\x0C\x0E-\x1F]'
+
+
+def scan_invalid_chars_in_rss(content):
+    """
+    Scan for invalid characters in RSS content.
+    Returns True if invalid characters are found.
+    """
+    for match in re.finditer(BAD_CHARS_REGEX, content):
+        i = match.start()
+        bad_char = content[i]
+        hex_value = f"0x{ord(bad_char):02x}"
+        # Grab context
+        start = max(0, i - 20)
+        end = min(len(content), i + 21)
+        context = content[start:end].replace('\n', '\\n').replace('\r', '\\r')
+        logger.warning(f"Invalid char {hex_value} at pos {i}: ...{context}...")
+        # First match is enough
+        return True
+
+    return False
+
+
+def clean_entry_content(content):
+    """
+    Remove invalid characters from RSS content.
+    """
+    cleaned = re.sub(BAD_CHARS_REGEX, '', content)
+    return cleaned
+
+
+def generate_watch_guid(watch, timestamp):
+    """
+    Generate a unique GUID for a watch RSS entry.
+
+    Args:
+        watch: The watch object
+        timestamp: The timestamp of the specific change this entry represents
+    """
+    return f"{watch['uuid']}/{timestamp}"
+
+
+def validate_rss_token(datastore, request):
+    """
+    Validate the RSS access token from the request.
+
+    Returns:
+        tuple: (is_valid, error_response) where error_response is None if valid
+    """
+    app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
+    rss_url_token = request.args.get('token')
+
+    if rss_url_token != app_rss_token:
+        return False, ("Access denied, bad token", 403)
+
+    return True, None
+
+
+def get_rss_template(datastore, watch, rss_content_format, default_html, default_plaintext):
+    """Get the appropriate template for RSS content."""
+    if datastore.data['settings']['application'].get('rss_template_type') == 'notification_body':
+        return _check_cascading_vars(datastore=datastore, var_name='notification_body', watch=watch)
+
+    override = datastore.data['settings']['application'].get('rss_template_override')
+    if override and override.strip():
+        return override
+    elif 'text' in rss_content_format:
+        return default_plaintext
+    else:
+        return default_html
+
+
+def get_watch_label(datastore, watch):
+    """Get the label for a watch based on settings."""
+    if datastore.data['settings']['application']['ui'].get('use_page_title_in_list') or watch.get('use_page_title_in_list'):
+        return watch.label
+    else:
+        return watch.get('url')
+
+
+def add_watch_categories(fe, watch, datastore):
+    """Add category tags to a feed entry based on watch tags."""
+    for tag_uuid in watch.get('tags', []):
+        tag = datastore.data['settings']['application'].get('tags', {}).get(tag_uuid)
+        if tag and tag.get('title'):
+            fe.category(term=tag.get('title'))
+
+
+def build_notification_context(watch, timestamp_from, timestamp_to, watch_label,
+                               n_body_template, rss_content_format):
+    """Build the notification context object."""
+    return NotificationContextData(initial_data={
+        'notification_urls': ['null://just-sending-a-null-test-for-the-render-in-RSS'],
+        'notification_body': n_body_template,
+        'timestamp_to': timestamp_to,
+        'timestamp_from': timestamp_from,
+        'watch_label': watch_label,
+        'notification_format': rss_content_format
+    })
+
+
+def render_notification(n_object, notification_service, watch, datastore,
+                       date_index_from=None, date_index_to=None):
+    """Process and render the notification content."""
+    kwargs = {'n_object': n_object, 'watch': watch}
+
+    if date_index_from is not None and date_index_to is not None:
+        kwargs['date_index_from'] = date_index_from
+        kwargs['date_index_to'] = date_index_to
+
+    n_object = notification_service.queue_notification_for_watch(**kwargs)
+    n_object['watch_mime_type'] = None
+
+    res = process_notification(n_object=n_object, datastore=datastore)
+    return res[0]
+
+
+def populate_feed_entry(fe, watch, content, guid, timestamp, link=None, title_suffix=None):
+    """Populate a feed entry with content and metadata."""
+    watch_label = watch.get('url')  # Already determined by caller
+
+    # Set link
+    if link:
+        fe.link(link=link)
+
+    # Set title
+    if title_suffix:
+        fe.title(title=f"{watch_label} - {title_suffix}")
+    else:
+        fe.title(title=watch_label)
+
+    # Clean and set content
+    if scan_invalid_chars_in_rss(content):
+        content = clean_entry_content(content)
+    fe.content(content=content, type='CDATA')
+
+    # Set GUID
+    fe.guid(guid, permalink=False)
+
+    # Set pubDate using the timestamp of this specific change
+    dt = datetime.datetime.fromtimestamp(int(timestamp))
+    dt = dt.replace(tzinfo=pytz.UTC)
+    fe.pubDate(dt)
+

changedetectionio/blueprint/rss/blueprint.py

@@ -1,155 +1,26 @@
 
-from changedetectionio.jinja2_custom import render as jinja_render
-from changedetectionio.notification.handler import apply_service_tweaks
 from changedetectionio.store import ChangeDetectionStore
-from feedgen.feed import FeedGenerator
-from flask import Blueprint, make_response, request, url_for, redirect
-from loguru import logger
-import datetime
-import pytz
-import re
-import time
+from flask import Blueprint
 
-
-BAD_CHARS_REGEX=r'[\x00-\x08\x0B\x0C\x0E-\x1F]'
-
-# Anything that is not text/UTF-8 should be stripped before it breaks feedgen (such as binary data etc)
-def scan_invalid_chars_in_rss(content):
-    for match in re.finditer(BAD_CHARS_REGEX, content):
-        i = match.start()
-        bad_char = content[i]
-        hex_value = f"0x{ord(bad_char):02x}"
-        # Grab context
-        start = max(0, i - 20)
-        end = min(len(content), i + 21)
-        context = content[start:end].replace('\n', '\\n').replace('\r', '\\r')
-        logger.warning(f"Invalid char {hex_value} at pos {i}: ...{context}...")
-        # First match is enough
-        return True
-
-    return False
-
-
-def clean_entry_content(content):
-    cleaned = re.sub(BAD_CHARS_REGEX, '', content)
-    return cleaned
+from . import tag as tag_routes
+from . import main_feed
+from . import single_watch
 
 def construct_blueprint(datastore: ChangeDetectionStore):
+    """
+    Construct and configure the RSS blueprint with all routes.
+
+    Args:
+        datastore: The ChangeDetectionStore instance
+
+    Returns:
+        The configured Flask blueprint
+    """
     rss_blueprint = Blueprint('rss', __name__)
 
-    # Some RSS reader situations ended up with rss/ (forward slash after RSS) due
-    # to some earlier blueprint rerouting work, it should goto feed.
-    @rss_blueprint.route("/", methods=['GET'])
-    def extraslash():
-        return redirect(url_for('rss.feed'))
-
-    # Import the login decorator if needed
-    # from changedetectionio.auth_decorator import login_optionally_required
-    @rss_blueprint.route("", methods=['GET'])
-    def feed():
-        now = time.time()
-        # Always requires token set
-        app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
-        rss_url_token = request.args.get('token')
-        if rss_url_token != app_rss_token:
-            return "Access denied, bad token", 403
-
-        from changedetectionio import diff
-        limit_tag = request.args.get('tag', '').lower().strip()
-        # Be sure limit_tag is a uuid
-        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
-            if limit_tag == tag.get('title', '').lower().strip():
-                limit_tag = uuid
-
-        # Sort by last_changed and add the uuid which is usually the key..
-        sorted_watches = []
-
-        # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
-        for uuid, watch in datastore.data['watching'].items():
-            # @todo tag notification_muted skip also (improve Watch model)
-            if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):
-                continue
-            if limit_tag and not limit_tag in watch['tags']:
-                continue
-            watch['uuid'] = uuid
-            sorted_watches.append(watch)
-
-        sorted_watches.sort(key=lambda x: x.last_changed, reverse=False)
-
-        fg = FeedGenerator()
-        fg.title('changedetection.io')
-        fg.description('Feed description')
-        fg.link(href='https://changedetection.io')
-
-        html_colour_enable = False
-        if datastore.data['settings']['application'].get('rss_content_format') == 'html':
-            html_colour_enable = True
-
-        for watch in sorted_watches:
-
-            dates = list(watch.history.keys())
-            # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected.
-            if len(dates) < 2:
-                continue
-
-            if not watch.viewed:
-                # Re #239 - GUID needs to be individual for each event
-                # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228)
-                guid = "{}/{}".format(watch['uuid'], watch.last_changed)
-                fe = fg.add_entry()
-
-                # Include a link to the diff page, they will have to login here to see if password protection is enabled.
-                # Description is the page you watch, link takes you to the diff JS UI page
-                # Dict val base_url will get overriden with the env var if it is set.
-                ext_base_url = datastore.data['settings']['application'].get('active_base_url')
-                # @todo fix
-
-                # Because we are called via whatever web server, flask should figure out the right path (
-                diff_link = {'href': url_for('ui.ui_views.diff_history_page', uuid=watch['uuid'], _external=True)}
-
-                fe.link(link=diff_link)
-
-                # Same logic as watch-overview.html
-                if datastore.data['settings']['application']['ui'].get('use_page_title_in_list') or watch.get('use_page_title_in_list'):
-                    watch_label = watch.label
-                else:
-                    watch_label = watch.get('url')
-
-                fe.title(title=watch_label)
-                try:
-
-                    html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]),
-                                                 newest_version_file_contents=watch.get_history_snapshot(dates[-1]),
-                                                 include_equal=False,
-                                                 line_feed_sep="<br>"
-                                                 )
-
-
-                    requested_output_format = 'htmlcolor' if html_colour_enable else 'html'
-                    html_diff = apply_service_tweaks(url='', n_body=html_diff, n_title=None, requested_output_format=requested_output_format)
-
-                except FileNotFoundError as e:
-                    html_diff = f"History snapshot file for watch {watch.get('uuid')}@{watch.last_changed} - '{watch.get('title')} not found."
-
-                # @todo Make this configurable and also consider html-colored markup
-                # @todo User could decide if <link> goes to the diff page, or to the watch link
-                rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n"
-
-                content = jinja_render(template_str=rss_template, watch_title=watch_label, html_diff=html_diff, watch_url=watch.link)
-
-                # Out of range chars could also break feedgen
-                if scan_invalid_chars_in_rss(content):
-                    content = clean_entry_content(content)
-
-                fe.content(content=content, type='CDATA')
-                fe.guid(guid, permalink=False)
-                dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key))
-                dt = dt.replace(tzinfo=pytz.UTC)
-                fe.pubDate(dt)
-
-        response = make_response(fg.rss_str())
-        response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8')
-        logger.trace(f"RSS generated in {time.time() - now:.3f}s")
-        return response
+    # Register all route modules
+    main_feed.construct_main_feed_routes(rss_blueprint, datastore)
+    single_watch.construct_single_watch_routes(rss_blueprint, datastore)
+    tag_routes.construct_tag_routes(rss_blueprint, datastore)
 
     return rss_blueprint

changedetectionio/blueprint/rss/main_feed.py

@@ -0,0 +1,105 @@
+from flask import make_response, request, url_for, redirect
+
+
+
+def construct_main_feed_routes(rss_blueprint, datastore):
+    """
+    Construct the main RSS feed routes.
+
+    Args:
+        rss_blueprint: The Flask blueprint to add routes to
+        datastore: The ChangeDetectionStore instance
+    """
+
+    # Some RSS reader situations ended up with rss/ (forward slash after RSS) due
+    # to some earlier blueprint rerouting work, it should goto feed.
+    @rss_blueprint.route("/", methods=['GET'])
+    def extraslash():
+        return redirect(url_for('rss.feed'))
+
+    # Import the login decorator if needed
+    # from changedetectionio.auth_decorator import login_optionally_required
+    @rss_blueprint.route("", methods=['GET'])
+    def feed():
+        from feedgen.feed import FeedGenerator
+        from loguru import logger
+        import time
+
+        from . import RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT
+        from ._util import (validate_rss_token, generate_watch_guid, get_rss_template,
+                           get_watch_label, build_notification_context, render_notification,
+                           populate_feed_entry, add_watch_categories)
+        from ...notification_service import NotificationService
+
+        now = time.time()
+
+        # Validate token
+        is_valid, error = validate_rss_token(datastore, request)
+        if not is_valid:
+            return error
+
+        rss_content_format = datastore.data['settings']['application'].get('rss_content_format')
+
+        limit_tag = request.args.get('tag', '').lower().strip()
+        # Be sure limit_tag is a uuid
+        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+            if limit_tag == tag.get('title', '').lower().strip():
+                limit_tag = uuid
+
+        # Sort by last_changed and add the uuid which is usually the key..
+        sorted_watches = []
+
+        # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
+        for uuid, watch in datastore.data['watching'].items():
+            # @todo tag notification_muted skip also (improve Watch model)
+            if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):
+                continue
+            if limit_tag and not limit_tag in watch['tags']:
+                continue
+            sorted_watches.append(watch)
+
+        sorted_watches.sort(key=lambda x: x.last_changed, reverse=False)
+
+        fg = FeedGenerator()
+        fg.title('changedetection.io')
+        fg.description('Feed description')
+        fg.link(href='https://changedetection.io')
+        notification_service = NotificationService(datastore=datastore, notification_q=False)
+
+        for watch in sorted_watches:
+
+            dates = list(watch.history.keys())
+            # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected.
+            if len(dates) < 2:
+                continue
+
+            if not watch.viewed:
+                # Re #239 - GUID needs to be individual for each event
+                # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228)
+                watch_label = get_watch_label(datastore, watch)
+                timestamp_to = dates[-1]
+                timestamp_from = dates[-2]
+                guid = generate_watch_guid(watch, timestamp_to)
+                # Because we are called via whatever web server, flask should figure out the right path
+                diff_link = {'href': url_for('ui.ui_views.diff_history_page', uuid=watch['uuid'], _external=True)}
+
+                # Get template and build notification context
+                n_body_template = get_rss_template(datastore, watch, rss_content_format,
+                                                   RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT)
+
+                n_object = build_notification_context(watch, timestamp_from, timestamp_to,
+                                                     watch_label, n_body_template, rss_content_format)
+
+                # Render notification
+                res = render_notification(n_object, notification_service, watch, datastore)
+
+                # Create and populate feed entry
+                fe = fg.add_entry()
+                populate_feed_entry(fe, watch, res['body'], guid, timestamp_to, link=diff_link)
+                fe.title(title=watch_label)  # Override title to not include suffix
+                add_watch_categories(fe, watch, datastore)
+
+        response = make_response(fg.rss_str())
+        response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8')
+        logger.trace(f"RSS generated in {time.time() - now:.3f}s")
+        return response

changedetectionio/blueprint/rss/single_watch.py

@@ -0,0 +1,115 @@
+
+
+def construct_single_watch_routes(rss_blueprint, datastore):
+    """
+    Construct RSS feed routes for single watches.
+
+    Args:
+        rss_blueprint: The Flask blueprint to add routes to
+        datastore: The ChangeDetectionStore instance
+    """
+
+    @rss_blueprint.route("/watch/<string:uuid>", methods=['GET'])
+    def rss_single_watch(uuid):
+        import time
+
+        from flask import make_response, request
+        from feedgen.feed import FeedGenerator
+        from loguru import logger
+
+        from . import RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT
+        from ._util import (validate_rss_token, get_rss_template, get_watch_label,
+                           build_notification_context, render_notification,
+                           populate_feed_entry, add_watch_categories)
+        from ...notification_service import NotificationService
+
+        """
+        Display the most recent changes for a single watch as RSS feed.
+        Returns RSS XML with multiple entries showing diffs between consecutive snapshots.
+        The number of entries is controlled by the rss_diff_length setting.
+        """
+        now = time.time()
+
+        # Validate token
+        is_valid, error = validate_rss_token(datastore, request)
+        if not is_valid:
+            return error
+
+        rss_content_format = datastore.data['settings']['application'].get('rss_content_format')
+
+        # Get the watch by UUID
+        watch = datastore.data['watching'].get(uuid)
+        if not watch:
+            return f"Watch with UUID {uuid} not found", 404
+
+        # Check if watch has at least 2 history snapshots
+        dates = list(watch.history.keys())
+        if len(dates) < 2:
+            return f"Watch {uuid} does not have enough history snapshots to show changes (need at least 2)", 400
+
+        # Add uuid to watch for proper functioning
+        watch['uuid'] = uuid
+
+        # Get the number of diffs to include (default: 5)
+        rss_diff_length = datastore.data['settings']['application'].get('rss_diff_length', 5)
+
+        # Calculate how many diffs we can actually show (limited by available history)
+        # We need at least 2 snapshots to create 1 diff
+        max_possible_diffs = len(dates) - 1
+        num_diffs = min(rss_diff_length, max_possible_diffs) if rss_diff_length > 0 else max_possible_diffs
+
+        # Create RSS feed
+        fg = FeedGenerator()
+
+        # Set title: use "label (url)" if label differs from url, otherwise just url
+        watch_url = watch.get('url', '')
+        watch_label = get_watch_label(datastore, watch)
+
+        if watch_label != watch_url:
+            feed_title = f'changedetection.io - {watch_label} ({watch_url})'
+        else:
+            feed_title = f'changedetection.io - {watch_url}'
+
+        fg.title(feed_title)
+        fg.description('Changes')
+        fg.link(href='https://changedetection.io')
+
+        # Loop through history and create RSS entries for each diff
+        # Add entries in reverse order because feedgen reverses them
+        # This way, the newest change appears first in the final RSS
+
+        notification_service = NotificationService(datastore=datastore, notification_q=False)
+        for i in range(num_diffs - 1, -1, -1):
+            # Calculate indices for this diff (working backwards from newest)
+            # i=0: compare dates[-2] to dates[-1] (most recent change)
+            # i=1: compare dates[-3] to dates[-2] (previous change)
+            # etc.
+            date_index_to = -(i + 1)
+            date_index_from = -(i + 2)
+            timestamp_to = dates[date_index_to]
+            timestamp_from = dates[date_index_from]
+
+            # Get template and build notification context
+            n_body_template = get_rss_template(datastore, watch, rss_content_format,
+                                               RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT)
+
+            n_object = build_notification_context(watch, timestamp_from, timestamp_to,
+                                                 watch_label, n_body_template, rss_content_format)
+
+            # Render notification with date indices
+            res = render_notification(n_object, notification_service, watch, datastore,
+                                     date_index_from, date_index_to)
+
+            # Create and populate feed entry
+            guid = f"{watch['uuid']}/{timestamp_to}"
+            fe = fg.add_entry()
+            title_suffix = f"Change @ {res['original_context']['change_datetime']}"
+            populate_feed_entry(fe, watch, res.get('body', ''), guid, timestamp_to,
+                              link={'href': watch.get('url')}, title_suffix=title_suffix)
+            add_watch_categories(fe, watch, datastore)
+
+        response = make_response(fg.rss_str())
+        response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8')
+        logger.debug(f"RSS Single watch built in {time.time()-now:.2f}s")
+
+        return response

changedetectionio/blueprint/rss/tag.py

@@ -0,0 +1,98 @@
+def construct_tag_routes(rss_blueprint, datastore):
+    """
+    Construct RSS feed routes for tags.
+
+    Args:
+        rss_blueprint: The Flask blueprint to add routes to
+        datastore: The ChangeDetectionStore instance
+    """
+
+    @rss_blueprint.route("/tag/<string:tag_uuid>", methods=['GET'])
+    def rss_tag_feed(tag_uuid):
+
+        from flask import make_response, request, url_for
+        from feedgen.feed import FeedGenerator
+
+        from . import RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT
+        from ._util import (validate_rss_token, generate_watch_guid, get_rss_template,
+                           get_watch_label, build_notification_context, render_notification,
+                           populate_feed_entry, add_watch_categories)
+        from ...notification_service import NotificationService
+
+        """
+        Display an RSS feed for all unviewed watches that belong to a specific tag.
+        Returns RSS XML with entries for each unviewed watch with sufficient history.
+        """
+        # Validate token
+        is_valid, error = validate_rss_token(datastore, request)
+        if not is_valid:
+            return error
+
+        rss_content_format = datastore.data['settings']['application'].get('rss_content_format')
+
+        # Verify tag exists
+        tag = datastore.data['settings']['application'].get('tags', {}).get(tag_uuid)
+        if not tag:
+            return f"Tag with UUID {tag_uuid} not found", 404
+
+        tag_title = tag.get('title', 'Unknown Tag')
+
+        # Create RSS feed
+        fg = FeedGenerator()
+        fg.title(f'changedetection.io - {tag_title}')
+        fg.description(f'Changes for watches tagged with {tag_title}')
+        fg.link(href='https://changedetection.io')
+        notification_service = NotificationService(datastore=datastore, notification_q=False)
+        # Find all watches with this tag
+        for uuid, watch in datastore.data['watching'].items():
+            #@todo  This is wrong, it needs to sort by most recently changed and then limit it  datastore.data['watching'].items().sorted(?)
+            # So get all watches in this tag then sort
+
+            # Skip if watch doesn't have this tag
+            if tag_uuid not in watch.get('tags', []):
+                continue
+
+            # Skip muted watches if configured
+            if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):
+                continue
+
+            # Check if watch has at least 2 history snapshots
+            dates = list(watch.history.keys())
+            if len(dates) < 2:
+                continue
+
+            # Only include unviewed watches
+            if not watch.viewed:
+                # Add uuid to watch for proper functioning
+                watch['uuid'] = uuid
+
+                # Include a link to the diff page
+                diff_link = {'href': url_for('ui.ui_views.diff_history_page', uuid=watch['uuid'], _external=True)}
+
+                # Get watch label
+                watch_label = get_watch_label(datastore, watch)
+
+                # Get template and build notification context
+                timestamp_to = dates[-1]
+                timestamp_from = dates[-2]
+
+                # Generate GUID for this entry
+                guid = generate_watch_guid(watch, timestamp_to)
+                n_body_template = get_rss_template(datastore, watch, rss_content_format,
+                                                   RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT)
+
+                n_object = build_notification_context(watch, timestamp_from, timestamp_to,
+                                                     watch_label, n_body_template, rss_content_format)
+
+                # Render notification
+                res = render_notification(n_object, notification_service, watch, datastore)
+
+                # Create and populate feed entry
+                fe = fg.add_entry()
+                title_suffix = f"Change @ {res['original_context']['change_datetime']}"
+                populate_feed_entry(fe, watch, res['body'], guid, timestamp_to, link=diff_link, title_suffix=title_suffix)
+                add_watch_categories(fe, watch, datastore)
+
+        response = make_response(fg.rss_str())
+        response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8')
+        return response

changedetectionio/blueprint/settings/templates/settings.html

@@ -2,7 +2,7 @@
 
 {% block content %}
 {% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, render_ternary_field, render_fieldlist_with_inline_errors %}
-{% from '_common_fields.html' import render_common_settings_form %}
+{% from '_common_fields.html' import render_common_settings_form, show_token_placeholders %}
 <script>
     const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="global-settings")}}";
 {% if emailprefix %}
@@ -24,7 +24,8 @@
             <li class="tab"><a href="#filters">Global Filters</a></li>
             <li class="tab"><a href="#ui-options">UI Options</a></li>
             <li class="tab"><a href="#api">API</a></li>
-            <li class="tab"><a href="#timedate">Time &amp Date</a></li>
+            <li class="tab"><a href="#rss">RSS</a></li>
+            <li class="tab"><a href="#timedate">Time &amp; Date</a></li>
             <li class="tab"><a href="#proxies">CAPTCHA &amp; Proxies</a></li>
         </ul>
     </div>
@@ -43,10 +44,6 @@
                                 </div>
                         </div>
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
-                        <span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span>
-                    </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class="filter_failure_notification_threshold_attempts") }}
                         <span class="pure-form-message-inline">After this many consecutive times that the CSS/xPath filter is missing, send a notification
@@ -69,34 +66,19 @@
 
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.shared_diff_access, class="shared_diff_access") }}
-                        <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page)
+                        <span class="pure-form-message-inline">Allow access to the watch change history page when password is enabled (Good for sharing the diff page)
                         </span>
                     </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
                         <span class="pure-form-message-inline">When a request returns no content, or the HTML does not contain any text, is this considered a change?</span>
                     </div>
-                    <div class="grey-form-border">
-                        <div class="pure-control-group">
-                            {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }}
-                        </div>
-                        <div class="pure-control-group">
-                            {{ render_field(form.application.form.rss_content_format) }}
-                            <span class="pure-form-message-inline">Love RSS? Does your reader support HTML? Set it here</span>
-                        </div>
-                        <div class="pure-control-group">
-                            {{ render_checkbox_field(form.application.form.rss_reader_mode) }}
-                            <span class="pure-form-message-inline">Transforms RSS/RDF feed watches into beautiful text only</span>
-                        </div>
-                    </div>
                 </fieldset>
             </div>
 
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
-                    <div class="field-group">
-                        {{ render_common_settings_form(form.application.form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
-                    </div>
+                    {{ render_common_settings_form(form.application.form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
                 </fieldset>
                 <div class="pure-control-group" id="notification-base-url">
                     {{ render_field(form.application.form.base_url, class="m-d") }}
@@ -131,9 +113,13 @@
                     <span class="pure-form-message-inline">Number of concurrent workers to process watches. More workers = faster processing but higher memory usage.<br>
                     Currently running: <strong>{{ worker_info.count }}</strong> operational {{ worker_info.type }} workers{% if worker_info.active_workers > 0 %} ({{ worker_info.active_workers }} actively processing){% endif %}.</span>
                 </div>
+                <div class="pure-control-group">
+                    {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }}
+                    <span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span>
+                </div>
                 <div class="pure-control-group">
                     {{ render_field(form.requests.form.timeout) }}
-                    <span class="pure-form-message-inline">For regular plain requests (not chrome based), maximum number of seconds until timeout, 1-999.<br>
+                    <span class="pure-form-message-inline">For regular plain requests (not chrome based), maximum number of seconds until timeout, 1-999.</span><br>
                 </div>
                 <div class="pure-control-group inline-radio">
                     {{ render_field(form.requests.form.default_ua) }}
@@ -224,25 +210,55 @@ nav
                         <a id="chrome-extension-link"
                            title="Try our new Chrome Extension!"
                            href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop">
-                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='google-chrome-icon.png') }}" alt="Chrome">
+                            <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='google-chrome-icon.png') }}" >
                             Chrome Webstore
                         </a>
                     </p>
                 </div>
             </div>
-            <div class="tab-pane-inner" id="timedate">
+            <div class="tab-pane-inner" id="rss">
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }}
+                </div>
+                <div class="pure-control-group">
+                    {{ render_field(form.application.form.rss_diff_length) }}
+                    <span class="pure-form-message-inline">Maximum number of history snapshots to include in the watch specific RSS feed.</span>
+                </div>
+                <div class="pure-control-group">
+                    {{ render_checkbox_field(form.application.form.rss_reader_mode) }}
+                    <span class="pure-form-message-inline">For watching other RSS feeds - When watching RSS/Atom feeds, convert them into clean text for better change detection.</span>
+                </div>
+                <div class="pure-control-group grey-form-border">
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.rss_content_format) }}
+                        <span class="pure-form-message-inline">Does your reader support HTML? Set it here</span>
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.rss_template_type) }}
+                        <span class="pure-form-message-inline">'System default' for the same template for all items, or re-use your "Notification Body" as the template.</span>
+                    </div>
+                    <div>
+                       {{ render_field(form.application.form.rss_template_override) }}
+                        {{ show_token_placeholders(extra_notification_token_placeholder_info=extra_notification_token_placeholder_info, suffix="-rss") }}
+                    </div>
+                </div>
+                <br>
+
+
+            </div>
+                <div class="tab-pane-inner" id="timedate">
                 <div class="pure-control-group">
                     Ensure the settings below are correct, they are used to manage the time schedule for checking your web page watches.
                 </div>
                 <div class="pure-control-group">
-                    <p><strong>UTC Time &amp Date from Server:</strong> <span id="utc-time" >{{ utc_time }}</span></p>
-                    <p><strong>Local Time &amp Date in Browser:</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p>
-                    <p>
+                    <p><strong>UTC Time &amp; Date from Server:</strong> <span id="utc-time" >{{ utc_time }}</span></p>
+                    <p><strong>Local Time &amp; Date in Browser:</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p>
+                    <div>
                        {{ render_field(form.application.form.scheduler_timezone_default) }}
                         <datalist id="timezones" style="display: none;">
                             {%- for timezone in available_timezones -%}<option value="{{ timezone }}">{{ timezone }}</option>{%- endfor -%}
                         </datalist>
-                    </p>
+                    </div>
                 </div>
             </div>
             <div class="tab-pane-inner" id="ui-options">
@@ -311,7 +327,7 @@ nav
                     </div>
                 </div>
 
-               <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
+                <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.</p>
 
                 <div class="pure-control-group" id="extra-proxies-setting">
                 {{ render_fieldlist_with_inline_errors(form.requests.form.extra_proxies) }}

changedetectionio/blueprint/tags/__init__.py

@@ -21,9 +21,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         tag_count = Counter(tag for watch in datastore.data['watching'].values() if watch.get('tags') for tag in watch['tags'])
 
         output = render_template("groups-overview.html",
+                                 app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),
                                  available_tags=sorted_tags,
                                  form=add_form,
-                                 tag_count=tag_count
+                                 tag_count=tag_count,
                                  )
 
         return output
@@ -149,9 +150,9 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             included_content = template.render(**template_args)
 
         output = render_template("edit-tag.html",
-                                 settings_application=datastore.data['settings']['application'],
-                                 extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,
                                  extra_form_content=included_content,
+                                 extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,
+                                 settings_application=datastore.data['settings']['application'],
                                  **template_args
                                  )
 

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -52,6 +52,7 @@
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>
                     <a class="pure-button pure-button-primary" href="{{ url_for('tags.unlink', uuid=uuid) }}" title="Keep the tag but unlink any watches">Unlink</a>
+                    <a href="{{ url_for('rss.rss_tag_feed', tag_uuid=uuid, token=app_rss_token)}}"><img alt="RSS Feed for this watch" style="padding-left: 1em;" src="{{url_for('static_content', group='images', filename='generic_feed-icon.svg')}}" height="15"></a>
                 </td>
             </tr>
             {% endfor %}

changedetectionio/blueprint/ui/edit.py

@@ -236,7 +236,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
             # Import the global plugin system
             from changedetectionio.pluggy_interface import collect_ui_edit_stats_extras
-            
+            app_rss_token = datastore.data['settings']['application'].get('rss_access_token'),
             template_args = {
                 'available_processors': processors.available_processors(),
                 'available_timezones': sorted(available_timezones()),
@@ -252,6 +252,11 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
                 'has_special_tag_options': _watch_has_tag_options_set(watch=watch),
                 'jq_support': jq_support,
                 'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False),
+                'app_rss_token': app_rss_token,
+                'rss_uuid_feed' : {
+                    'label': watch.label,
+                    'url': url_for('rss.rss_single_watch', uuid=watch['uuid'], token=app_rss_token)
+                },
                 'settings_application': datastore.data['settings']['application'],
                 'system_has_playwright_configured': os.getenv('PLAYWRIGHT_DRIVER_URL'),
                 'system_has_webdriver_configured': os.getenv('WEBDRIVER_URL'),

changedetectionio/blueprint/ui/notification.py

@@ -2,7 +2,6 @@ from flask import Blueprint, request, make_response
 import random
 from loguru import logger
 
-from changedetectionio.notification_service import NotificationContextData
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
 
@@ -15,7 +14,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @notification_blueprint.route("/notification/send-test/", methods=['POST'])
     @login_optionally_required
     def ajax_callback_send_notification_test(watch_uuid=None):
-
+        from changedetectionio.notification_service import NotificationContextData, set_basic_notification_vars
         # Watch_uuid could be unset in the case it`s used in tag editor, global settings
         import apprise
         from changedetectionio.notification.handler import process_notification
@@ -39,11 +38,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             return make_response("Error: You must have atleast one watch configured for 'test notification' to work", 400)
 
         watch = datastore.data['watching'].get(watch_uuid)
-
-        notification_urls = None
-
-        if request.form.get('notification_urls'):
-            notification_urls = request.form['notification_urls'].strip().splitlines()
+        notification_urls = request.form.get('notification_urls','').strip().splitlines()
 
         if not notification_urls:
             logger.debug("Test notification - Trying by group/tag in the edit form if available")
@@ -81,6 +76,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             # Only use if present, if not set in n_object it should use the default system value
             if 'notification_format' in request.form and request.form['notification_format'].strip():
                 n_object['notification_format'] = request.form.get('notification_format', '').strip()
+            else:
+                n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
 
             if 'notification_title' in request.form and request.form['notification_title'].strip():
                 n_object['notification_title'] = request.form.get('notification_title', '').strip()
@@ -97,7 +94,28 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 n_object['notification_body'] = "Test body"
 
             n_object['as_async'] = False
-            n_object.update(watch.extra_notification_token_values())
+
+            #  Same like in notification service, should be refactored
+            dates = list(watch.history.keys())
+            trigger_text = ''
+            snapshot_contents = ''
+
+            # Could be called as a 'test notification' with only 1 snapshot available
+            prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
+            current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
+
+            if len(dates) > 1:
+                prev_snapshot = watch.get_history_snapshot(timestamp=dates[-2])
+                current_snapshot = watch.get_history_snapshot(timestamp=dates[-1])
+
+            n_object.update(set_basic_notification_vars(snapshot_contents=snapshot_contents,
+                                                        current_snapshot=current_snapshot,
+                                                        prev_snapshot=prev_snapshot,
+                                                        watch=watch,
+                                                        triggered_text=trigger_text,
+                                                        timestamp_changed=dates[-1] if dates else None))
+
+
             sent_obj = process_notification(n_object, datastore)
 
         except Exception as e:

changedetectionio/blueprint/ui/templates/edit.html

@@ -476,6 +476,7 @@ Math: {{ 1 + 1 }}") }}
                        class="pure-button button-error">Clear History</a>{% endif %}
                     <a href="{{url_for('ui.form_clone', uuid=uuid)}}"
                        class="pure-button">Clone &amp; Edit</a>
+                 <a href="{{ url_for('rss.rss_single_watch', uuid=uuid, token=app_rss_token)}}"><img alt="RSS Feed for this watch" style="padding: .5em 1em;" src="{{url_for('static_content', group='images', filename='generic_feed-icon.svg')}}" height="15"></a>
                 </div>
             </div>
         </form>

changedetectionio/blueprint/ui/views.py

@@ -47,7 +47,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
             try:
                 versions = list(watch.history.keys())
-                content = watch.get_history_snapshot(timestamp)
+                content = watch.get_history_snapshot(timestamp=timestamp)
 
                 triggered_line_numbers = html_tools.strip_ignore_text(content=content,
                                                                       wordlist=watch['trigger_text'],

changedetectionio/blueprint/watchlist/templates/watch-overview.html

@@ -155,7 +155,7 @@ document.addEventListener('DOMContentLoaded', function() {
                     <div class="flex-wrapper">
                     {% if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] %}
                         <div>{# A page might have hundreds of these images, set IMG options for lazy loading, don't set SRC if we dont have it so it doesnt fetch the placeholder'  #}
-                            <img alt="Favicon thumbnail" class="favicon" loading="lazy" decoding="async" fetchpriority="low" {% if favicon %} src="{{url_for('static_content', group='favicon', filename=watch.uuid)}}" {% else %} src='data:image/svg+xml;utf8,%3Csvg xmlns="http://www.w3.org/2000/svg" width="7.087" height="7.087" viewBox="0 0 7.087 7.087"%3E%3Ccircle cx="3.543" cy="3.543" r="3.279" stroke="%23e1e1e1" stroke-width="0.45" fill="none" opacity="0.74"/%3E%3C/svg%3E' {%  endif %} />
+                            <img alt="Favicon thumbnail" class="favicon" loading="lazy" decoding="async" fetchpriority="low" {% if favicon %} src="{{url_for('static_content', group='favicon', filename=watch.uuid)}}" {% else %} src='data:image/svg+xml;utf8,%3Csvg xmlns="http://www.w3.org/2000/svg" width="7.087" height="7.087" viewBox="0 0 7.087 7.087"%3E%3Ccircle cx="3.543" cy="3.543" r="3.279" stroke="%23e1e1e1" stroke-width="0.45" fill="none" opacity="0.74"/%3E%3C/svg%3E' {%  endif %} >
                         </div>
                     {%  endif %}
                         <div>

changedetectionio/conditions/plugins/wordcount_plugin.py

@@ -14,7 +14,7 @@ def count_words_in_history(watch, incoming_text=None):
         elif watch.history.keys():
             # When called from UI extras to count latest snapshot
             latest_key = list(watch.history.keys())[-1]
-            latest_content = watch.get_history_snapshot(latest_key)
+            latest_content = watch.get_history_snapshot(timestamp=latest_key)
             return len(latest_content.split())
         return 0
     except Exception as e:

changedetectionio/content_fetchers/playwright.py

@@ -139,7 +139,7 @@ class fetcher(Fetcher):
         content = await self.page.content()
         destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
         logger.debug(f"Saving step HTML to {destination}")
-        with open(destination, 'w') as f:
+        with open(destination, 'w', encoding='utf-8') as f:
             f.write(content)
 
     async def run(self,

changedetectionio/content_fetchers/requests.py

@@ -1,6 +1,7 @@
 from loguru import logger
 import hashlib
 import os
+import re
 import asyncio
 from changedetectionio import strtobool
 from changedetectionio.content_fetchers.exceptions import BrowserStepsInUnsupportedFetcher, EmptyReply, Non200ErrorCodeReceived
@@ -76,9 +77,22 @@ class fetcher(Fetcher):
         if not is_binary:
             # Don't run this for PDF (and requests identified as binary) takes a _long_ time
             if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
-                encoding = chardet.detect(r.content)['encoding']
-                if encoding:
-                    r.encoding = encoding
+                # For XML/RSS feeds, check the XML declaration for encoding attribute
+                # This is more reliable than chardet which can misdetect UTF-8 as MacRoman
+                content_type = r.headers.get('content-type', '').lower()
+                if 'xml' in content_type or 'rss' in content_type:
+                    # Look for <?xml version="1.0" encoding="UTF-8"?>
+                    xml_encoding_match = re.search(rb'<\?xml[^>]+encoding=["\']([^"\']+)["\']', r.content[:200])
+                    if xml_encoding_match:
+                        r.encoding = xml_encoding_match.group(1).decode('ascii')
+                    else:
+                        # Default to UTF-8 for XML if no encoding found
+                        r.encoding = 'utf-8'
+                else:
+                    # For other content types, use chardet
+                    encoding = chardet.detect(r.content)['encoding']
+                    if encoding:
+                        r.encoding = encoding
 
         self.headers = r.headers
 

changedetectionio/flask_app.py

@@ -101,12 +101,12 @@ def init_app_secret(datastore_path):
     path = os.path.join(datastore_path, "secret.txt")
 
     try:
-        with open(path, "r") as f:
+        with open(path, "r", encoding='utf-8') as f:
             secret = f.read()
 
     except FileNotFoundError:
         import secrets
-        with open(path, "w") as f:
+        with open(path, "w", encoding='utf-8') as f:
             secret = secrets.token_hex(32)
             f.write(secret)
 
@@ -680,7 +680,7 @@ def notification_runner():
                 # At the moment only one thread runs (single runner)
                 n_object = notification_q.get(block=False)
             except queue.Empty:
-                time.sleep(1)
+                app.config.exit.wait(1)
 
             else:
 
@@ -775,7 +775,7 @@ def ticker_thread_check_time_launch_checks():
         # Re #438 - Don't place more watches in the queue to be checked if the queue is already large
         while update_q.qsize() >= 2000:
             logger.warning(f"Recheck watches queue size limit reached ({MAX_QUEUE_SIZE}), skipping adding more items")
-            time.sleep(3)
+            app.config.exit.wait(10.0)
 
 
         recheck_time_system_seconds = int(datastore.threshold_seconds)
@@ -794,15 +794,19 @@ def ticker_thread_check_time_launch_checks():
 
             # @todo - Maybe make this a hook?
             # Time schedule limit - Decide between watch or global settings
+            scheduler_source = None
             if watch.get('time_between_check_use_default'):
                 time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})
-                logger.trace(f"{uuid} Time scheduler - Using system/global settings")
+                scheduler_source = 'system/global settings'
+
             else:
                 time_schedule_limit = watch.get('time_schedule_limit')
-                logger.trace(f"{uuid} Time scheduler - Using watch settings (not global settings)")
+                scheduler_source = 'watch'
+
             tz_name = datastore.data['settings']['application'].get('scheduler_timezone_default', os.getenv('TZ', 'UTC').strip())
 
             if time_schedule_limit and time_schedule_limit.get('enabled'):
+                logger.trace(f"{uuid} Time scheduler - Using scheduler settings from {scheduler_source}")
                 try:
                     result = is_within_schedule(time_schedule_limit=time_schedule_limit,
                                                 default_tz=tz_name
@@ -814,6 +818,7 @@ def ticker_thread_check_time_launch_checks():
                     logger.error(
                         f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
                     return False
+
             # If they supplied an individual entry minutes to threshold.
             threshold = recheck_time_system_seconds if watch.get('time_between_check_use_default') else watch.threshold_seconds()
 
@@ -868,8 +873,5 @@ def ticker_thread_check_time_launch_checks():
                     # Reset for next time
                     watch.jitter_seconds = 0
 
-        # Wait before checking the list again - saves CPU
-        time.sleep(1)
-
         # Should be low so we can break this out in testing
         app.config.exit.wait(1)

changedetectionio/forms.py

@@ -3,7 +3,7 @@ import re
 from loguru import logger
 from wtforms.widgets.core import TimeInput
 
-from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES
+from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES, RSS_TEMPLATE_TYPE_OPTIONS, RSS_TEMPLATE_HTML_DEFAULT
 from changedetectionio.conditions.form import ConditionFormRow
 from changedetectionio.notification_service import NotificationContextData
 from changedetectionio.strtobool import strtobool
@@ -503,7 +503,9 @@ class ValidateJinja2Template(object):
             jinja2_env = create_jinja_env(loader=BaseLoader)
 
             # Add notification tokens for validation
-            jinja2_env.globals.update(NotificationContextData())
+            static_token_placeholders = NotificationContextData()
+            static_token_placeholders.set_random_for_validation()
+            jinja2_env.globals.update(static_token_placeholders)
             if hasattr(field, 'extra_notification_tokens'):
                 jinja2_env.globals.update(field.extra_notification_tokens)
 
@@ -998,7 +1000,9 @@ class globalSettingsApplicationForm(commonSettingsForm):
                               validators=[validators.NumberRange(min=0,
                                                                  message="Should be atleast zero (disabled)")])
 
-    rss_content_format = SelectField('RSS Content format', choices=RSS_FORMAT_TYPES)
+    rss_content_format = SelectField('RSS Content format', choices=list(RSS_FORMAT_TYPES.items()))
+    rss_template_type = SelectField('RSS <description> body built from', choices=list(RSS_TEMPLATE_TYPE_OPTIONS.items()))
+    rss_template_override = TextAreaField('RSS "System default" template override', render_kw={"rows": "5", "placeholder": RSS_TEMPLATE_HTML_DEFAULT}, validators=[validators.Optional(), ValidateJinja2Template()])
 
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
@@ -1007,8 +1011,10 @@ class globalSettingsApplicationForm(commonSettingsForm):
     rss_hide_muted_watches = BooleanField('Hide muted watches from RSS feed', default=True,
                                       validators=[validators.Optional()])
 
-    rss_reader_mode = BooleanField('RSS reader mode ', default=False,
-                                      validators=[validators.Optional()])
+    rss_reader_mode = BooleanField('Enable RSS reader mode ', default=False, validators=[validators.Optional()])
+    rss_diff_length = IntegerField(label='Number of changes to show in watch RSS feed',
+                                   render_kw={"style": "width: 5em;"},
+                                   validators=[validators.NumberRange(min=0, message="Should contain zero or more attempts")])
 
     filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
                                                                   render_kw={"style": "width: 5em;"},

changedetectionio/html_tools.py

@@ -172,99 +172,131 @@ def elementpath_tostring(obj):
     return str(obj)
 
 # Return str Utf-8 of matched rules
-def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False, is_rss=False):
+def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False, is_xml=False):
+    """
+
+    :param xpath_filter:
+    :param html_content:
+    :param append_pretty_line_formatting:
+    :param is_xml: set to true if is XML or is RSS (RSS is XML)
+    :return:
+    """
     from lxml import etree, html
     import elementpath
     # xpath 2.0-3.1
     from elementpath.xpath3 import XPath3Parser
 
     parser = etree.HTMLParser()
-    if is_rss:
-        # So that we can keep CDATA for cdata_in_document_to_text() to process
-        parser = etree.XMLParser(strip_cdata=False)
-
-    tree = html.fromstring(bytes(html_content, encoding='utf-8'), parser=parser)
-    html_block = ""
-
-    # Build namespace map for XPath queries
-    namespaces = {'re': 'http://exslt.org/regular-expressions'}
-
-    # Handle default namespace in documents (common in RSS/Atom feeds, but can occur in any XML)
-    # XPath spec: unprefixed element names have no namespace, not the default namespace
-    # Solution: Register the default namespace with empty string prefix in elementpath
-    # This is primarily for RSS/Atom feeds but works for any XML with default namespace
-    if hasattr(tree, 'nsmap') and tree.nsmap and None in tree.nsmap:
-        # Register the default namespace with empty string prefix for elementpath
-        # This allows //title to match elements in the default namespace
-        namespaces[''] = tree.nsmap[None]
-
-    r = elementpath.select(tree, xpath_filter.strip(), namespaces=namespaces, parser=XPath3Parser)
-    #@note: //title/text() now works with default namespaces (fixed by registering '' prefix)
-    #@note: //title/text() wont work where <title>CDATA.. (use cdata_in_document_to_text first)
-
-    if type(r) != list:
-        r = [r]
-
-    for element in r:
-        # When there's more than 1 match, then add the suffix to separate each line
-        # And where the matched result doesn't include something that will cause Inscriptis to add a newline
-        # (This way each 'match' reliably has a new-line in the diff)
-        # Divs are converted to 4 whitespaces by inscriptis
-        if append_pretty_line_formatting and len(html_block) and (not hasattr( element, 'tag' ) or not element.tag in (['br', 'hr', 'div', 'p'])):
-            html_block += TEXT_FILTER_LIST_LINE_SUFFIX
-
-        if type(element) == str:
-            html_block += element
-        elif issubclass(type(element), etree._Element) or issubclass(type(element), etree._ElementTree):
-            html_block += etree.tostring(element, pretty_print=True).decode('utf-8')
+    tree = None
+    try:
+        if is_xml:
+            # So that we can keep CDATA for cdata_in_document_to_text() to process
+            parser = etree.XMLParser(strip_cdata=False)
+            # For XML/RSS content, use etree.fromstring to properly handle XML declarations
+            tree = etree.fromstring(html_content.encode('utf-8') if isinstance(html_content, str) else html_content, parser=parser)
         else:
-            html_block += elementpath_tostring(element)
+            tree = html.fromstring(html_content, parser=parser)
+        html_block = ""
 
-    return html_block
+        # Build namespace map for XPath queries
+        namespaces = {'re': 'http://exslt.org/regular-expressions'}
+
+        # Handle default namespace in documents (common in RSS/Atom feeds, but can occur in any XML)
+        # XPath spec: unprefixed element names have no namespace, not the default namespace
+        # Solution: Register the default namespace with empty string prefix in elementpath
+        # This is primarily for RSS/Atom feeds but works for any XML with default namespace
+        if hasattr(tree, 'nsmap') and tree.nsmap and None in tree.nsmap:
+            # Register the default namespace with empty string prefix for elementpath
+            # This allows //title to match elements in the default namespace
+            namespaces[''] = tree.nsmap[None]
+
+        r = elementpath.select(tree, xpath_filter.strip(), namespaces=namespaces, parser=XPath3Parser)
+        #@note: //title/text() now works with default namespaces (fixed by registering '' prefix)
+        #@note: //title/text() wont work where <title>CDATA.. (use cdata_in_document_to_text first)
+
+        if type(r) != list:
+            r = [r]
+
+        for element in r:
+            # When there's more than 1 match, then add the suffix to separate each line
+            # And where the matched result doesn't include something that will cause Inscriptis to add a newline
+            # (This way each 'match' reliably has a new-line in the diff)
+            # Divs are converted to 4 whitespaces by inscriptis
+            if append_pretty_line_formatting and len(html_block) and (not hasattr( element, 'tag' ) or not element.tag in (['br', 'hr', 'div', 'p'])):
+                html_block += TEXT_FILTER_LIST_LINE_SUFFIX
+
+            if type(element) == str:
+                html_block += element
+            elif issubclass(type(element), etree._Element) or issubclass(type(element), etree._ElementTree):
+                # Use 'xml' method for RSS/XML content, 'html' for HTML content
+                # parser will be XMLParser if we detected XML content
+                method = 'xml' if (is_xml or isinstance(parser, etree.XMLParser)) else 'html'
+                html_block += etree.tostring(element, pretty_print=True, method=method, encoding='unicode')
+            else:
+                html_block += elementpath_tostring(element)
+
+        return html_block
+    finally:
+        # Explicitly clear the tree to free memory
+        # lxml trees can hold significant memory, especially with large documents
+        if tree is not None:
+            tree.clear()
 
 # Return str Utf-8 of matched rules
 # 'xpath1:'
-def xpath1_filter(xpath_filter, html_content, append_pretty_line_formatting=False, is_rss=False):
+def xpath1_filter(xpath_filter, html_content, append_pretty_line_formatting=False, is_xml=False):
     from lxml import etree, html
 
     parser = None
-    if is_rss:
-        # So that we can keep CDATA for cdata_in_document_to_text() to process
-        parser = etree.XMLParser(strip_cdata=False)
-
-    tree = html.fromstring(bytes(html_content, encoding='utf-8'), parser=parser)
-    html_block = ""
-
-    # Build namespace map for XPath queries
-    namespaces = {'re': 'http://exslt.org/regular-expressions'}
-
-    # NOTE: lxml's native xpath() does NOT support empty string prefix for default namespace
-    # For documents with default namespace (RSS/Atom feeds), users must use:
-    #   - local-name(): //*[local-name()='title']/text()
-    #   - Or use xpath_filter (not xpath1_filter) which supports default namespaces
-    # XPath spec: unprefixed element names have no namespace, not the default namespace
-
-    r = tree.xpath(xpath_filter.strip(), namespaces=namespaces)
-    #@note: xpath1 (lxml) does NOT automatically handle default namespaces
-    #@note: Use //*[local-name()='element'] or switch to xpath_filter for default namespace support
-    #@note: //title/text() wont work where <title>CDATA.. (use cdata_in_document_to_text first)
-
-    for element in r:
-        # When there's more than 1 match, then add the suffix to separate each line
-        # And where the matched result doesn't include something that will cause Inscriptis to add a newline
-        # (This way each 'match' reliably has a new-line in the diff)
-        # Divs are converted to 4 whitespaces by inscriptis
-        if append_pretty_line_formatting and len(html_block) and (not hasattr(element, 'tag') or not element.tag in (['br', 'hr', 'div', 'p'])):
-            html_block += TEXT_FILTER_LIST_LINE_SUFFIX
-
-        # Some kind of text, UTF-8 or other
-        if isinstance(element, (str, bytes)):
-            html_block += element
+    tree = None
+    try:
+        if is_xml:
+            # So that we can keep CDATA for cdata_in_document_to_text() to process
+            parser = etree.XMLParser(strip_cdata=False)
+            # For XML/RSS content, use etree.fromstring to properly handle XML declarations
+            tree = etree.fromstring(html_content.encode('utf-8') if isinstance(html_content, str) else html_content, parser=parser)
         else:
-            # Return the HTML which will get parsed as text
-            html_block += etree.tostring(element, pretty_print=True).decode('utf-8')
+            tree = html.fromstring(html_content, parser=parser)
+        html_block = ""
 
-    return html_block
+        # Build namespace map for XPath queries
+        namespaces = {'re': 'http://exslt.org/regular-expressions'}
+
+        # NOTE: lxml's native xpath() does NOT support empty string prefix for default namespace
+        # For documents with default namespace (RSS/Atom feeds), users must use:
+        #   - local-name(): //*[local-name()='title']/text()
+        #   - Or use xpath_filter (not xpath1_filter) which supports default namespaces
+        # XPath spec: unprefixed element names have no namespace, not the default namespace
+
+        r = tree.xpath(xpath_filter.strip(), namespaces=namespaces)
+        #@note: xpath1 (lxml) does NOT automatically handle default namespaces
+        #@note: Use //*[local-name()='element'] or switch to xpath_filter for default namespace support
+        #@note: //title/text() wont work where <title>CDATA.. (use cdata_in_document_to_text first)
+
+        for element in r:
+            # When there's more than 1 match, then add the suffix to separate each line
+            # And where the matched result doesn't include something that will cause Inscriptis to add a newline
+            # (This way each 'match' reliably has a new-line in the diff)
+            # Divs are converted to 4 whitespaces by inscriptis
+            if append_pretty_line_formatting and len(html_block) and (not hasattr(element, 'tag') or not element.tag in (['br', 'hr', 'div', 'p'])):
+                html_block += TEXT_FILTER_LIST_LINE_SUFFIX
+
+            # Some kind of text, UTF-8 or other
+            if isinstance(element, (str, bytes)):
+                html_block += element
+            else:
+                # Return the HTML/XML which will get parsed as text
+                # Use 'xml' method for RSS/XML content, 'html' for HTML content
+                # parser will be XMLParser if we detected XML content
+                method = 'xml' if (is_xml or isinstance(parser, etree.XMLParser)) else 'html'
+                html_block += etree.tostring(element, pretty_print=True, method=method, encoding='unicode')
+
+        return html_block
+    finally:
+        # Explicitly clear the tree to free memory
+        # lxml trees can hold significant memory, especially with large documents
+        if tree is not None:
+            tree.clear()
 
 # Extract/find element
 def extract_element(find='title', html_content=''):

changedetectionio/model/App.py

@@ -1,6 +1,7 @@
 from os import getenv
+from copy import deepcopy
 
-from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES
+from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES, RSS_CONTENT_FORMAT_DEFAULT
 
 from changedetectionio.notification import (
     default_notification_body,
@@ -53,7 +54,10 @@ class model(dict):
                     'password': False,
                     'render_anchor_tag_content': False,
                     'rss_access_token': None,
-                    'rss_content_format': RSS_FORMAT_TYPES[0][0],
+                    'rss_content_format': RSS_CONTENT_FORMAT_DEFAULT,
+                    'rss_template_type': 'system_default',
+                    'rss_template_override': None,
+                    'rss_diff_length': 5,
                     'rss_hide_muted_watches': True,
                     'rss_reader_mode': False,
                     'scheduler_timezone_default': None,  # Default IANA timezone name
@@ -74,12 +78,13 @@ class model(dict):
 
     def __init__(self, *arg, **kw):
         super(model, self).__init__(*arg, **kw)
-        self.update(self.base_config)
+        # CRITICAL: deepcopy to avoid sharing mutable objects between instances
+        self.update(deepcopy(self.base_config))
 
 
 def parse_headers_from_text_file(filepath):
     headers = {}
-    with open(filepath, 'r') as f:
+    with open(filepath, 'r', encoding='utf-8') as f:
         for l in f.readlines():
             l = l.strip()
             if not l.startswith('#') and ':' in l:

changedetectionio/model/Watch.py

@@ -188,7 +188,7 @@ class model(watch_base):
         fname = os.path.join(self.watch_data_dir, "history.txt")
         if os.path.isfile(fname):
             logger.debug(f"Reading watch history index for {self.get('uuid')}")
-            with open(fname, "r") as f:
+            with open(fname, "r", encoding='utf-8') as f:
                 for i in f.readlines():
                     if ',' in i:
                         k, v = i.strip().split(',', 2)
@@ -276,9 +276,17 @@ class model(watch_base):
         # When the 'last viewed' timestamp is less than the oldest snapshot, return oldest
         return sorted_keys[-1]
 
-    def get_history_snapshot(self, timestamp):
+    def get_history_snapshot(self, timestamp=None, filepath=None):
+        """
+        Accepts either timestamp or filepath
+        :param timestamp:
+        :param filepath:
+        :return:
+        """
         import brotli
-        filepath = self.history[timestamp]
+
+        if not filepath:
+            filepath = self.history[timestamp]
 
         # See if a brotli versions exists and switch to that
         if not filepath.endswith('.br') and os.path.isfile(f"{filepath}.br"):
@@ -382,7 +390,7 @@ class model(watch_base):
         # Compare each lines (set) against each history text file (set) looking for something new..
         existing_history = set({})
         for k, v in self.history.items():
-            content = self.get_history_snapshot(k)
+            content = self.get_history_snapshot(filepath=v)
 
             if ignore_whitespace:
                 alist = set([line.translate(TRANSLATE_WHITESPACE_TABLE).lower() for line in content.splitlines()])
@@ -586,7 +594,7 @@ class model(watch_base):
         """Return the text saved from a previous request that resulted in a non-200 error"""
         fname = os.path.join(self.watch_data_dir, "last-error.txt")
         if os.path.isfile(fname):
-            with open(fname, 'r') as f:
+            with open(fname, 'r', encoding='utf-8') as f:
                 return f.read()
         return False
 
@@ -639,7 +647,7 @@ class model(watch_base):
         for k, fname in self.history.items():
             if os.path.isfile(fname):
                 if True:
-                    contents = self.get_history_snapshot(k)
+                    contents = self.get_history_snapshot(timestamp=k)
                     res = re.findall(regex, contents, re.MULTILINE)
                     if res:
                         if not csv_writer:
@@ -732,7 +740,7 @@ class model(watch_base):
             # If a previous attempt doesnt yet exist, just snarf the previous snapshot instead
             dates = list(self.history.keys())
             if len(dates):
-                return self.get_history_snapshot(dates[-1])
+                return self.get_history_snapshot(timestamp=dates[-1])
             else:
                 return ''
 

changedetectionio/notification/email_helpers.py

@@ -35,7 +35,7 @@ def as_monospaced_html_email(content: str, title: str) -> str:
 </head>
 <body style="-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%;">
   <pre role="article" aria-roledescription="email" lang="en"
-       style="font-family: monospace, 'Courier New', Courier; font-size: 0.8em;
+       style="font-family: monospace, 'Courier New', Courier; font-size: 0.9rem;
               white-space: pre-wrap; word-break: break-word;">{content}</pre>
 </body>
 </html>"""

changedetectionio/notification/handler.py

@@ -5,13 +5,15 @@ from apprise import NotifyFormat
 from loguru import logger
 from urllib.parse import urlparse
 from .apprise_plugin.assets import apprise_asset, APPRISE_AVATAR_URL
-from .apprise_plugin.custom_handlers import SUPPORTED_HTTP_METHODS
 from .email_helpers import as_monospaced_html_email
 from ..diff import HTML_REMOVED_STYLE, REMOVED_PLACEMARKER_OPEN, REMOVED_PLACEMARKER_CLOSED, ADDED_PLACEMARKER_OPEN, HTML_ADDED_STYLE, \
     ADDED_PLACEMARKER_CLOSED, CHANGED_INTO_PLACEMARKER_OPEN, CHANGED_INTO_PLACEMARKER_CLOSED, CHANGED_PLACEMARKER_OPEN, \
     CHANGED_PLACEMARKER_CLOSED, HTML_CHANGED_STYLE, HTML_CHANGED_INTO_STYLE
-from ..notification_service import NotificationContextData, CUSTOM_LINEBREAK_PLACEHOLDER
+import re
 
+from ..notification_service import NotificationContextData
+
+newline_re = re.compile(r'\r\n|\r|\n')
 
 
 def markup_text_links_to_html(body):
@@ -127,8 +129,66 @@ def apply_standard_markdown_to_body(n_body):
     return n_body
 
 
+def replace_placemarkers_in_text(text, url, requested_output_format):
+    """
+    Replace diff placemarkers in text based on the URL service type and requested output format.
+    Used for both notification title and body to ensure consistent placeholder replacement.
+
+    :param text: The text to process
+    :param url: The notification URL (to detect service type)
+    :param requested_output_format: The output format (html, htmlcolor, markdown, text, etc.)
+    :return: Processed text with placemarkers replaced
+    """
+    if not text:
+        return text
+
+    if url.startswith('tgram://'):
+        # Telegram only supports a limited subset of HTML
+        # Use strikethrough for removed content, bold for added content
+        text = text.replace(REMOVED_PLACEMARKER_OPEN, '<s>')
+        text = text.replace(REMOVED_PLACEMARKER_CLOSED, '</s>')
+        text = text.replace(ADDED_PLACEMARKER_OPEN, '<b>')
+        text = text.replace(ADDED_PLACEMARKER_CLOSED, '</b>')
+        # Handle changed/replaced lines (old → new)
+        text = text.replace(CHANGED_PLACEMARKER_OPEN, '<s>')
+        text = text.replace(CHANGED_PLACEMARKER_CLOSED, '</s>')
+        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, '<b>')
+        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, '</b>')
+    elif (url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks')
+          or url.startswith('https://discord.com/api')) and requested_output_format == 'html':
+        # Discord doesn't support HTML, use Discord markdown
+        text = apply_discord_markdown_to_body(n_body=text)
+    elif requested_output_format == 'htmlcolor':
+        # https://github.com/dgtlmoon/changedetection.io/issues/821#issuecomment-1241837050
+        text = text.replace(REMOVED_PLACEMARKER_OPEN, f'<span style="{HTML_REMOVED_STYLE}" role="deletion" aria-label="Removed text" title="Removed text">')
+        text = text.replace(REMOVED_PLACEMARKER_CLOSED, f'</span>')
+        text = text.replace(ADDED_PLACEMARKER_OPEN, f'<span style="{HTML_ADDED_STYLE}" role="insertion" aria-label="Added text" title="Added text">')
+        text = text.replace(ADDED_PLACEMARKER_CLOSED, f'</span>')
+        # Handle changed/replaced lines (old → new)
+        text = text.replace(CHANGED_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_STYLE}" role="note" aria-label="Changed text" title="Changed text">')
+        text = text.replace(CHANGED_PLACEMARKER_CLOSED, f'</span>')
+        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_INTO_STYLE}" role="note" aria-label="Changed into" title="Changed into">')
+        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'</span>')
+    elif requested_output_format == 'markdown':
+        # Markdown to HTML - Apprise will convert this to HTML
+        text = apply_standard_markdown_to_body(n_body=text)
+    else:
+        # plaintext, html, and default - use simple text markers
+        text = text.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')
+        text = text.replace(REMOVED_PLACEMARKER_CLOSED, '')
+        text = text.replace(ADDED_PLACEMARKER_OPEN, '(added) ')
+        text = text.replace(ADDED_PLACEMARKER_CLOSED, '')
+        text = text.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')
+        text = text.replace(CHANGED_PLACEMARKER_CLOSED, f'')
+        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')
+        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')
+
+    return text
+
 def apply_service_tweaks(url, n_body, n_title, requested_output_format):
 
+    logger.debug(f"Applying markup in '{requested_output_format}' mode")
+
     # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
     # Because different notifications may require different pre-processing, run each sequentially :(
     # 2000 bytes minus -
@@ -138,6 +198,12 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
     if not n_body or not n_body.strip():
         return url, n_body, n_title
 
+    # Normalize URL scheme to lowercase to prevent case-sensitivity issues
+    # e.g., "Discord://webhook" -> "discord://webhook", "TGRAM://bot123" -> "tgram://bot123"
+    scheme_separator_pos = url.find('://')
+    if scheme_separator_pos > 0:
+        url = url[:scheme_separator_pos].lower() + url[scheme_separator_pos:]
+
     # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
     parsed = urlparse(url)
     k = '?' if not parsed.query else '&'
@@ -149,24 +215,22 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
             and not url.startswith('put'):
         url += k + f"avatar_url={APPRISE_AVATAR_URL}"
 
+    # Replace placemarkers in title first (this was the missing piece causing the bug)
+    # Titles are ALWAYS plain text across all notification services (Discord embeds, Slack attachments,
+    # email Subject headers, etc.), so we always use 'text' format for title placemarker replacement
+    # Looking over apprise library it seems that all plugins only expect plain-text.
+    n_title = replace_placemarkers_in_text(n_title, url, 'text')
+
     if url.startswith('tgram://'):
         # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
         # re https://github.com/dgtlmoon/changedetection.io/issues/555
         # @todo re-use an existing library we have already imported to strip all non-allowed tags
         n_body = n_body.replace('<br>', '\n')
         n_body = n_body.replace('</br>', '\n')
-        n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\n')
+        n_body = newline_re.sub('\n', n_body)
 
-        # Use strikethrough for removed content, bold for added content
-        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '<s>')
-        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '</s>')
-        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '<b>')
-        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '</b>')
-        # Handle changed/replaced lines (old → new)
-        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, '<s>')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, '</s>')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, '<b>')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, '</b>')
+        # Replace placemarkers for body
+        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
 
         # real limit is 4096, but minus some for extra metadata
         payload_max_size = 3600
@@ -180,7 +244,7 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
         # Discord doesn't support HTML, replace <br> with newlines
         n_body = n_body.strip().replace('<br>', '\n')
         n_body = n_body.replace('</br>', '\n')
-        n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\n')
+        n_body = newline_re.sub('\n', n_body)
 
         # Don't replace placeholders or truncate here - let the custom Discord plugin handle it
         # The plugin will use embeds (6000 char limit across all embeds) if placeholders are present,
@@ -190,7 +254,7 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
         if requested_output_format == 'html':
             # No diff placeholders, use Discord markdown for any other formatting
             # Use Discord markdown: strikethrough for removed, bold for added
-            n_body = apply_discord_markdown_to_body(n_body=n_body)
+            n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
 
             # Apply 2000 char limit for plain content
             payload_max_size = 1700
@@ -201,40 +265,17 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
 
     # Is not discord/tgram and they want htmlcolor
     elif requested_output_format == 'htmlcolor':
-        # https://github.com/dgtlmoon/changedetection.io/issues/821#issuecomment-1241837050
-        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, f'<span style="{HTML_REMOVED_STYLE}" role="deletion" aria-label="Removed text" title="Removed text">')
-        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, f'</span>')
-        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, f'<span style="{HTML_ADDED_STYLE}" role="insertion" aria-label="Added text" title="Added text">')
-        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, f'</span>')
-        # Handle changed/replaced lines (old → new)
-        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_STYLE}" role="note" aria-label="Changed text" title="Changed text">')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'</span>')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_INTO_STYLE}" role="note" aria-label="Changed into" title="Changed into">')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'</span>')
-        n_body = n_body.replace('\n', f'{CUSTOM_LINEBREAK_PLACEHOLDER}\n')
+        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
+        n_body = newline_re.sub('<br>\n', n_body)
     elif requested_output_format == 'html':
-        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')
-        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '')
-        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '(added) ')
-        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')
-        n_body = n_body.replace('\n', f'{CUSTOM_LINEBREAK_PLACEHOLDER}\n')
+        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
+        n_body = newline_re.sub('<br>\n', n_body)
     elif requested_output_format == 'markdown':
         # Markdown to HTML - Apprise will convert this to HTML
-        n_body = apply_standard_markdown_to_body(n_body=n_body)
+        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
 
     else: #plaintext etc default
-        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')
-        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '')
-        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '(added) ')
-        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')
-        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')
-        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')
+        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
 
     return url, n_body, n_title
 
@@ -295,24 +336,18 @@ def process_notification(n_object: NotificationContextData, datastore):
     with (apprise.LogCapture(level=apprise.logging.DEBUG) as logs):
         for url in n_object['notification_urls']:
 
-            # Get the notification body from datastore
             n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
+            n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
 
             if n_object.get('markup_text_links_to_html_links'):
                 n_body = markup_text_links_to_html(body=n_body)
 
-            n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
-
             url = url.strip()
-            if url.startswith('#'):
-                logger.trace(f"Skipping commented out notification URL - {url}")
+            if not url or url.startswith('#'):
+                logger.debug(f"Skipping commented out or empty notification URL - '{url}'")
                 continue
 
-            if not url:
-                logger.warning(f"Process Notification: skipping empty notification URL.")
-                continue
-
-            logger.info(f">> Process Notification: AppRise notifying {url}")
+            logger.info(f">> Process Notification: AppRise start notifying '{url}'")
             url = jinja_render(template_str=url, **notification_parameters)
 
             # If it's a plaintext document, and they want HTML type email/alerts, so it needs to be escaped
@@ -353,29 +388,26 @@ def process_notification(n_object: NotificationContextData, datastore):
                     requested_output_format = NotifyFormat.HTML.value
                     apprise_input_format = NotifyFormat.HTML.value  # Changed from MARKDOWN to HTML
 
-                # Could have arrived at any stage, so we dont end up running .escape on it
-                if 'html' in requested_output_format:
-                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '<br>\r\n')
-                else:
-                    # texty types
-                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\r\n')
-
             else:
                 # ?format was IN the apprise URL, they are kind of on their own here, we will try our best
                 if 'format=html' in url:
-                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '<br>\r\n')
+                    n_body = newline_re.sub('<br>\r\n', n_body)
                     # This will also prevent apprise from doing conversion
                     apprise_input_format = NotifyFormat.HTML.value
                     requested_output_format = NotifyFormat.HTML.value
                 elif 'format=text' in url:
-                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\r\n')
                     apprise_input_format = NotifyFormat.TEXT.value
                     requested_output_format = NotifyFormat.TEXT.value
 
+#@todo on null:// (only if its a 1 url with null) probably doesnt need to actually .add/setup/etc
             sent_objs.append({'title': n_title,
                               'body': n_body,
-                              'url': url})
-            apobj.add(url)
+                              'url': url,
+                              # So that we can do a null:// call and get back exactly what would have been sent
+                              'original_context': n_object })
+
+            if not url.startswith('null://'):
+                apobj.add(url)
 
             # Since the output is always based on the plaintext of the 'diff' engine, wrap it nicely.
             # It should always be similar to the 'history' part of the UI.
@@ -383,15 +415,16 @@ def process_notification(n_object: NotificationContextData, datastore):
                 if not '<pre' in n_body and not '<body' in n_body: # No custom HTML-ish body was setup already
                     n_body = as_monospaced_html_email(content=n_body, title=n_title)
 
-        apobj.notify(
-            title=n_title,
-            body=n_body,
-            # `body_format` Tell apprise what format the INPUT is in, specify a wrong/bad type and it will force skip conversion in apprise
-            # &format= in URL Tell apprise what format the OUTPUT should be in (it can convert between)
-            body_format=apprise_input_format,
-            # False is not an option for AppRise, must be type None
-            attach=n_object.get('screenshot', None)
-        )
+        if not url.startswith('null://'):
+            apobj.notify(
+                title=n_title,
+                body=n_body,
+                # `body_format` Tell apprise what format the INPUT is in, specify a wrong/bad type and it will force skip conversion in apprise
+                # &format= in URL Tell apprise what format the OUTPUT should be in (it can convert between)
+                body_format=apprise_input_format,
+                # False is not an option for AppRise, must be type None
+                attach=n_object.get('screenshot', None)
+            )
 
         # Returns empty string if nothing found, multi-line string otherwise
         log_value = logs.getvalue()
@@ -410,6 +443,8 @@ def create_notification_parameters(n_object: NotificationContextData, datastore)
     if not isinstance(n_object, NotificationContextData):
         raise TypeError(f"Expected NotificationContextData, got {type(n_object)}")
 
+    ext_base_url = datastore.data['settings']['application'].get('active_base_url').strip('/')+'/'
+
     watch = datastore.data['watching'].get(n_object['uuid'])
     if watch:
         watch_title = datastore.data['watching'][n_object['uuid']].label
@@ -423,20 +458,29 @@ def create_notification_parameters(n_object: NotificationContextData, datastore)
         watch_title = 'Change Detection'
         watch_tag = ''
 
-    # Create URLs to customise the notification with
-    # active_base_url - set in store.py data property
-    base_url = datastore.data['settings']['application'].get('active_base_url')
-
     watch_url = n_object['watch_url']
 
-    diff_url = "{}/diff/{}".format(base_url, n_object['uuid'])
-    preview_url = "{}/preview/{}".format(base_url, n_object['uuid'])
+    # Build URLs manually instead of using url_for() to avoid requiring a request context
+    # This allows notifications to be processed in background threads
+    uuid = n_object['uuid']
 
+    if n_object.get('timestamp_from') and n_object.get('timestamp_to'):
+        # Include a link to the diff page with specific versions
+        diff_url = f"{ext_base_url}diff/{uuid}?from_version={n_object['timestamp_from']}&to_version={n_object['timestamp_to']}"
+    else:
+        diff_url = f"{ext_base_url}diff/{uuid}"
+
+    preview_url = f"{ext_base_url}preview/{uuid}"
+    edit_url = f"{ext_base_url}edit/{uuid}"
+
+    # @todo test that preview_url is correct when running in not-null mode?
+    # if not, first time app loads i think it can set a flask context
     n_object.update(
         {
-            'base_url': base_url,
+            'base_url': ext_base_url,
             'diff_url': diff_url,
-            'preview_url': preview_url,
+            'preview_url': preview_url, #@todo include 'version='
+            'edit_url': edit_url, #@todo also pause, also mute link
             'watch_tag': watch_tag if watch_tag is not None else '',
             'watch_title': watch_title if watch_title is not None else '',
             'watch_url': watch_url,

changedetectionio/notification_service.py

@@ -5,15 +5,53 @@ Notification Service Module
 Extracted from update_worker.py to provide standalone notification functionality
 for both sync and async workers
 """
+import datetime
 
+import pytz
 from loguru import logger
 import time
 
-from changedetectionio.model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 from changedetectionio.notification import default_notification_format, valid_notification_formats
 
-# This gets modified on notification time (handler.py) depending on the required notification output
-CUSTOM_LINEBREAK_PLACEHOLDER='@BR@'
+
+def _check_cascading_vars(datastore, var_name, watch):
+    """
+    Check notification variables in cascading priority:
+    Individual watch settings > Tag settings > Global settings
+    """
+    from changedetectionio.notification import (
+        USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH,
+        default_notification_body,
+        default_notification_title
+    )
+
+    # Would be better if this was some kind of Object where Watch can reference the parent datastore etc
+    v = watch.get(var_name)
+    if v and not watch.get('notification_muted'):
+        if var_name == 'notification_format' and v == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:
+            return datastore.data['settings']['application'].get('notification_format')
+
+        return v
+
+    tags = datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
+    if tags:
+        for tag_uuid, tag in tags.items():
+            v = tag.get(var_name)
+            if v and not tag.get('notification_muted'):
+                return v
+
+    if datastore.data['settings']['application'].get(var_name):
+        return datastore.data['settings']['application'].get(var_name)
+
+    # Otherwise could be defaults
+    if var_name == 'notification_format':
+        return USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
+    if var_name == 'notification_body':
+        return default_notification_body
+    if var_name == 'notification_title':
+        return default_notification_title
+
+    return None
 
 
 # What is passed around as notification context, also used as the complete list of valid {{ tokens }}
@@ -23,16 +61,22 @@ class NotificationContextData(dict):
             'base_url': None,
             'current_snapshot': None,
             'diff': None,
+            'diff_clean': None,
             'diff_added': None,
+            'diff_added_clean': None,
             'diff_full': None,
+            'diff_full_clean': None,
             'diff_patch': None,
             'diff_removed': None,
+            'diff_removed_clean': None,
             'diff_url': None,
             'markup_text_links_to_html_links': False, # If automatic conversion of plaintext to HTML should happen
             'notification_timestamp': time.time(),
             'preview_url': None,
             'screenshot': None,
             'triggered_text': None,
+            'timestamp_from': None,
+            'timestamp_to': None,
             'uuid': 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX',  # Converted to 'watch_uuid' in create_notification_parameters
             'watch_mime_type': None,
             'watch_tag': None,
@@ -71,6 +115,57 @@ class NotificationContextData(dict):
 
         super().__setitem__(key, value)
 
+
+def timestamp_to_localtime(timestamp):
+    # Format the date using locale-aware formatting with timezone
+    dt = datetime.datetime.fromtimestamp(int(timestamp))
+    dt = dt.replace(tzinfo=pytz.UTC)
+
+    # Get local timezone-aware datetime
+    local_tz = datetime.datetime.now().astimezone().tzinfo
+    local_dt = dt.astimezone(local_tz)
+
+    # Format date with timezone - using strftime for locale awareness
+    try:
+        formatted_date = local_dt.strftime('%Y-%m-%d %H:%M:%S %Z')
+    except:
+        # Fallback if locale issues
+        formatted_date = local_dt.isoformat()
+
+    return formatted_date
+
+def set_basic_notification_vars(snapshot_contents, current_snapshot, prev_snapshot, watch, triggered_text, timestamp_changed=None):
+    now = time.time()
+    from changedetectionio import diff
+
+    n_object = {
+        'current_snapshot': snapshot_contents,
+        'diff': diff.render_diff(prev_snapshot, current_snapshot),
+        'diff_clean': diff.render_diff(prev_snapshot, current_snapshot, include_change_type_prefix=False),
+        'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False),
+        'diff_added_clean': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, include_change_type_prefix=False),
+        'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True),
+        'diff_full_clean': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, include_change_type_prefix=False),
+        'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, patch_format=True),
+        'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False),
+        'diff_removed_clean': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, include_change_type_prefix=False),
+        'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
+        'change_datetime': timestamp_to_localtime(timestamp_changed) if timestamp_changed else None,
+        'triggered_text': triggered_text,
+        'uuid': watch.get('uuid') if watch else None,
+        'watch_url': watch.get('url') if watch else None,
+        'watch_uuid': watch.get('uuid') if watch else None,
+        'watch_mime_type': watch.get('content-type')
+    }
+
+    # The \n's in the content from the above will get converted to <br> etc depending on the notification format
+
+    if watch:
+        n_object.update(watch.extra_notification_token_values())
+
+    logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time() - now:.3f}s")
+    return n_object
+
 class NotificationService:
     """
     Standalone notification service that handles all notification functionality
@@ -81,11 +176,10 @@ class NotificationService:
         self.datastore = datastore
         self.notification_q = notification_q
     
-    def queue_notification_for_watch(self, n_object: NotificationContextData, watch):
+    def queue_notification_for_watch(self, n_object: NotificationContextData, watch, date_index_from=-2, date_index_to=-1):
         """
         Queue a notification for a watch with full diff rendering and template variables
         """
-        from changedetectionio import diff
         from changedetectionio.notification import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 
         if not isinstance(n_object, NotificationContextData):
@@ -94,8 +188,6 @@ class NotificationService:
         dates = []
         trigger_text = ''
 
-        now = time.time()
-
         if watch:
             watch_history = watch.history
             dates = list(watch_history.keys())
@@ -103,7 +195,7 @@ class NotificationService:
 
         # Add text that was triggered
         if len(dates):
-            snapshot_contents = watch.get_history_snapshot(dates[-1])
+            snapshot_contents = watch.get_history_snapshot(timestamp=dates[-1])
         else:
             snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
 
@@ -117,76 +209,30 @@ class NotificationService:
             from . import html_tools
             triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
             if triggered_text:
-                triggered_text = CUSTOM_LINEBREAK_PLACEHOLDER.join(triggered_text)
+                triggered_text = '\n'.join(triggered_text)
 
         # Could be called as a 'test notification' with only 1 snapshot available
         prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
         current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
 
         if len(dates) > 1:
-            prev_snapshot = watch.get_history_snapshot(dates[-2])
-            current_snapshot = watch.get_history_snapshot(dates[-1])
+            prev_snapshot = watch.get_history_snapshot(timestamp=dates[date_index_from])
+            current_snapshot = watch.get_history_snapshot(timestamp=dates[date_index_to])
 
-        n_object.update({
-            'current_snapshot': snapshot_contents,
-            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
-            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
-            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
-            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER, patch_format=True),
-            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
-            'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
-            'triggered_text': triggered_text,
-            'uuid': watch.get('uuid') if watch else None,
-            'watch_url': watch.get('url') if watch else None,
-            'watch_uuid': watch.get('uuid') if watch else None,
-            'watch_mime_type': watch.get('content-type')
-        })
 
-        if watch:
-            n_object.update(watch.extra_notification_token_values())
+        n_object.update(set_basic_notification_vars(snapshot_contents=snapshot_contents,
+                                                    current_snapshot=current_snapshot,
+                                                    prev_snapshot=prev_snapshot,
+                                                    watch=watch,
+                                                    triggered_text=triggered_text,
+                                                    timestamp_changed=dates[date_index_to]))
 
-        logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
-        logger.debug("Queued notification for sending")
-        self.notification_q.put(n_object)
-
-    def _check_cascading_vars(self, var_name, watch):
-        """
-        Check notification variables in cascading priority:
-        Individual watch settings > Tag settings > Global settings
-        """
-        from changedetectionio.notification import (
-            USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH,
-            default_notification_body,
-            default_notification_title
-        )
-
-        # Would be better if this was some kind of Object where Watch can reference the parent datastore etc
-        v = watch.get(var_name)
-        if v and not watch.get('notification_muted'):
-            if var_name == 'notification_format' and v == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:
-                return self.datastore.data['settings']['application'].get('notification_format')
-
-            return v
-
-        tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
-        if tags:
-            for tag_uuid, tag in tags.items():
-                v = tag.get(var_name)
-                if v and not tag.get('notification_muted'):
-                    return v
-
-        if self.datastore.data['settings']['application'].get(var_name):
-            return self.datastore.data['settings']['application'].get(var_name)
-
-        # Otherwise could be defaults
-        if var_name == 'notification_format':
-            return USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
-        if var_name == 'notification_body':
-            return default_notification_body
-        if var_name == 'notification_title':
-            return default_notification_title
-
-        return None
+        if self.notification_q:
+            logger.debug("Queued notification for sending")
+            self.notification_q.put(n_object)
+        else:
+            logger.debug("Not queued, no queue defined. Just returning processed data")
+            return n_object
 
     def send_content_changed_notification(self, watch_uuid):
         """
@@ -209,10 +255,11 @@ class NotificationService:
         # Should be a better parent getter in the model object
 
         # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
-        n_object['notification_urls'] = self._check_cascading_vars('notification_urls', watch)
-        n_object['notification_title'] = self._check_cascading_vars('notification_title', watch)
-        n_object['notification_body'] = self._check_cascading_vars('notification_body', watch)
-        n_object['notification_format'] = self._check_cascading_vars('notification_format', watch)
+        # this change probably not needed?
+        n_object['notification_urls'] = _check_cascading_vars(self.datastore, 'notification_urls', watch)
+        n_object['notification_title'] = _check_cascading_vars(self.datastore,'notification_title', watch)
+        n_object['notification_body'] = _check_cascading_vars(self.datastore,'notification_body', watch)
+        n_object['notification_format'] = _check_cascading_vars(self.datastore,'notification_format', watch)
 
         # (Individual watch) Only prepare to notify if the rules above matched
         queued = False
@@ -251,7 +298,7 @@ Thanks - Your omniscient changedetection.io installation.
         n_object = NotificationContextData({
             'notification_title': 'Changedetection.io - Alert - CSS/xPath filter was not present in the page',
             'notification_body': body,
-            'notification_format': self._check_cascading_vars('notification_format', watch),
+            'notification_format': _check_cascading_vars(self.datastore, 'notification_format', watch),
         })
         n_object['markup_text_links_to_html_links'] = n_object.get('notification_format').startswith('html')
 

changedetectionio/processors/__init__.py

@@ -91,6 +91,8 @@ class difference_detection_processor():
             else:
                 logger.debug("Skipping adding proxy data when custom Browser endpoint is specified. ")
 
+        logger.debug(f"Using proxy '{proxy_url}' for {self.watch['uuid']}")
+
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
         self.fetcher = fetcher_obj(proxy_override=proxy_url,

changedetectionio/processors/magic.py

@@ -88,7 +88,7 @@ class guess_stream_type():
                         magic_content_header = mime
 
         except Exception as e:
-            logger.error(f"Error getting a more precise mime type from 'puremagic' library ({str(e)}), using content-based detection")
+            logger.warning(f"Error getting a more precise mime type from 'puremagic' library ({str(e)}), using content-based detection")
 
         # Content-based detection (most reliable for text formats)
         # Check for HTML patterns first - if found, override magic's text/plain
@@ -103,15 +103,15 @@ class guess_stream_type():
             self.is_json = True
         elif 'pdf' in magic_content_header:
             self.is_pdf = True
-        elif has_html_patterns or http_content_header == 'text/html':
-            self.is_html = True
-        elif any(s in magic_content_header for s in JSON_CONTENT_TYPES):
-            self.is_json = True
         # magic will call a rss document 'xml'
         # Rarely do endpoints give the right header, usually just text/xml, so we check also for <rss
         # This also triggers the automatic CDATA text parser so the RSS goes back a nice content list
         elif '<rss' in test_content_normalized or '<feed' in test_content_normalized or any(s in magic_content_header for s in RSS_XML_CONTENT_TYPES) or '<rdf:' in test_content_normalized:
             self.is_rss = True
+        elif has_html_patterns or http_content_header == 'text/html':
+            self.is_html = True
+        elif any(s in magic_content_header for s in JSON_CONTENT_TYPES):
+            self.is_json = True
         elif any(s in http_content_header for s in XML_CONTENT_TYPES):
             # Only mark as generic XML if not already detected as RSS
             if not self.is_rss:

changedetectionio/processors/text_json_diff/__init__.py

@@ -32,7 +32,7 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
     '''Used by @app.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])'''
     from changedetectionio import forms, html_tools
     from changedetectionio.model.Watch import model as watch_model
-    from concurrent.futures import ProcessPoolExecutor
+    from concurrent.futures import ThreadPoolExecutor
     from copy import deepcopy
     from flask import request
     import brotli
@@ -76,13 +76,16 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
             update_handler.fetcher.headers['content-type'] = tmp_watch.get('content-type')
 
             # Process our watch with filters and the HTML from disk, and also a blank watch with no filters but also with the same HTML from disk
-            # Do this as a parallel process because it could take some time
-            with ProcessPoolExecutor(max_workers=2) as executor:
-                future1 = executor.submit(_task, tmp_watch, update_handler)
-                future2 = executor.submit(_task, blank_watch_no_filters, update_handler)
+            # Do this as parallel threads (not processes) to avoid pickle issues with Lock objects
+            try:
+                with ThreadPoolExecutor(max_workers=2) as executor:
+                    future1 = executor.submit(_task, tmp_watch, update_handler)
+                    future2 = executor.submit(_task, blank_watch_no_filters, update_handler)
 
-                text_after_filter = future1.result()
-                text_before_filter = future2.result()
+                    text_after_filter = future1.result()
+                    text_before_filter = future2.result()
+            except Exception as e:
+                x=1
 
     try:
         trigger_line_numbers = html_tools.strip_ignore_text(content=text_after_filter,

changedetectionio/processors/text_json_diff/processor.py

@@ -280,7 +280,7 @@ class ContentProcessor:
 
         # Sort JSON to avoid false alerts from reordering
         try:
-            content = json.dumps(json.loads(content), sort_keys=True, indent=4)
+            content = json.dumps(json.loads(content), sort_keys=True, indent=2, ensure_ascii=False)
         except Exception:
             # Might be malformed JSON, continue anyway
             pass
@@ -298,7 +298,7 @@ class ContentProcessor:
                     xpath_filter=filter_rule.replace('xpath:', ''),
                     html_content=content,
                     append_pretty_line_formatting=not self.watch.is_source_type_url,
-                    is_rss=stream_content_type.is_rss
+                    is_xml=stream_content_type.is_rss or stream_content_type.is_xml
                 )
 
             # XPath1 filters (first match only)
@@ -307,7 +307,7 @@ class ContentProcessor:
                     xpath_filter=filter_rule.replace('xpath1:', ''),
                     html_content=content,
                     append_pretty_line_formatting=not self.watch.is_source_type_url,
-                    is_rss=stream_content_type.is_rss
+                    is_xml=stream_content_type.is_rss or stream_content_type.is_xml
                 )
 
             # JSON filters

changedetectionio/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-addopts = --no-start-live-server --live-server-port=5005
+addopts = --no-start-live-server --live-server-port=0
 #testpaths = tests pytest_invenio
 #live_server_scope = function
 

changedetectionio/realtime/socket_server.py

@@ -37,18 +37,6 @@ class SignalHandler:
         notification_event_signal.connect(self.handle_notification_event, weak=False)
         logger.info("SignalHandler: Connected to notification_event signal")
 
-        # Create and start the queue update thread using standard threading
-        import threading
-        self.polling_emitter_thread = threading.Thread(
-            target=self.polling_emit_running_or_queued_watches_threaded,
-            daemon=True
-        )
-        self.polling_emitter_thread.start()
-        logger.info("Started polling thread using threading (eventlet-free)")
-
-        # Store the thread reference in socketio for clean shutdown
-        self.socketio_instance.polling_emitter_thread = self.polling_emitter_thread
-
     def handle_signal(self, *args, **kwargs):
         logger.trace(f"SignalHandler: Signal received with {len(args)} args and {len(kwargs)} kwargs")
         # Safely extract the watch UUID from kwargs
@@ -124,74 +112,6 @@ class SignalHandler:
         except Exception as e:
             logger.error(f"Socket.IO error in handle_notification_event: {str(e)}")
 
-    def polling_emit_running_or_queued_watches_threaded(self):
-        """Threading version of polling for Windows compatibility"""
-        import time
-        import threading
-        logger.info("Queue update thread started (threading mode)")
-
-        # Import here to avoid circular imports
-        from changedetectionio.flask_app import app
-        from changedetectionio import worker_handler
-        watch_check_update = signal('watch_check_update')
-
-        # Track previous state to avoid unnecessary emissions
-        previous_running_uuids = set()
-
-        # Run until app shutdown - check exit flag more frequently for fast shutdown
-        exit_event = getattr(app.config, 'exit', threading.Event())
-
-        while not exit_event.is_set():
-            try:
-                # Get current running UUIDs from async workers
-                running_uuids = set(worker_handler.get_running_uuids())
-
-                # Only send updates for UUIDs that changed state
-                newly_running = running_uuids - previous_running_uuids
-                no_longer_running = previous_running_uuids - running_uuids
-
-                # Send updates for newly running UUIDs (but exit fast if shutdown requested)
-                for uuid in newly_running:
-                    if exit_event.is_set():
-                        break
-                    logger.trace(f"Threading polling: UUID {uuid} started processing")
-                    with app.app_context():
-                        watch_check_update.send(app_context=app, watch_uuid=uuid)
-                    time.sleep(0.01)  # Small yield
-
-                # Send updates for UUIDs that finished processing (but exit fast if shutdown requested)
-                if not exit_event.is_set():
-                    for uuid in no_longer_running:
-                        if exit_event.is_set():
-                            break
-                        logger.trace(f"Threading polling: UUID {uuid} finished processing")
-                        with app.app_context():
-                            watch_check_update.send(app_context=app, watch_uuid=uuid)
-                        time.sleep(0.01)  # Small yield
-
-                # Update tracking for next iteration
-                previous_running_uuids = running_uuids
-
-                # Sleep between polling cycles, but check exit flag every 0.5 seconds for fast shutdown
-                for _ in range(20):  # 20 * 0.5 = 10 seconds total
-                    if exit_event.is_set():
-                        break
-                    time.sleep(0.5)
-
-            except Exception as e:
-                logger.error(f"Error in threading polling: {str(e)}")
-                # Even during error recovery, check for exit quickly
-                for _ in range(1):  # 1 * 0.5 = 0.5 seconds
-                    if exit_event.is_set():
-                        break
-                    time.sleep(0.5)
-
-        # Check if we're in pytest environment - if so, be more gentle with logging
-        import sys
-        in_pytest = "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ
-
-        if not in_pytest:
-            logger.info("Queue update thread stopped (threading mode)")
 
 
 def handle_watch_update(socketio, **kwargs):
@@ -383,19 +303,6 @@ def init_socketio(app, datastore):
         """Shutdown the SocketIO server fast and aggressively"""
         try:
             logger.info("Socket.IO: Fast shutdown initiated...")
-
-            # For threading mode, give the thread a very short time to exit gracefully
-            if hasattr(socketio, 'polling_emitter_thread'):
-                if socketio.polling_emitter_thread.is_alive():
-                    logger.info("Socket.IO: Waiting 1 second for polling thread to stop...")
-                    socketio.polling_emitter_thread.join(timeout=1.0)  # Only 1 second timeout
-                    if socketio.polling_emitter_thread.is_alive():
-                        logger.info("Socket.IO: Polling thread still running after timeout - continuing with shutdown")
-                    else:
-                        logger.info("Socket.IO: Polling thread stopped quickly")
-                else:
-                    logger.info("Socket.IO: Polling thread already stopped")
-
             logger.info("Socket.IO: Fast shutdown complete")
         except Exception as e:
             logger.error(f"Socket.IO error during shutdown: {str(e)}")

changedetectionio/rss_tools.py

@@ -29,16 +29,135 @@ def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False
     return re.sub(pattern, repl, html_content)
 
 
+# Jinja2 template for formatting RSS/Atom feed entries
+# Covers all common feedparser entry fields including namespaced elements
+# Outputs HTML that will be converted to text via html_to_text
+# @todo - This could be a UI setting in the future
+RSS_ENTRY_TEMPLATE = """<article class="rss-item" id="{{ entry.id|replace('"', '')|replace(' ', '-') }}">{%- if entry.title -%}Title: {{ entry.title }}<br>{%- endif -%}
+{%- if entry.link -%}<strong>Link:</strong> <a href="{{ entry.link }}">{{ entry.link }}</a><br>
+{%- endif -%}
+{%- if entry.id -%}
+<strong>Guid:</strong> {{ entry.id }}<br>
+{%- endif -%}
+{%- if entry.published -%}
+<strong>PubDate:</strong> {{ entry.published }}<br>
+{%- endif -%}
+{%- if entry.updated and entry.updated != entry.published -%}
+<strong>Updated:</strong> {{ entry.updated }}<br>
+{%- endif -%}
+{%- if entry.author -%}
+<strong>Author:</strong> {{ entry.author }}<br>
+{%- elif entry.author_detail and entry.author_detail.name -%}
+<strong>Author:</strong> {{ entry.author_detail.name }}
+{%- if entry.author_detail.email %} ({{ entry.author_detail.email }}){% endif -%}
+<br>
+{%- endif -%}
+{%- if entry.contributors -%}
+<strong>Contributors:</strong> {% for contributor in entry.contributors -%}
+{{ contributor.name if contributor.name else contributor }}
+{%- if not loop.last %}, {% endif -%}
+{%- endfor %}<br>
+{%- endif -%}
+{%- if entry.publisher -%}
+<strong>Publisher:</strong> {{ entry.publisher }}<br>
+{%- endif -%}
+{%- if entry.rights -%}
+<strong>Rights:</strong> {{ entry.rights }}<br>
+{%- endif -%}
+{%- if entry.license -%}
+<strong>License:</strong> {{ entry.license }}<br>
+{%- endif -%}
+{%- if entry.language -%}
+<strong>Language:</strong> {{ entry.language }}<br>
+{%- endif -%}
+{%- if entry.tags -%}
+<strong>Tags:</strong> {% for tag in entry.tags -%}
+{{ tag.term if tag.term else tag }}
+{%- if not loop.last %}, {% endif -%}
+{%- endfor %}<br>
+{%- endif -%}
+{%- if entry.category -%}
+<strong>Category:</strong> {{ entry.category }}<br>
+{%- endif -%}
+{%- if entry.comments -%}
+<strong>Comments:</strong> <a href="{{ entry.comments }}">{{ entry.comments }}</a><br>
+{%- endif -%}
+{%- if entry.slash_comments -%}
+<strong>Comment Count:</strong> {{ entry.slash_comments }}<br>
+{%- endif -%}
+{%- if entry.enclosures -%}
+<strong>Enclosures:</strong><br>
+{%- for enclosure in entry.enclosures %}
+- <a href="{{ enclosure.href }}">{{ enclosure.href }}</a> ({{ enclosure.type if enclosure.type else 'unknown type' }}
+{%- if enclosure.length %}, {{ enclosure.length }} bytes{% endif -%}
+)<br>
+{%- endfor -%}
+{%- endif -%}
+{%- if entry.media_content -%}
+<strong>Media:</strong><br>
+{%- for media in entry.media_content %}
+- <a href="{{ media.url }}">{{ media.url }}</a>
+{%- if media.type %} ({{ media.type }}){% endif -%}
+{%- if media.width and media.height %} {{ media.width }}x{{ media.height }}{% endif -%}
+<br>
+{%- endfor -%}
+{%- endif -%}
+{%- if entry.media_thumbnail -%}
+<strong>Thumbnail:</strong> <a href="{{ entry.media_thumbnail[0].url if entry.media_thumbnail[0].url else entry.media_thumbnail[0] }}">{{ entry.media_thumbnail[0].url if entry.media_thumbnail[0].url else entry.media_thumbnail[0] }}</a><br>
+{%- endif -%}
+{%- if entry.media_description -%}
+<strong>Media Description:</strong> {{ entry.media_description }}<br>
+{%- endif -%}
+{%- if entry.itunes_duration -%}
+<strong>Duration:</strong> {{ entry.itunes_duration }}<br>
+{%- endif -%}
+{%- if entry.itunes_author -%}
+<strong>Podcast Author:</strong> {{ entry.itunes_author }}<br>
+{%- endif -%}
+{%- if entry.dc_identifier -%}
+<strong>Identifier:</strong> {{ entry.dc_identifier }}<br>
+{%- endif -%}
+{%- if entry.dc_source -%}
+<strong>DC Source:</strong> {{ entry.dc_source }}<br>
+{%- endif -%}
+{%- if entry.dc_type -%}
+<strong>Type:</strong> {{ entry.dc_type }}<br>
+{%- endif -%}
+{%- if entry.dc_format -%}
+<strong>Format:</strong> {{ entry.dc_format }}<br>
+{%- endif -%}
+{%- if entry.dc_relation -%}
+<strong>Related:</strong> {{ entry.dc_relation }}<br>
+{%- endif -%}
+{%- if entry.dc_coverage -%}
+<strong>Coverage:</strong> {{ entry.dc_coverage }}<br>
+{%- endif -%}
+{%- if entry.source and entry.source.title -%}
+<strong>Source:</strong> {{ entry.source.title }}
+{%- if entry.source.link %} (<a href="{{ entry.source.link }}">{{ entry.source.link }}</a>){% endif -%}
+<br>
+{%- endif -%}
+{%- if entry.dc_content -%}
+<strong>Content:</strong> {{ entry.dc_content | safe }}
+{%- elif entry.content and entry.content[0].value -%}
+<strong>Content:</strong> {{ entry.content[0].value | safe }}
+{%- elif entry.summary -%}
+<strong>Summary:</strong> {{ entry.summary | safe }}
+{%- endif -%}</article>
+"""
+
+
 def format_rss_items(rss_content: str, render_anchor_tag_content=False) -> str:
     """
-    Format RSS/Atom feed items in a readable text format using feedparser.
+    Format RSS/Atom feed items in a readable text format using feedparser and Jinja2.
 
-    Converts RSS <item> or Atom <entry> elements to formatted text with:
-    - <title> → <h1>Title</h1>
-    - <link> → Link: [url]
-    - <guid> → Guid: [id]
-    - <pubDate> → PubDate: [date]
-    - <description> or <content> → Raw HTML content (CDATA and entities automatically handled)
+    Converts RSS <item> or Atom <entry> elements to formatted text with all available fields:
+    - Basic fields: title, link, id/guid, published date, updated date
+    - Author fields: author, author_detail, contributors, publisher
+    - Content fields: content, summary, description
+    - Metadata: tags, category, rights, license
+    - Media: enclosures, media_content, media_thumbnail
+    - Dublin Core elements: dc:creator, dc:date, dc:publisher, etc. (mapped by feedparser)
 
     Args:
         rss_content: The RSS/Atom feed content
@@ -49,65 +168,19 @@ def format_rss_items(rss_content: str, render_anchor_tag_content=False) -> str:
     """
     try:
         import feedparser
-        from xml.sax.saxutils import escape as xml_escape
+        from changedetectionio.jinja2_custom import safe_jinja
 
         # Parse the feed - feedparser handles all RSS/Atom variants, CDATA, entity unescaping, etc.
         feed = feedparser.parse(rss_content)
 
-        formatted_items = []
-
-        # Determine feed type for appropriate labels when fields are missing
-        # feedparser sets feed.version to things like 'rss20', 'atom10', etc.
+        # Determine feed type for appropriate labels
         is_atom = feed.version and 'atom' in feed.version
 
+        formatted_items = []
         for entry in feed.entries:
-            item_parts = []
-
-            # Title - feedparser handles CDATA and entity unescaping automatically
-            if hasattr(entry, 'title') and entry.title:
-                item_parts.append(f'<h1>{xml_escape(entry.title)}</h1>')
-
-            # Link
-            if hasattr(entry, 'link') and entry.link:
-                item_parts.append(f'Link: {xml_escape(entry.link)}<br>')
-
-            # GUID/ID
-            if hasattr(entry, 'id') and entry.id:
-                item_parts.append(f'Guid: {xml_escape(entry.id)}<br>')
-
-            # Date - feedparser normalizes all date field names to 'published'
-            if hasattr(entry, 'published') and entry.published:
-                item_parts.append(f'PubDate: {xml_escape(entry.published)}<br>')
-
-            # Description/Content - feedparser handles CDATA and entity unescaping automatically
-            # Only add "Summary:" label for Atom <summary> tags
-            content = None
-            add_label = False
-
-            if hasattr(entry, 'content') and entry.content:
-                # Atom <content> - no label, just content
-                content = entry.content[0].value if entry.content[0].value else None
-            elif hasattr(entry, 'summary'):
-                # Could be RSS <description> or Atom <summary>
-                # feedparser maps both to entry.summary
-                content = entry.summary if entry.summary else None
-                # Only add "Summary:" label for Atom feeds (which use <summary> tag)
-                if is_atom:
-                    add_label = True
-
-            # Add content with or without label
-            if content:
-                if add_label:
-                    item_parts.append(f'Summary:<br>{content}')
-                else:
-                    item_parts.append(content)
-            else:
-                # No content - just show <none>
-                item_parts.append('&lt;none&gt;')
-
-            # Join all parts of this item
-            if item_parts:
-                formatted_items.append('\n'.join(item_parts))
+            # Render the entry using Jinja2 template
+            rendered = safe_jinja.render(RSS_ENTRY_TEMPLATE, entry=entry, is_atom=is_atom)
+            formatted_items.append(rendered.strip())
 
         # Wrap each item in a div with classes (first, last, item-N)
         items_html = []
@@ -122,7 +195,8 @@ def format_rss_items(rss_content: str, render_anchor_tag_content=False) -> str:
 
             class_str = ' '.join(classes)
             items_html.append(f'<div class="{class_str}">{item}</div>')
-        return '<html><body>\n'+"\n<br><br>".join(items_html)+'\n</body></html>'
+
+        return '<html><body>\n' + "\n<br>".join(items_html) + '\n</body></html>'
 
     except Exception as e:
         logger.warning(f"Error formatting RSS items: {str(e)}")

changedetectionio/run_basic_tests.sh

@@ -11,13 +11,60 @@ set -e
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-find tests/test_*py -type f|while read test_name
-do
-  echo "TEST RUNNING $test_name"
-  # REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser
-  REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 --tb=long $test_name
-done
+# Since theres no curl installed lets roll with python3
+check_sanity() {
+  local port="$1"
+  if [ -z "$port" ]; then
+    echo "Usage: check_sanity <port>" >&2
+    return 1
+  fi
 
+  python3 - "$port" <<'PYCODE'
+import sys, time, urllib.request, socket
+
+port = sys.argv[1]
+url = f'http://localhost:{port}'
+ok = False
+
+for _ in range(6):  # --retry 6
+    try:
+        r = urllib.request.urlopen(url, timeout=3).read().decode()
+        if 'est-url-is-sanity' in r:
+            ok = True
+            break
+    except (urllib.error.URLError, ConnectionRefusedError, socket.error):
+        time.sleep(1)
+sys.exit(0 if ok else 1)
+PYCODE
+}
+
+data_sanity_test () {
+  # Restart data sanity test
+  cd ..
+  TMPDIR=$(mktemp -d)
+  PORT_N=$((5000 + RANDOM % (6501 - 5000)))
+  ./changedetection.py -p $PORT_N -d $TMPDIR -u "https://localhost?test-url-is-sanity=1" &
+  PID=$!
+  sleep 5
+  kill $PID
+  sleep 2
+  ./changedetection.py -p $PORT_N -d $TMPDIR &
+  PID=$!
+  sleep 5
+  # On a restart the URL should still be there
+  check_sanity $PORT_N || exit 1
+  kill $PID
+  cd $OLDPWD
+
+  # datastore looks alright, continue
+}
+
+data_sanity_test
+
+# REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser
+REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -n 30 --dist load  tests/test_*.py
+
+#time pytest -n auto --dist loadfile -vv --tb=long tests/test_*.py
 echo "RUNNING WITH BASE_URL SET"
 
 # Now re-run some tests with BASE_URL enabled
@@ -43,4 +90,7 @@ FETCH_WORKERS=130 pytest  tests/test_history_consistency.py -v -l
 
 # Check file:// will pickup a file when enabled
 echo "Hello world" > /tmp/test-file.txt
-ALLOW_FILE_URI=yes pytest tests/test_security.py
+ALLOW_FILE_URI=yes pytest -vv -s  tests/test_security.py
+
+
+

changedetectionio/run_proxy_tests.sh

@@ -5,8 +5,6 @@ set -e
 # enable debug
 set -x
 
-docker network inspect changedet-network >/dev/null 2>&1 || docker network create changedet-network
-
 # Test proxy list handling, starting two squids on different ports
 # Each squid adds a different header to the response, which is the main thing we test for.
 docker run --network changedet-network -d --name squid-one --hostname squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
@@ -21,12 +19,13 @@ docker run --network changedet-network -d \
   -v `pwd`/tests/proxy_list/squid-passwords.txt:/etc/squid3/passwords \
   ubuntu/squid:4.13-21.10_edge
 
-
+sleep 5
 ## 2nd test actually choose the preferred proxy from proxies.json
+# This will force a request via "proxy-two"
 docker run --network changedet-network \
-  -v `pwd`/tests/proxy_list/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
+  -v `pwd`/tests/proxy_list/proxies.json-example:/tmp/proxies.json \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_multiple_proxy.py'
+  bash -c 'cd changedetectionio && pytest -s tests/proxy_list/test_multiple_proxy.py --datastore-path /tmp'
 
 set +e
 echo "- Looking for chosen.changedetection.io request in squid-one - it should NOT be here"
@@ -50,8 +49,10 @@ fi
 # Test the UI configurable proxies
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py'
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py --datastore-path /tmp'
 
+# Give squid proxies a moment to flush their logs
+sleep 2
 
 # Should see a request for one.changedetection.io in there
 echo "- Looking for .changedetection.io request in squid-custom"
@@ -65,7 +66,10 @@ fi
 # Test "no-proxy" option
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py'
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py --datastore-path /tmp'
+
+# Give squid proxies a moment to flush their logs
+sleep 2
 
 # We need to handle grep returning 1
 set +e
@@ -82,6 +86,8 @@ for c in $(echo "squid-one squid-two squid-custom"); do
   fi
 done
 
+echo "docker ps output"
+docker ps
 
 docker kill squid-one squid-two squid-custom
 
@@ -90,19 +96,19 @@ docker kill squid-one squid-two squid-custom
 # Requests
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_proxy_noconnect.py'
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'
 
 # Playwright
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py'
+  bash -c 'cd changedetectionio && PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'
 
 # Puppeteer fast
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && FAST_PUPPETEER_CHROME_FETCHER=1 PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py'
+  bash -c 'cd changedetectionio && FAST_PUPPETEER_CHROME_FETCHER=1 PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'
 
 # Selenium
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && WEBDRIVER_URL=http://selenium:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py'
+  bash -c 'cd changedetectionio && WEBDRIVER_URL=http://selenium:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'

changedetectionio/run_socks_proxy_tests.sh

@@ -15,13 +15,13 @@ docker run --network changedet-network -d --hostname socks5proxy-noauth --rm -p
 echo "---------------------------------- SOCKS5 -------------------"
 # SOCKS5 related - test from proxies.json
 docker run --network changedet-network \
-  -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
+  -v `pwd`/tests/proxy_socks5/proxies.json-example:/tmp/proxies.json \
   --rm \
   -e "FLASK_SERVER_NAME=cdio" \
   --hostname cdio \
   -e "SOCKSTEST=proxiesjson" \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py'
+  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py  --datastore-path /tmp'
 
 # SOCKS5 related - by manually entering in UI
 docker run --network changedet-network \
@@ -30,18 +30,18 @@ docker run --network changedet-network \
   --hostname cdio \
   -e "SOCKSTEST=manual" \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy.py'
+  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy.py --datastore-path /tmp'
 
 # SOCKS5 related - test from proxies.json via playwright - NOTE- PLAYWRIGHT DOESNT SUPPORT AUTHENTICATING PROXY
 docker run --network changedet-network \
   -e "SOCKSTEST=manual-playwright" \
   --hostname cdio \
   -e "FLASK_SERVER_NAME=cdio" \
-  -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \
+  -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/tmp/proxies.json \
   -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" \
   --rm \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py'
+  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py --datastore-path /tmp'
 
 echo "socks5 server logs"
 docker logs socks5proxy

changedetectionio/static/js/notifications.js

@@ -14,10 +14,10 @@ $(document).ready(function () {
         e.preventDefault();
 
         data = {
-            notification_body: $('#notification_body').val(),
-            notification_format: $('#notification_format').val(),
-            notification_title: $('#notification_title').val(),
-            notification_urls: $('.notification-urls').val(),
+            notification_urls: $('textarea.notification-urls').val(),
+            notification_title: $('input.notification-title').val(),
+            notification_body: $('textarea.notification-body').val(),
+            notification_format: $('select.notification-format').val(),
             tags: $('#tags').val(),
             window_url: window.location.href,
         }

changedetectionio/static/styles/scss/styles.scss

@@ -329,12 +329,18 @@ a.pure-button-selected {
 .notifications-wrapper {
   padding-top: 0.5rem;
   #notification-test-log {
-    padding-top: 1rem;
+    margin-top: 1rem;
+    padding: 1rem;
     white-space: pre-wrap;
     word-break: break-word;
     overflow-wrap: break-word;
     max-width: 100%;
     box-sizing: border-box;
+    max-height: 12rem;
+    overflow-y: scroll;
+    border: 1px solid var(--color-border-notification);
+    border-radius: 5px;
+
   }
 }
 

changedetectionio/store.py

@@ -6,6 +6,7 @@ from flask import (
     flash
 )
 
+from .blueprint.rss import RSS_CONTENT_FORMAT_DEFAULT
 from .html_tools import TRANSLATE_WHITESPACE_TABLE
 from .model import App, Watch, USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 from copy import deepcopy, copy
@@ -22,6 +23,13 @@ import uuid as uuid_builder
 from loguru import logger
 from blinker import signal
 
+# Try to import orjson for faster JSON serialization
+try:
+    import orjson
+    HAS_ORJSON = True
+except ImportError:
+    HAS_ORJSON = False
+
 from .processors import get_custom_watch_obj_for_processor
 from .processors.restock_diff import Restock
 
@@ -37,22 +45,41 @@ class ChangeDetectionStore:
     lock = Lock()
     # For general updates/writes that can wait a few seconds
     needs_write = False
-
+    datastore_path = None
     # For when we edit, we should write to disk
     needs_write_urgent = False
 
     __version_check = True
+    save_data_thread = None
 
     def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
         # Should only be active for docker
         # logging.basicConfig(filename='/dev/stdout', level=logging.INFO)
-        self.__data = App.model()
         self.datastore_path = datastore_path
-        self.json_store_path = os.path.join(self.datastore_path, "url-watches.json")
-        logger.info(f"Datastore path is '{self.json_store_path}'")
         self.needs_write = False
         self.start_time = time.time()
         self.stop_thread = False
+        self.save_version_copy_json_db(version_tag)
+        self.reload_state(datastore_path=datastore_path, include_default_watches=include_default_watches, version_tag=version_tag)
+
+    def save_version_copy_json_db(self, version_tag):
+        import re
+
+        version_text = re.sub(r'\D+', '-', version_tag)
+        db_path = os.path.join(self.datastore_path, "url-watches.json")
+        db_path_version_backup = os.path.join(self.datastore_path, f"url-watches-{version_text}.json")
+
+        if not os.path.isfile(db_path_version_backup) and os.path.isfile(db_path):
+            from shutil import copyfile
+            logger.info(f"Backing up JSON DB due to new version to '{db_path_version_backup}'.")
+            copyfile(db_path, db_path_version_backup)
+
+
+    def reload_state(self, datastore_path, include_default_watches, version_tag):
+        logger.info(f"Datastore path is '{datastore_path}'")
+
+        self.__data = App.model()
+        self.json_store_path = os.path.join(self.datastore_path, "url-watches.json")
         # Base definition for all watchers
         # deepcopy part of #569 - not sure why its needed exactly
         self.generic_definition = deepcopy(Watch.model(datastore_path = datastore_path, default={}))
@@ -64,37 +91,46 @@ class ChangeDetectionStore:
                 self.__data['build_sha'] = f.read()
 
         try:
-            # @todo retest with ", encoding='utf-8'"
-            with open(self.json_store_path) as json_file:
-                from_disk = json.load(json_file)
+            if HAS_ORJSON:
+                # orjson.loads() expects UTF-8 encoded bytes #3611
+                with open(self.json_store_path, 'rb') as json_file:
+                    from_disk = orjson.loads(json_file.read())
+            else:
+                with open(self.json_store_path, encoding='utf-8') as json_file:
+                    from_disk = json.load(json_file)
 
-                # @todo isnt there a way todo this dict.update recursively?
-                # Problem here is if the one on the disk is missing a sub-struct, it wont be present anymore.
-                if 'watching' in from_disk:
-                    self.__data['watching'].update(from_disk['watching'])
+            if not from_disk:
+                # No FileNotFound exception was thrown but somehow the JSON was empty - abort for safety.
+                logger.critical(f"JSON DB existed but was empty on load - empty JSON file? '{self.json_store_path}' Aborting")
+                raise Exception('JSON DB existed but was empty on load - Aborting')
 
-                if 'app_guid' in from_disk:
-                    self.__data['app_guid'] = from_disk['app_guid']
+            # @todo isnt there a way todo this dict.update recursively?
+            # Problem here is if the one on the disk is missing a sub-struct, it wont be present anymore.
+            if 'watching' in from_disk:
+                self.__data['watching'].update(from_disk['watching'])
 
-                if 'settings' in from_disk:
-                    if 'headers' in from_disk['settings']:
-                        self.__data['settings']['headers'].update(from_disk['settings']['headers'])
+            if 'app_guid' in from_disk:
+                self.__data['app_guid'] = from_disk['app_guid']
 
-                    if 'requests' in from_disk['settings']:
-                        self.__data['settings']['requests'].update(from_disk['settings']['requests'])
+            if 'settings' in from_disk:
+                if 'headers' in from_disk['settings']:
+                    self.__data['settings']['headers'].update(from_disk['settings']['headers'])
 
-                    if 'application' in from_disk['settings']:
-                        self.__data['settings']['application'].update(from_disk['settings']['application'])
+                if 'requests' in from_disk['settings']:
+                    self.__data['settings']['requests'].update(from_disk['settings']['requests'])
 
-                # Convert each existing watch back to the Watch.model object
-                for uuid, watch in self.__data['watching'].items():
-                    self.__data['watching'][uuid] = self.rehydrate_entity(uuid, watch)
-                    logger.info(f"Watching: {uuid} {watch['url']}")
+                if 'application' in from_disk['settings']:
+                    self.__data['settings']['application'].update(from_disk['settings']['application'])
 
-                # And for Tags also, should be Restock type because it has extra settings
-                for uuid, tag in self.__data['settings']['application']['tags'].items():
-                    self.__data['settings']['application']['tags'][uuid] = self.rehydrate_entity(uuid, tag, processor_override='restock_diff')
-                    logger.info(f"Tag: {uuid} {tag['title']}")
+            # Convert each existing watch back to the Watch.model object
+            for uuid, watch in self.__data['watching'].items():
+                self.__data['watching'][uuid] = self.rehydrate_entity(uuid, watch)
+                logger.info(f"Watching: {uuid} {watch['url']}")
+
+            # And for Tags also, should be Restock type because it has extra settings
+            for uuid, tag in self.__data['settings']['application']['tags'].items():
+                self.__data['settings']['application']['tags'][uuid] = self.rehydrate_entity(uuid, tag, processor_override='restock_diff')
+                logger.info(f"Tag: {uuid} {tag['title']}")
 
         # First time ran, Create the datastore.
         except (FileNotFoundError):
@@ -145,7 +181,10 @@ class ChangeDetectionStore:
         self.needs_write = True
 
         # Finally start the thread that will manage periodic data saves to JSON
-        save_data_thread = threading.Thread(target=self.save_datastore).start()
+        # Only start if thread is not already running (reload_state might be called multiple times)
+        if not self.save_data_thread or not self.save_data_thread.is_alive():
+            self.save_data_thread = threading.Thread(target=self.save_datastore)
+            self.save_data_thread.start()
 
     def rehydrate_entity(self, uuid, entity, processor_override=None):
         """Set the dict back to the dict Watch object"""
@@ -251,7 +290,8 @@ class ChangeDetectionStore:
                 self.__data['watching'] = {}
                 time.sleep(1) # Mainly used for testing to allow all items to flush before running next test
                 for uuid in self.data['watching']:
-                    path = pathlib.Path(os.path.join(self.datastore_path, uuid))
+                    path = pathlib.Path(
+                        os.path.join(self.datastore_path, uuid))
                     if os.path.exists(path):
                         self.delete(uuid)
 
@@ -411,14 +451,19 @@ class ChangeDetectionStore:
             self.sync_to_json()
             return
         else:
-
             try:
                 # Re #286  - First write to a temp file, then confirm it looks OK and rename it
                 # This is a fairly basic strategy to deal with the case that the file is corrupted,
                 # system was out of memory, out of RAM etc
-                with open(self.json_store_path+".tmp", 'w') as json_file:
-                    # Use compact JSON in production for better performance
-                    json.dump(data, json_file, indent=2)
+                if HAS_ORJSON:
+                    # Use orjson for faster serialization
+                    # orjson.dumps() always returns UTF-8 encoded bytes #3611
+                    with open(self.json_store_path+".tmp", 'wb') as json_file:
+                        json_file.write(orjson.dumps(data, option=orjson.OPT_INDENT_2))
+                else:
+                    # Fallback to standard json module
+                    with open(self.json_store_path+".tmp", 'w', encoding='utf-8') as json_file:
+                        json.dump(data, json_file, indent=2, ensure_ascii=False)
                 os.replace(self.json_store_path+".tmp", self.json_store_path)
             except Exception as e:
                 logger.error(f"Error writing JSON!! (Main JSON file save was skipped) : {str(e)}")
@@ -441,7 +486,7 @@ class ChangeDetectionStore:
                 logger.remove()
                 logger.add(sys.stderr)
 
-                logger.critical("Shutting down datastore thread")
+                logger.info(f"Shutting down datastore '{self.datastore_path}' thread")
                 return
 
             if self.needs_write or self.needs_write_urgent:
@@ -480,8 +525,13 @@ class ChangeDetectionStore:
 
         # Load from external config file
         if path.isfile(proxy_list_file):
-            with open(os.path.join(self.datastore_path, "proxies.json")) as f:
-                proxy_list = json.load(f)
+            if HAS_ORJSON:
+                # orjson.loads() expects UTF-8 encoded bytes #3611
+                with open(os.path.join(self.datastore_path, "proxies.json"), 'rb') as f:
+                    proxy_list = orjson.loads(f.read())
+            else:
+                with open(os.path.join(self.datastore_path, "proxies.json"), encoding='utf-8') as f:
+                    proxy_list = json.load(f)
 
         # Mapping from UI config if available
         extras = self.data['settings']['requests'].get('extra_proxies')
@@ -726,6 +776,28 @@ class ChangeDetectionStore:
 
         return updates_available
 
+    def add_notification_url(self, notification_url):
+
+        logger.debug(f">>> Adding new notification_url - '{notification_url}'")
+
+        notification_urls = self.data['settings']['application'].get('notification_urls', [])
+
+        if notification_url in notification_urls:
+            return notification_url
+
+        with self.lock:
+            notification_urls = self.__data['settings']['application'].get('notification_urls', [])
+
+            if notification_url in notification_urls:
+                return notification_url
+
+            # Append and update the datastore
+            notification_urls.append(notification_url)
+            self.__data['settings']['application']['notification_urls'] = notification_urls
+            self.needs_write = True
+
+        return notification_url
+
     # Run all updates
     # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
     #             So therefor - each `update_n` should be very careful about checking if it needs to actually run
@@ -738,7 +810,16 @@ class ChangeDetectionStore:
                 logger.critical(f"Applying update_{update_n}")
                 # Wont exist on fresh installs
                 if os.path.exists(self.json_store_path):
-                    shutil.copyfile(self.json_store_path, os.path.join(self.datastore_path, f"url-watches-before-{update_n}.json"))
+                    i = 0
+                    while True:
+                        i+=1
+                        dest = os.path.join(self.datastore_path, f"url-watches-before-{update_n}-{i}.json")
+                        if not os.path.exists(dest):
+                            logger.debug(f"Copying url-watches.json DB to '{dest}' backup.")
+                            shutil.copyfile(self.json_store_path, dest)
+                            break
+                        else:
+                            logger.warning(f"Backup of url-watches.json '{dest}', DB already exists, trying {i+1}.. ")
 
                 try:
                     update_method = getattr(self, f"update_{update_n}")()
@@ -996,48 +1077,48 @@ class ChangeDetectionStore:
 
 
     # Some notification formats got the wrong name type
-    def update_22(self):
+    def update_23(self):
+
+        def re_run(formats):
+            sys_n_format = self.data['settings']['application'].get('notification_format')
+            key_exists_as_value = next((k for k, v in formats.items() if v == sys_n_format), None)
+            if key_exists_as_value:  # key of "Plain text"
+                logger.success(f"['settings']['application']['notification_format'] '{sys_n_format}' -> '{key_exists_as_value}'")
+                self.data['settings']['application']['notification_format'] = key_exists_as_value
+
+            for uuid, watch in self.data['watching'].items():
+                n_format = self.data['watching'][uuid].get('notification_format')
+                key_exists_as_value = next((k for k, v in formats.items() if v == n_format), None)
+                if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text"
+                    logger.success(f"['watching'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'")
+                    self.data['watching'][uuid]['notification_format'] = key_exists_as_value  # should be 'text' or whatever
+
+            for uuid, tag in self.data['settings']['application']['tags'].items():
+                n_format = self.data['settings']['application']['tags'][uuid].get('notification_format')
+                key_exists_as_value = next((k for k, v in formats.items() if v == n_format), None)
+                if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text"
+                    logger.success(
+                        f"['settings']['application']['tags'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'")
+                    self.data['settings']['application']['tags'][uuid][
+                        'notification_format'] = key_exists_as_value  # should be 'text' or whatever
+
         from .notification import valid_notification_formats
+        formats = deepcopy(valid_notification_formats)
+        re_run(formats)
+        # And in previous versions, it was "text" instead of Plain text, Markdown instead of "Markdown to HTML"
+        formats['text'] = 'Text'
+        formats['markdown'] = 'Markdown'
+        re_run(formats)
 
-        sys_n_format = self.data['settings']['application'].get('notification_format')
-        key_exists_as_value = next((k for k, v in valid_notification_formats.items() if v == sys_n_format), None)
-        if key_exists_as_value: # key of "Plain text"
-            logger.success(f"['settings']['application']['notification_format'] '{sys_n_format}' -> '{key_exists_as_value}'")
-            self.data['settings']['application']['notification_format'] = key_exists_as_value
-
-        for uuid, watch in self.data['watching'].items():
-            n_format = self.data['watching'][uuid].get('notification_format')
-            key_exists_as_value = next((k for k, v in valid_notification_formats.items() if v == n_format), None)
-            if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text"
-                logger.success(f"['watching'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'")
-                self.data['watching'][uuid]['notification_format'] = key_exists_as_value # should be 'text' or whatever
-
-        for uuid, tag in self.data['settings']['application']['tags'].items():
-            n_format = self.data['settings']['application']['tags'][uuid].get('notification_format')
-            key_exists_as_value = next((k for k, v in valid_notification_formats.items() if v == n_format), None)
-            if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text"
-                logger.success(f"['settings']['application']['tags'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'")
-                self.data['settings']['application']['tags'][uuid]['notification_format'] = key_exists_as_value # should be 'text' or whatever
-
-    def add_notification_url(self, notification_url):
-        
-        logger.debug(f">>> Adding new notification_url - '{notification_url}'")
-
-        notification_urls = self.data['settings']['application'].get('notification_urls', [])
-
-        if notification_url in notification_urls:
-            return notification_url
-
-        with self.lock:
-            notification_urls = self.__data['settings']['application'].get('notification_urls', [])
-
-            if notification_url in notification_urls:
-                return notification_url
-
-            # Append and update the datastore
-            notification_urls.append(notification_url)
-            self.__data['settings']['application']['notification_urls'] = notification_urls
-            self.needs_write = True
-
-        return notification_url
 
+    # RSS types should be inline with the same names as notification types
+    def update_24(self):
+        rss_format = self.data['settings']['application'].get('rss_content_format')
+        if not rss_format or 'text' in rss_format:
+            # might have been 'plaintext, 'plain text' or something
+            self.data['settings']['application']['rss_content_format'] = RSS_CONTENT_FORMAT_DEFAULT
+        elif 'html' in rss_format:
+            self.data['settings']['application']['rss_content_format'] = 'htmlcolor'
+        else:
+            # safe fallback to text
+            self.data['settings']['application']['rss_content_format'] = RSS_CONTENT_FORMAT_DEFAULT

changedetectionio/templates/_common_fields.html

@@ -1,6 +1,118 @@
 
 {% from '_helpers.html' import render_field %}
 
+{% macro show_token_placeholders(extra_notification_token_placeholder_info, suffix="") %}
+
+
+    <div class="pure-controls">
+            <span class="pure-form-message-inline">
+        Body for all notifications &dash; You can use <a target="newwindow" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
+    </span><br>
+        <div data-target="#notification-tokens-info{{ suffix }}" class="toggle-show pure-button button-tag button-xsmall">Show
+            token/placeholders
+        </div>
+    </div>
+    <div class="pure-controls" style="display: none;" id="notification-tokens-info{{ suffix }}">
+        <table class="pure-table" id="token-table">
+            <thead>
+            <tr>
+                <th>Token</th>
+                <th>Description</th>
+            </tr>
+            </thead>
+            <tbody>
+            <tr>
+                <td><code>{{ '{{base_url}}' }}</code></td>
+                <td>The URL of the changedetection.io instance you are running.</td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{watch_url}}' }}</code></td>
+                <td>The URL being watched.</td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{watch_uuid}}' }}</code></td>
+                <td>The UUID of the watch.</td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{watch_title}}' }}</code></td>
+                <td>The page title of the watch, uses &lt;title&gt; if not set, falls back to URL</td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{watch_tag}}' }}</code></td>
+                <td>The watch group / tag</td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{preview_url}}' }}</code></td>
+                <td>The URL of the preview page generated by changedetection.io.</td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{diff_url}}' }}</code></td>
+                <td>The URL of the diff output for the watch.</td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{diff}}' }}</code></td>
+                <td>The diff output - only changes, additions, and removals</td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{diff_clean}}' }}</code></td>
+                <td>The diff output - only changes, additions, and removals &dash; <i>Without (added) prefix or colors</i>
+                </td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{diff_added}}' }}</code></td>
+                <td>The diff output - only changes and additions</td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{diff_added_clean}}' }}</code></td>
+                <td>The diff output - only changes and additions &dash; <i>Without (added) prefix or colors</i></td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{diff_removed}}' }}</code></td>
+                <td>The diff output - only changes and removals</td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{diff_removed_clean}}' }}</code></td>
+                <td>The diff output - only changes and removals &dash; <i>Without (added) prefix or colors</i></td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{diff_full}}' }}</code></td>
+                <td>The diff output - full difference output</td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{diff_full_clean}}' }}</code></td>
+                <td>The diff output - full difference output &dash; <i>Without (added) prefix or colors</i></td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{diff_patch}}' }}</code></td>
+                <td>The diff output - patch in unified format</td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{current_snapshot}}' }}</code></td>
+                <td>The current snapshot text contents value, useful when combined with JSON or CSS filters
+                </td>
+            </tr>
+            <tr>
+                <td><code>{{ '{{triggered_text}}' }}</code></td>
+                <td>Text that tripped the trigger from filters</td>
+
+                {% if extra_notification_token_placeholder_info %}
+                    {% for token in extra_notification_token_placeholder_info %}
+                        <tr>
+                            <td><code>{{ '{{' }}{{ token[0] }}{{ '}}' }}</code></td>
+                            <td>{{ token[1] }}</td>
+                        </tr>
+                    {% endfor %}
+                {% endif %}
+            </tbody>
+        </table>
+
+        <span class="pure-form-message-inline">
+        Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
+        For example, an addition or removal could be perceived as a change in some cases. <a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
+        </span>
+    </div>
+{%  endmacro %}
+
 {% macro render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) %}
                         <div class="pure-control-group">
                             {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
@@ -33,6 +145,7 @@
                                 <div id="notification-test-log" style="display: none;"><span class="pure-form-message-inline">Processing..</span></div>
                             </div>
                         </div>
+
                         <div class="pure-control-group grey-form-border">
                             <div class="pure-control-group">
                                 {{ render_field(form.notification_title, class="m-d notification-title", placeholder=settings_application['notification_title']) }}
@@ -40,109 +153,26 @@
                             </div>
                             <div class="pure-control-group">
                                 {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }}
-                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="newwindow" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
-                                </span>
-
-                            </div>
-                            <div class="pure-controls">
-                                <div data-target="#notification-tokens-info" class="toggle-show pure-button button-tag button-xsmall">Show token/placeholders</div>
-                            </div>
-                            <div class="pure-controls" style="display: none;" id="notification-tokens-info">
-                                <table class="pure-table" id="token-table">
-                                    <thead>
-                                    <tr>
-                                        <th>Token</th>
-                                        <th>Description</th>
-                                    </tr>
-                                    </thead>
-                                    <tbody>
-                                    <tr>
-                                        <td><code>{{ '{{base_url}}' }}</code></td>
-                                        <td>The URL of the changedetection.io instance you are running.</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{watch_url}}' }}</code></td>
-                                        <td>The URL being watched.</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{watch_uuid}}' }}</code></td>
-                                        <td>The UUID of the watch.</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{watch_title}}' }}</code></td>
-                                        <td>The page title of the watch, uses &lt;title&gt; if not set, falls back to URL</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{watch_tag}}' }}</code></td>
-                                        <td>The watch group / tag</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{preview_url}}' }}</code></td>
-                                        <td>The URL of the preview page generated by changedetection.io.</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{diff_url}}' }}</code></td>
-                                        <td>The URL of the diff output for the watch.</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff}}' }}</code></td>
-                                        <td>The diff output - only changes, additions, and removals</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff_added}}' }}</code></td>
-                                        <td>The diff output - only changes and additions</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff_removed}}' }}</code></td>
-                                        <td>The diff output - only changes and removals</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{diff_full}}' }}</code></td>
-                                        <td>The diff output - full difference output</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{diff_patch}}' }}</code></td>
-                                        <td>The diff output - patch in unified format</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{current_snapshot}}' }}</code></td>
-                                        <td>The current snapshot text contents value, useful when combined with JSON or CSS filters
-                                        </td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{{ '{{triggered_text}}' }}</code></td>
-                                        <td>Text that tripped the trigger from filters</td>
-
-                                        {% if extra_notification_token_placeholder_info %}
-                                            {% for token in extra_notification_token_placeholder_info %}
-                                                <tr>
-                                                    <td><code>{{ '{{' }}{{ token[0] }}{{ '}}' }}</code></td>
-                                                    <td>{{ token[1] }}</td>
-                                                </tr>
-                                            {% endfor %}
-                                        {% endif %}
-                                    </tbody>
-                                </table>
+                                {{ show_token_placeholders(extra_notification_token_placeholder_info=extra_notification_token_placeholder_info) }}
                                 <div class="pure-form-message-inline">
-                                    <p>
-									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
-                                    For example, an addition or removal could be perceived as a change in some cases. <a target="newwindow" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
-                                    </p>
-                                    <p>
+                                    <ul>
+                                    <li><span class="pure-form-message-inline">
                                         For JSON payloads, use <strong>|tojson</strong> without quotes for automatic escaping, for example - <code>{ "name": {{ '{{ watch_title|tojson }}' }} }</code>
-                                    </p>
-                                    <p>
+                                    </span></li>
+                                    <li><span class="pure-form-message-inline">
                                         URL encoding, use <strong>|urlencode</strong>, for example - <code>gets://hook-website.com/test.php?title={{ '{{ watch_title|urlencode }}' }}</code>
-                                    </p>
-                                    <p>
+                                    </span></li>
+                                    <li><span class="pure-form-message-inline">
                                         Regular-expression replace, use <strong>|regex_replace</strong>, for example -   <code>{{ "{{ \"hello world 123\" | regex_replace('[0-9]+', 'no-more-numbers') }}" }}</code>
-                                    </p>
-                                    <p>
+                                    </span></li>
+                                    <li><span class="pure-form-message-inline">
                                         For a complete reference of all Jinja2 built-in filters, users can refer to the <a href="https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters">https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters</a>
-                                    </p>
+                                    </span></li>
+                                    </ul>
+                                    <br>
                                 </div>
                             </div>
-                            <div class="pure-control-group">
+                            <div class="">
                                 {{ render_field(form.notification_format , class="notification-format") }}
                                 <span class="pure-form-message-inline">Format for all notifications</span>
                             </div>

changedetectionio/templates/base.html

@@ -8,8 +8,13 @@
     <meta name="robots" content="noindex">
     <title>Change Detection{{extra_title}}</title>
     {% if app_rss_token %}
-      <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag_uuid %}- {{active_tag.title}}{% endif %}" href="{{ url_for('rss.feed', tag=active_tag_uuid , token=app_rss_token)}}" >
-    {% endif %}
+        <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag_uuid %}- {{active_tag.title}}{% endif %}" href="{{ url_for('rss.feed', tag=active_tag_uuid, token=app_rss_token, _external=True )}}" >
+
+        {% if rss_uuid_feed %}
+        <link rel="alternate" type="application/rss+xml" title="Feed » {{ rss_uuid_feed['label'] }}" href="{{ rss_uuid_feed['url'] }}" >
+
+        {%- endif -%}
+    {%- endif -%}
     <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}" >
     <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}?v={{ get_css_version() }}" >
     {% if extra_stylesheets %}

changedetectionio/tests/conftest.py

@@ -11,6 +11,7 @@ import os
 import sys
 from loguru import logger
 
+from changedetectionio.flask_app import init_app_secret
 from changedetectionio.tests.util import live_server_setup, new_live_server_setup
 
 # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
@@ -87,7 +88,6 @@ def measure_memory_usage(request):
 def cleanup(datastore_path):
     import glob
     # Unlink test output files
-
     for g in ["*.txt", "*.json", "*.pdf"]:
         files = glob.glob(os.path.join(datastore_path, g))
         for f in files:
@@ -97,34 +97,121 @@ def cleanup(datastore_path):
             if os.path.isfile(f):
                 os.unlink(f)
 
-@pytest.fixture(scope='function', autouse=True)
-def prepare_test_function(live_server):
+def pytest_addoption(parser):
+    """Add custom command-line options for pytest.
 
+    Provides --datastore-path option for specifying custom datastore location.
+    Note: Cannot use -d short option as it's reserved by pytest for debug mode.
+    """
+    parser.addoption(
+        "--datastore-path",
+        action="store",
+        default=None,
+        help="Custom datastore path for tests"
+    )
+
+@pytest.fixture(scope='session')
+def datastore_path(tmp_path_factory, request):
+    """Provide datastore path unique to this worker.
+
+    Supports custom path via --datastore-path/-d flag (mirrors main app).
+
+    CRITICAL for xdist isolation:
+    - Each WORKER gets its own directory
+    - Tests on same worker run SEQUENTIALLY and cleanup between tests
+    - No subdirectories needed since tests don't overlap on same worker
+    - Example: /tmp/test-datastore-gw0/ for worker gw0
+    """
+    # Check for custom path first (mirrors main app's -d flag)
+    custom_path = request.config.getoption("--datastore-path")
+    if custom_path:
+        # Ensure the directory exists
+        os.makedirs(custom_path, exist_ok=True)
+        logger.info(f"Using custom datastore path: {custom_path}")
+        return custom_path
+
+    # Otherwise use default tmp_path_factory logic
+    worker_id = getattr(request.config, 'workerinput', {}).get('workerid', 'master')
+    if worker_id == 'master':
+        path = tmp_path_factory.mktemp("test-datastore")
+    else:
+        path = tmp_path_factory.mktemp(f"test-datastore-{worker_id}")
+    return str(path)
+
+
+@pytest.fixture(scope='function', autouse=True)
+def prepare_test_function(live_server, datastore_path):
+    """Prepare each test with complete isolation.
+
+    CRITICAL for xdist per-test isolation:
+    - Reuses the SAME datastore instance (so blueprint references stay valid)
+    - Clears all watches and state for a clean slate
+    - First watch will get uuid="first"
+    """
     routes = [rule.rule for rule in live_server.app.url_map.iter_rules()]
     if '/test-random-content-endpoint' not in routes:
         logger.debug("Setting up test URL routes")
         new_live_server_setup(live_server)
 
+    # CRITICAL: Point app to THIS test's unique datastore directory
+    live_server.app.config['TEST_DATASTORE_PATH'] = datastore_path
+
+    # CRITICAL: Get datastore and stop it from writing stale data
+    datastore = live_server.app.config.get('DATASTORE')
+
+    # Prevent background thread from writing during cleanup/reload
+    datastore.needs_write = False
+    datastore.needs_write_urgent = False
+
+    # CRITICAL: Clean up any files from previous tests
+    # This ensures a completely clean directory
+    cleanup(datastore_path)
+
+    # CRITICAL: Reload the EXISTING datastore instead of creating a new one
+    # This keeps blueprint references valid (they capture datastore at construction)
+    # reload_state() completely resets the datastore to a clean state
+
+    # Reload state with clean data (no default watches)
+    datastore.reload_state(
+        datastore_path=datastore_path,
+        include_default_watches=False,
+        version_tag=datastore.data.get('version_tag', '0.0.0')
+    )
+    live_server.app.secret_key = init_app_secret(datastore_path)
+    logger.debug(f"prepare_test_function: Reloaded datastore at {hex(id(datastore))}")
+    logger.debug(f"prepare_test_function: Path {datastore.datastore_path}")
 
     yield
-    # Then cleanup/shutdown
-    live_server.app.config['DATASTORE'].data['watching']={}
-    time.sleep(0.3)
-    live_server.app.config['DATASTORE'].data['watching']={}
+
+    # Cleanup: Clear watches again after test
+    try:
+        datastore.data['watching'] = {}
+        datastore.needs_write = True
+    except Exception as e:
+        logger.warning(f"Error during datastore cleanup: {e}")
+
+
+# So the app can also know which test name it was
+@pytest.fixture(autouse=True)
+def set_test_name(request):
+  """Automatically set TEST_NAME env var for every test"""
+  test_name = request.node.name
+  os.environ['PYTEST_CURRENT_TEST'] = test_name
+  yield
+  # Cleanup if needed
 
 
 @pytest.fixture(scope='session')
-def app(request):
-    """Create application for the tests."""
-    datastore_path = "./test-datastore"
+def app(request, datastore_path):
+    """Create application once per worker (session).
 
+    Note: Actual per-test isolation is handled by:
+    - prepare_test_function() recreates datastore and cleans directory
+    - All tests on same worker use same directory (cleaned between tests)
+    """
     # So they don't delay in fetching
     os.environ["MINIMUM_SECONDS_RECHECK_TIME"] = "0"
-    try:
-        os.mkdir(datastore_path)
-    except FileExistsError:
-        pass
-
+    logger.debug(f"Testing with datastore_path={datastore_path}")
     cleanup(datastore_path)
 
     app_config = {'datastore_path': datastore_path, 'disable_checkver' : True}
@@ -147,6 +234,8 @@ def app(request):
     # Disable CSRF while running tests
     app.config['WTF_CSRF_ENABLED'] = False
     app.config['STOP_THREADS'] = True
+    # Store datastore_path so Flask routes can access it
+    app.config['TEST_DATASTORE_PATH'] = datastore_path
 
     def teardown():
         # Stop all threads and services

changedetectionio/tests/custom_browser_url/test_custom_browser_url.py

@@ -73,13 +73,13 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
 
 
 # Requires playwright to be installed
-def test_request_via_custom_browser_url(client, live_server, measure_memory_usage):
+def test_request_via_custom_browser_url(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     # We do this so we can grep the logs of the custom container and see if the request actually went through that container
     do_test(client, live_server, make_test_use_extra_browser=True)
 
 
-def test_request_not_via_custom_browser_url(client, live_server, measure_memory_usage):
+def test_request_not_via_custom_browser_url(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     # We do this so we can grep the logs of the custom container and see if the request actually went through that container
     do_test(client, live_server, make_test_use_extra_browser=False)

changedetectionio/tests/fetchers/test_content.py

@@ -8,7 +8,7 @@ import logging
 
 
 # Requires playwright to be installed
-def test_fetch_webdriver_content(client, live_server, measure_memory_usage):
+def test_fetch_webdriver_content(client, live_server, measure_memory_usage, datastore_path):
     #  live_server_setup(live_server) # Setup on conftest per function
 
     #####################

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -3,7 +3,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 
-def test_execute_custom_js(client, live_server, measure_memory_usage):
+def test_execute_custom_js(client, live_server, measure_memory_usage, datastore_path):
 
    #  live_server_setup(live_server) # Setup on conftest per function
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks
 
 
-def test_preferred_proxy(client, live_server, measure_memory_usage):
+def test_preferred_proxy(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     url = "http://chosen.changedetection.io"
 

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 
-def test_noproxy_option(client, live_server, measure_memory_usage):
+def test_noproxy_option(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     # Run by run_proxy_tests.sh
     # Call this URL then scan the containers that it never went through them

changedetectionio/tests/proxy_list/test_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 # just make a request, we will grep in the docker logs to see it actually got called
-def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     res = client.post(
         url_for("imports.import_page"),

changedetectionio/tests/proxy_list/test_proxy_noconnect.py

@@ -12,7 +12,7 @@ from ... import strtobool
 # FAST_PUPPETEER_CHROME_FETCHER=True PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000 pytest tests/proxy_list/test_proxy_noconnect.py
 # WEBDRIVER_URL=http://127.0.0.1:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py
 
-def test_proxy_noconnect_custom(client, live_server, measure_memory_usage):
+def test_proxy_noconnect_custom(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
     # Goto settings, add our custom one

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -6,7 +6,7 @@ from ..util import live_server_setup, wait_for_all_checks
 import os
 
 # just make a request, we will grep in the docker logs to see it actually got called
-def test_select_custom(client, live_server, measure_memory_usage):
+def test_select_custom(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
     # Goto settings, add our custom one
@@ -50,7 +50,7 @@ def test_select_custom(client, live_server, measure_memory_usage):
     # Now we should see the request in the container logs for "squid-squid-custom" because it will be the only default
 
 
-def test_custom_proxy_validation(client, live_server, measure_memory_usage):
+def test_custom_proxy_validation(client, live_server, measure_memory_usage, datastore_path):
     #  live_server_setup(live_server) # Setup on conftest per function
 
     # Goto settings, add our custom one

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from changedetectionio.tests.util import live_server_setup, wait_for_all_checks, extract_UUID_from_client, delete_all_watches
 
 
-def set_response():
+def set_response(datastore_path):
     import time
     data = """<html>
        <body>
@@ -15,13 +15,13 @@ def set_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(data)
     time.sleep(1)
 
-def test_socks5(client, live_server, measure_memory_usage):
+def test_socks5(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_response()
+    set_response(datastore_path)
 
     # Setup a proxy
     res = client.post(

changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py

@@ -4,7 +4,7 @@ from flask import url_for
 from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
 
 
-def set_response():
+def set_response(datastore_path):
     import time
     data = """<html>
        <body>
@@ -14,15 +14,15 @@ def set_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(data)
     time.sleep(1)
 
 # should be proxies.json mounted from run_proxy_tests.sh already
 # -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json
-def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage):
+def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_response()
+    set_response(datastore_path)
     # Because the socks server should connect back to us
     test_url = url_for('test_endpoint', _external=True) + f"?socks-test-tag={os.getenv('SOCKSTEST', '')}"
     test_url = test_url.replace('localhost.localdomain', 'cdio')

changedetectionio/tests/restock/test_restock.py

@@ -11,7 +11,7 @@ from changedetectionio.notification import (
 )
 
 
-def set_original_response():
+def set_original_response(datastore_path):
     test_return_data = """<html>
        <body>
        <section id=header style="padding: 50px; height: 350px">This is the header which should be ignored always - <span>add to cart</span></section>
@@ -26,13 +26,13 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
 
 
-def set_back_in_stock_response():
+def set_back_in_stock_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -45,14 +45,14 @@ def set_back_in_stock_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
-def test_restock_detection(client, live_server, measure_memory_usage):
+def test_restock_detection(client, live_server, measure_memory_usage, datastore_path):
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
     #assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
    #  live_server_setup(live_server) # Setup on conftest per function
     #####################
@@ -88,24 +88,25 @@ def test_restock_detection(client, live_server, measure_memory_usage):
     assert b'not-in-stock' in res.data # should be out of stock
 
     # Is it correctly shown as in stock
-    set_back_in_stock_response()
+    set_back_in_stock_response(datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'not-in-stock' not in res.data
 
     # We should have a notification
-    wait_for_notification_endpoint_output()
-    assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
-    os.unlink("test-datastore/notification.txt")
+    notification_file = os.path.join(datastore_path, "notification.txt")
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
+    assert os.path.isfile(notification_file), "Notification received"
+    os.unlink(notification_file)
 
     # Default behaviour is to only fire notification when it goes OUT OF STOCK -> IN STOCK
     # So here there should be no file, because we go IN STOCK -> OUT OF STOCK
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     time.sleep(5)
-    assert not os.path.isfile("test-datastore/notification.txt"), "No notification should have fired when it went OUT OF STOCK by default"
+    assert not os.path.isfile(notification_file), "No notification should have fired when it went OUT OF STOCK by default"
 
     # BUT we should see that it correctly shows "not in stock"
     res = client.get(url_for("watchlist.index"))

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -3,8 +3,9 @@ from flask import url_for
 from email import message_from_string
 from email.policy import default as email_policy
 
-from changedetectionio.diff import HTML_REMOVED_STYLE, HTML_ADDED_STYLE, HTML_CHANGED_STYLE
-from changedetectionio.notification_service import NotificationContextData, CUSTOM_LINEBREAK_PLACEHOLDER
+from changedetectionio.diff import HTML_REMOVED_STYLE, HTML_ADDED_STYLE, HTML_CHANGED_STYLE, REMOVED_PLACEMARKER_OPEN, \
+    CHANGED_PLACEMARKER_OPEN, ADDED_PLACEMARKER_OPEN
+from changedetectionio.notification_service import NotificationContextData
 from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
     wait_for_all_checks, \
     set_longer_modified_response, delete_all_watches
@@ -40,9 +41,10 @@ def get_last_message_from_smtp_server():
 
 # Requires running the test SMTP server
 
-def test_check_notification_email_formats_default_HTML(client, live_server, measure_memory_usage):
+def test_check_notification_email_formats_default_HTML(client, live_server, measure_memory_usage, datastore_path):
     ##  live_server_setup(live_server) # Setup on conftest per function
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
+
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
 
@@ -71,7 +73,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     assert b"Watch added" in res.data
 
     wait_for_all_checks(client)
-    set_longer_modified_response()
+    set_longer_modified_response(datastore_path=datastore_path)
     time.sleep(2)
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -99,7 +101,6 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     text_content = text_part.get_content()
     assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
     assert 'fallback-body\r\n' in text_content  # The plaintext part
-    assert CUSTOM_LINEBREAK_PLACEHOLDER not in text_content
 
     # Second part should be text/html
     html_part = parts[1]
@@ -108,12 +109,12 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     assert 'some text<br>' in html_content  # We converted \n from the notification body
     assert 'fallback-body<br>' in html_content  # kept the original <br>
     assert '(added) So let\'s see what happens.<br>' in html_content  # the html part
-    assert CUSTOM_LINEBREAK_PLACEHOLDER not in html_content
     delete_all_watches(client)
 
 
-def test_check_notification_plaintext_format(client, live_server, measure_memory_usage):
-    set_original_response()
+def test_check_notification_plaintext_format(client, live_server, measure_memory_usage, datastore_path):
+    set_original_response(datastore_path=datastore_path)
+
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
 
@@ -122,8 +123,8 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
     res = client.post(
         url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": "some text\n" + default_notification_body,
+              "application-notification_title": "fallback-title {{watch_title}}  {{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}  " + default_notification_title,
+              "application-notification_body": f"some text\n" + default_notification_body + f"\nMore output test\n{ALL_MARKUP_TOKENS}",
               "application-notification_format": 'text',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
@@ -138,7 +139,7 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     time.sleep(2)
 
-    set_longer_modified_response()
+    set_longer_modified_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -146,9 +147,18 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
 
     msg_raw = get_last_message_from_smtp_server()
     assert len(msg_raw) >= 1
-
+    #time.sleep(60)
     # Parse the email properly using Python's email library
     msg = message_from_string(msg_raw, policy=email_policy)
+    # Subject/title got marked up
+    subject = msg['subject']
+    # Subject should always be plaintext and never marked up to anything else
+    assert REMOVED_PLACEMARKER_OPEN not in subject
+    assert CHANGED_PLACEMARKER_OPEN not in subject
+    assert ADDED_PLACEMARKER_OPEN not in subject
+    assert 'diff added didnt split' not in subject
+    assert '(changed) Which is across' in subject
+    assert 'PLACEMARKER' not in subject
 
     # The email should be plain text only (not multipart)
     assert not msg.is_multipart()
@@ -164,8 +174,9 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
 
 
 
-def test_check_notification_html_color_format(client, live_server, measure_memory_usage):
-    set_original_response()
+def test_check_notification_html_color_format(client, live_server, measure_memory_usage, datastore_path):
+    set_original_response(datastore_path=datastore_path)
+
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
 
@@ -174,7 +185,7 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
     res = client.post(
         url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_title": "fallback-title {{watch_title}} - diff_added_lines_test : '{{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}' " + default_notification_title,
               "application-notification_body": f"some text\n{default_notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}",
               "application-notification_format": 'htmlcolor',
               "requests-time_between_check-minutes": 180,
@@ -195,7 +206,7 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
     assert b"Watch added" in res.data
 
     wait_for_all_checks(client)
-    set_longer_modified_response()
+    set_longer_modified_response(datastore_path=datastore_path)
     time.sleep(2)
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -208,6 +219,19 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
 
     # Parse the email properly using Python's email library
     msg = message_from_string(msg_raw, policy=email_policy)
+    # Subject/title got marked up
+    subject = msg['subject']
+    # Subject should always be plaintext and never marked up to anything else
+    assert REMOVED_PLACEMARKER_OPEN not in subject
+    assert CHANGED_PLACEMARKER_OPEN not in subject
+    assert ADDED_PLACEMARKER_OPEN not in subject
+    assert 'diff added didnt split' not in subject
+    assert '(changed) Which is across' in subject
+    assert 'PLACEMARKER' not in subject
+    assert 'head title' in subject
+    assert "span" not in subject
+    assert 'background-color' not in subject
+
 
     # The email should have two bodies (multipart/alternative with text/plain and text/html)
     assert msg.is_multipart()
@@ -235,8 +259,9 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
     assert 'some text<br>' in html_content
     delete_all_watches(client)
 
-def test_check_notification_markdown_format(client, live_server, measure_memory_usage):
-    set_original_response()
+def test_check_notification_markdown_format(client, live_server, measure_memory_usage, datastore_path):
+    set_original_response(datastore_path=datastore_path)
+
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
 
@@ -245,7 +270,7 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_
     res = client.post(
         url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_title": "fallback-title  diff_added_lines_test : '{{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}' " + default_notification_title,
               "application-notification_body": "*header*\n\nsome text\n" + default_notification_body,
               "application-notification_format": 'markdown',
               "requests-time_between_check-minutes": 180,
@@ -266,7 +291,7 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_
     assert b"Watch added" in res.data
 
     wait_for_all_checks(client)
-    set_longer_modified_response()
+    set_longer_modified_response(datastore_path=datastore_path)
     time.sleep(2)
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -283,6 +308,14 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_
     # The email should have two bodies (multipart/alternative with text/plain and text/html)
     assert msg.is_multipart()
     assert msg.get_content_type() == 'multipart/alternative'
+    subject = msg['subject']
+    # Subject should always be plaintext and never marked up to anything else
+    assert REMOVED_PLACEMARKER_OPEN not in subject
+    assert CHANGED_PLACEMARKER_OPEN not in subject
+    assert ADDED_PLACEMARKER_OPEN not in subject
+    assert 'diff added didnt split' not in subject
+    assert '(changed) Which is across' in subject
+
 
     # Get the parts
     parts = list(msg.iter_parts())
@@ -301,16 +334,20 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_
     assert html_part.get_content_type() == 'text/html'
     html_content = html_part.get_content()
     assert '<p><em>header</em></p>' in html_content
-    assert '<strong>So let\'s see what happens.</strong><br>' in html_content # Additions are <strong> in markdown
+    assert '<strong>So let\'s see what happens.</strong><br />' in html_content # Additions are <strong> in markdown
+    # the '<br />' will come from apprises conversion, not from our code, we would rather use '<br>' correctly
+    # the '<br />' is actually a nice way to know if apprise done the conversion.
+
     delete_all_watches(client)
 
 # Custom notification body with HTML, that is either sent as HTML or rendered to plaintext and sent
-def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage):
+def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage, datastore_path):
 
     # HTML problems? see this
     # https://github.com/caronc/apprise/issues/633
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
+
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
     notification_body = f"""<!DOCTYPE html>
 <html lang="en">
@@ -350,7 +387,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     #################################### FIRST SITUATION, PLAIN TEXT NOTIFICATION IS WANTED BUT WE HAVE HTML IN OUR TEMPLATE AND CONTENT ##########
     wait_for_all_checks(client)
-    set_longer_modified_response()
+    set_longer_modified_response(datastore_path=datastore_path)
     time.sleep(2)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
@@ -375,7 +412,8 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
 
     #################################### SECOND SITUATION, HTML IS CORRECTLY PASSED THROUGH TO THE EMAIL ####################
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
+
     # Now override as HTML format
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
@@ -424,10 +462,11 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     delete_all_watches(client)
 
-def test_check_plaintext_document_plaintext_notification_smtp(client, live_server, measure_memory_usage):
+def test_check_plaintext_document_plaintext_notification_smtp(client, live_server, measure_memory_usage, datastore_path):
     """When following a plaintext document, notification in Plain Text format is sent correctly"""
+    import os
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("Some nice plain text\nwhich we add some extra data\nover here\n")
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
@@ -454,7 +493,7 @@ def test_check_plaintext_document_plaintext_notification_smtp(client, live_serve
     wait_for_all_checks(client)
 
     # Change the content
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("Some nice plain text\nwhich we add some extra data\nAnd let's talk about <title> tags\nover here\n")
 
 
@@ -476,10 +515,11 @@ def test_check_plaintext_document_plaintext_notification_smtp(client, live_serve
     assert '<pre' not in body
     delete_all_watches(client)
 
-def test_check_plaintext_document_html_notifications(client, live_server, measure_memory_usage):
+def test_check_plaintext_document_html_notifications(client, live_server, measure_memory_usage, datastore_path):
     """When following a plaintext document, notification in Plain Text format is sent correctly"""
+    import os
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("    Some nice plain text\nwhich we add some extra data\nover here\n")
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
@@ -506,7 +546,7 @@ def test_check_plaintext_document_html_notifications(client, live_server, measur
     wait_for_all_checks(client)
 
     # Change the content
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("    Some nice plain text\nwhich we add some extra data\nAnd let's talk about <title> tags\nover here\n")
 
 
@@ -545,6 +585,7 @@ def test_check_plaintext_document_html_notifications(client, live_server, measur
     # Should be the HTML, but not HTML Color
     assert 'background-color' not in html_content
     assert '<br>(added) And let&#39;s talk about &lt;title&gt; tags<br>' in html_content
+    assert 'PLACEMARKER' not in html_content
     assert '&lt;br' not in html_content
     assert '<pre role="article"' in html_content # Should have got wrapped nicely in email_helpers.py
 
@@ -554,10 +595,11 @@ def test_check_plaintext_document_html_notifications(client, live_server, measur
     delete_all_watches(client)
 
 
-def test_check_plaintext_document_html_color_notifications(client, live_server, measure_memory_usage):
+def test_check_plaintext_document_html_color_notifications(client, live_server, measure_memory_usage, datastore_path):
     """When following a plaintext document, notification in Plain Text format is sent correctly"""
+    import os
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("Some nice plain text\nwhich we add some extra data\nover here\n")
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
@@ -585,7 +627,7 @@ def test_check_plaintext_document_html_color_notifications(client, live_server,
     wait_for_all_checks(client)
 
     # Change the content
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("Some nice plain text\nwhich we add some extra data\nAnd let's talk about <title> tags\nover here\n")
 
     time.sleep(1)
@@ -626,10 +668,11 @@ def test_check_plaintext_document_html_color_notifications(client, live_server,
     assert '<pre role="article"' in html_content # Should have got wrapped nicely in email_helpers.py
     delete_all_watches(client)
 
-def test_check_html_document_plaintext_notification(client, live_server, measure_memory_usage):
+def test_check_html_document_plaintext_notification(client, live_server, measure_memory_usage, datastore_path):
     """When following a HTML document, notification in Plain Text format is sent correctly"""
+    import os
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("<html><body>some stuff<br>and more stuff<br>and even more stuff<br></body></html>")
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
@@ -656,7 +699,7 @@ def test_check_html_document_plaintext_notification(client, live_server, measure
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("<html><body>sxome stuff<br>and more stuff<br>lets slip this in<br>and this in<br>and even more stuff<br>&lt;tag&gt;</body></html>")
 
     time.sleep(0.1)
@@ -673,6 +716,7 @@ def test_check_html_document_plaintext_notification(client, live_server, measure
 
     assert '<tag>' in body # Should have got converted from original HTML to plaintext
     assert '(changed) some stuff\r\n' in body
+    assert 'PLACEMARKER' not in body
     assert '(into) sxome stuff\r\n' in body
     assert '(added) lets slip this in\r\n' in body
     assert '(added) and this in\r\n' in body
@@ -682,9 +726,10 @@ def test_check_html_document_plaintext_notification(client, live_server, measure
     delete_all_watches(client)
 
 
-def test_check_html_notification_with_apprise_format_is_html(client, live_server, measure_memory_usage):
+def test_check_html_notification_with_apprise_format_is_html(client, live_server, measure_memory_usage, datastore_path):
     ##  live_server_setup(live_server) # Setup on conftest per function
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
+
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com&format=html'
 
@@ -713,7 +758,7 @@ def test_check_html_notification_with_apprise_format_is_html(client, live_server
     assert b"Watch added" in res.data
 
     wait_for_all_checks(client)
-    set_longer_modified_response()
+    set_longer_modified_response(datastore_path=datastore_path)
     time.sleep(2)
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -741,7 +786,6 @@ def test_check_html_notification_with_apprise_format_is_html(client, live_server
     text_content = text_part.get_content()
     assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
     assert 'fallback-body\r\n' in text_content  # The plaintext part
-    assert CUSTOM_LINEBREAK_PLACEHOLDER not in text_content
 
     # Second part should be text/html
     html_part = parts[1]
@@ -750,5 +794,4 @@ def test_check_html_notification_with_apprise_format_is_html(client, live_server
     assert 'some text<br>' in html_content  # We converted \n from the notification body
     assert 'fallback-body<br>' in html_content  # kept the original <br>
     assert '(added) So let\'s see what happens.<br>' in html_content  # the html part
-    assert CUSTOM_LINEBREAK_PLACEHOLDER not in html_content
     delete_all_watches(client)

changedetectionio/tests/test_access_control.py

@@ -2,7 +2,7 @@ from .util import live_server_setup, wait_for_all_checks
 from flask import url_for
 import time
 
-def test_check_access_control(app, client, live_server, measure_memory_usage):
+def test_check_access_control(app, client, live_server, measure_memory_usage, datastore_path):
     # Still doesnt work, but this is closer.
    #  live_server_setup(live_server) # Setup on conftest per function
 

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os.path
+import os
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output, delete_all_watches
@@ -9,7 +10,7 @@ import time
 from ..diff import ADDED_PLACEMARKER_OPEN
 
 
-def set_original(excluding=None, add_line=None):
+def set_original(datastore_path, excluding=None, add_line=None):
     test_return_data = """<html>
      <body>
      <p>Some initial text</p>
@@ -35,16 +36,16 @@ def set_original(excluding=None, add_line=None):
 
         test_return_data = output
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
-# def test_setup(client, live_server, measure_memory_usage):
+# def test_setup(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage):
+def test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage, datastore_path):
 
     # Give the endpoint time to spin up
-    set_original()
+    set_original(datastore_path=datastore_path)
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
@@ -64,9 +65,10 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
               "time_between_check_use_default": "y"},
         follow_redirects=True
     )
+
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
-    set_original(excluding='Something irrelevant')
+    set_original(excluding='Something irrelevant', datastore_path=datastore_path)
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -77,7 +79,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     assert b'has-unread-changes' not in res.data
 
     # The trigger line is REMOVED,  this should trigger
-    set_original(excluding='The golden line')
+    set_original(excluding='The golden line', datastore_path=datastore_path)
 
     # Check in the processor here what's going on, its triggering empty-reply and no change.
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -92,7 +94,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     time.sleep(0.2)
 
     time.sleep(1)
-    set_original(excluding=None)
+    set_original(excluding=None, datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     time.sleep(1)
@@ -100,7 +102,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     assert b'has-unread-changes' not in res.data
 
     # Remove it again, and we should get a trigger
-    set_original(excluding='The golden line')
+    set_original(excluding='The golden line', datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -109,7 +111,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     delete_all_watches(client)
 
 
-def test_check_add_line_contains_trigger(client, live_server, measure_memory_usage):
+def test_check_add_line_contains_trigger(client, live_server, measure_memory_usage, datastore_path):
     
     delete_all_watches(client)
     time.sleep(1)
@@ -132,7 +134,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     )
     assert b'Settings updated' in res.data
 
-    set_original()
+    set_original(datastore_path=datastore_path)
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
@@ -155,7 +157,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     )
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
-    set_original(excluding='Something irrelevant')
+    set_original(excluding='Something irrelevant', datastore_path=datastore_path)
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -166,7 +168,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     assert b'has-unread-changes' not in res.data
 
     # The trigger line is ADDED,  this should trigger
-    set_original(add_line='<p>Oh yes please</p>')
+    set_original(add_line='<p>Oh yes please</p>', datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -174,9 +176,9 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     assert b'has-unread-changes' in res.data
 
     # Takes a moment for apprise to fire
-    wait_for_notification_endpoint_output()
-    assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file"
-    with open("test-datastore/notification.txt", 'rb') as f:
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
+    assert os.path.isfile(os.path.join(datastore_path, "notification.txt")), "Notification fired because I can see the output file"
+    with open(os.path.join(datastore_path, "notification.txt"), 'rb') as f:
         response = f.read()
         assert ADDED_PLACEMARKER_OPEN.encode('utf-8') not in response #  _apply_diff_filtering shouldnt add something here
         assert b'-Oh yes please' in response

changedetectionio/tests/test_api.py

@@ -3,12 +3,13 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
+import os
 
 import json
 import uuid
 
 
-def set_original_response():
+def set_original_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -21,12 +22,12 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
 
-def set_modified_response():
+def set_modified_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -39,7 +40,7 @@ def set_modified_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
     return None
@@ -52,17 +53,17 @@ def is_valid_uuid(val):
         return False
 
 
-# def test_setup(client, live_server, measure_memory_usage):
+# def test_setup(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
 
-def test_api_simple(client, live_server, measure_memory_usage):
+def test_api_simple(client, live_server, measure_memory_usage, datastore_path):
     
 
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
     # Create a watch
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
     # Validate bad URL
     test_url = url_for('test_endpoint', _external=True )
@@ -111,7 +112,7 @@ def test_api_simple(client, live_server, measure_memory_usage):
     time.sleep(1)
     wait_for_all_checks(client)
 
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
     # Trigger recheck of all ?recheck_all=1
     client.get(
         url_for("createwatch", recheck_all='1'),
@@ -244,7 +245,7 @@ def test_api_simple(client, live_server, measure_memory_usage):
     )
     assert len(res.json) == 0, "Watch list should be empty"
 
-def test_access_denied(client, live_server, measure_memory_usage):
+def test_access_denied(client, live_server, measure_memory_usage, datastore_path):
     # `config_api_token_enabled` Should be On by default
     res = client.get(
         url_for("createwatch")
@@ -289,11 +290,11 @@ def test_access_denied(client, live_server, measure_memory_usage):
     )
     assert b"Settings updated." in res.data
 
-def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
+def test_api_watch_PUT_update(client, live_server, measure_memory_usage, datastore_path):
 
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
     # Create a watch
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
     test_url = url_for('test_endpoint', _external=True)
 
     # Create new
@@ -398,7 +399,7 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
     delete_all_watches(client)
 
 
-def test_api_import(client, live_server, measure_memory_usage):
+def test_api_import(client, live_server, measure_memory_usage, datastore_path):
 
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -420,7 +421,7 @@ def test_api_import(client, live_server, measure_memory_usage):
     res = client.get(url_for('tags.tags_overview_page'))
     assert b'import-test' in res.data
 
-def test_api_conflict_UI_password(client, live_server, measure_memory_usage):
+def test_api_conflict_UI_password(client, live_server, measure_memory_usage, datastore_path):
 
     
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
@@ -438,7 +439,7 @@ def test_api_conflict_UI_password(client, live_server, measure_memory_usage):
     assert b"Password protection enabled." in res.data
 
     # Create a watch
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
     test_url = url_for('test_endpoint', _external=True)
 
     # Create new

changedetectionio/tests/test_api_notifications.py

@@ -4,7 +4,7 @@ from flask import url_for
 from .util import live_server_setup
 import json
 
-def test_api_notifications_crud(client, live_server, measure_memory_usage):
+def test_api_notifications_crud(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 

changedetectionio/tests/test_api_openapi.py

@@ -12,7 +12,7 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 
 
-def test_openapi_validation_invalid_content_type_on_create_watch(client, live_server, measure_memory_usage):
+def test_openapi_validation_invalid_content_type_on_create_watch(client, live_server, measure_memory_usage, datastore_path):
     """Test that creating a watch with invalid content-type triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -29,7 +29,7 @@ def test_openapi_validation_invalid_content_type_on_create_watch(client, live_se
     assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
 
 
-def test_openapi_validation_missing_required_field_create_watch(client, live_server, measure_memory_usage):
+def test_openapi_validation_missing_required_field_create_watch(client, live_server, measure_memory_usage, datastore_path):
     """Test that creating a watch without required URL field triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -46,7 +46,7 @@ def test_openapi_validation_missing_required_field_create_watch(client, live_ser
     assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
 
 
-def test_openapi_validation_invalid_field_in_request_body(client, live_server, measure_memory_usage):
+def test_openapi_validation_invalid_field_in_request_body(client, live_server, measure_memory_usage, datastore_path):
     """Test that including invalid fields triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -83,7 +83,7 @@ def test_openapi_validation_invalid_field_in_request_body(client, live_server, m
     assert b"Additional properties are not allowed" in res.data, "Should contain validation error about additional properties"
 
 
-def test_openapi_validation_import_wrong_content_type(client, live_server, measure_memory_usage):
+def test_openapi_validation_import_wrong_content_type(client, live_server, measure_memory_usage, datastore_path):
     """Test that import endpoint with wrong content-type triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -100,7 +100,7 @@ def test_openapi_validation_import_wrong_content_type(client, live_server, measu
     assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
 
 
-def test_openapi_validation_import_correct_content_type_succeeds(client, live_server, measure_memory_usage):
+def test_openapi_validation_import_correct_content_type_succeeds(client, live_server, measure_memory_usage, datastore_path):
     """Test that import endpoint with correct content-type succeeds (positive test)."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -117,7 +117,7 @@ def test_openapi_validation_import_correct_content_type_succeeds(client, live_se
     assert len(res.json) == 2, "Should import 2 URLs"
 
 
-def test_openapi_validation_get_requests_bypass_validation(client, live_server, measure_memory_usage):
+def test_openapi_validation_get_requests_bypass_validation(client, live_server, measure_memory_usage, datastore_path):
     """Test that GET requests bypass OpenAPI validation entirely."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -141,7 +141,7 @@ def test_openapi_validation_get_requests_bypass_validation(client, live_server,
     assert isinstance(res.json, dict), "Should return JSON dictionary for watch list"
 
 
-def test_openapi_validation_create_tag_missing_required_title(client, live_server, measure_memory_usage):
+def test_openapi_validation_create_tag_missing_required_title(client, live_server, measure_memory_usage, datastore_path):
     """Test that creating a tag without required title triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -158,7 +158,7 @@ def test_openapi_validation_create_tag_missing_required_title(client, live_serve
     assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
 
 
-def test_openapi_validation_watch_update_allows_partial_updates(client, live_server, measure_memory_usage):
+def test_openapi_validation_watch_update_allows_partial_updates(client, live_server, measure_memory_usage, datastore_path):
     """Test that watch updates allow partial updates without requiring all fields (positive test)."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 

changedetectionio/tests/test_api_search.py

@@ -6,7 +6,7 @@ import time
 from .util import live_server_setup, wait_for_all_checks
 
 
-def test_api_search(client, live_server, measure_memory_usage):
+def test_api_search(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 

changedetectionio/tests/test_api_tags.py

@@ -5,13 +5,14 @@ from .util import live_server_setup, wait_for_all_checks, set_original_response
 import json
 import time
 
-def test_api_tags_listing(client, live_server, measure_memory_usage):
+def test_api_tags_listing(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
     tag_title = 'Test Tag'
 
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
+
 
     res = client.get(
         url_for("tags"),

changedetectionio/tests/test_auth.py

@@ -5,7 +5,7 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 
 # test pages with http://username@password:foobar.com/ work
-def test_basic_auth(client, live_server, measure_memory_usage):
+def test_basic_auth(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
 

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -3,9 +3,10 @@
 import time
 from flask import url_for
 from .util import live_server_setup, extract_UUID_from_client, wait_for_all_checks
+import os
 
 
-def set_response_with_ldjson():
+def set_response_with_ldjson(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -55,11 +56,11 @@ def set_response_with_ldjson():
      </html>
 """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-def set_response_without_ldjson():
+def set_response_without_ldjson(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -72,17 +73,17 @@ def set_response_without_ldjson():
      </html>
 """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-# def test_setup(client, live_server, measure_memory_usage):
+# def test_setup(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
 # actually only really used by the distll.io importer, but could be handy too
-def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage):
+def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage, datastore_path):
     
-    set_response_with_ldjson()
+    set_response_with_ldjson(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -121,7 +122,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
 
     ##########################################################################################
     # And we shouldnt see the offer
-    set_response_without_ldjson()
+    set_response_without_ldjson(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -151,7 +152,7 @@ def _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_
     client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
 
 
-def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usage):
+def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usage, datastore_path):
     
     test_return_data = """
             <html>
@@ -181,7 +182,7 @@ def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usa
             <div class="yes">Some extra stuff</div>
             </body></html>
      """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
     _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=True)
@@ -215,7 +216,7 @@ def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usa
     #         <div class="yes">Some extra stuff</div>
     #         </body></html>
     #  """
-    # with open("test-datastore/endpoint-content.txt", "w") as f:
+    # with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
     #     f.write(test_return_data)
     #
     # _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=False)

changedetectionio/tests/test_backend.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import os
 
 import time
 from flask import url_for
@@ -16,20 +17,11 @@ def test_inscriptus():
     assert stripped_text_from_html == 'test!\nok man'
 
 
-def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
-    set_original_response()
-   #  live_server_setup(live_server) # Setup on conftest per function
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path):
+    set_original_response(datastore_path=datastore_path)
 
-    # Add our URL to the import page
-    res = client.post(
-        url_for("imports.import_page"),
-        data={"urls": url_for('test_endpoint', _external=True)},
-        follow_redirects=True
-    )
+    uuid = client.application.config.get('DATASTORE').add_watch(url=url_for('test_endpoint', _external=True))
 
-    assert b"1 Imported" in res.data
-
-    wait_for_all_checks(client)
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(3):
@@ -60,7 +52,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     assert b'foobar-detection' not in res.data
 
     # Make a change
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     # Force recheck
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -85,10 +77,9 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     assert b'<rss' in res.data
 
     # re #16 should have the diff in here too
-    assert b'(into) which has this one new line' in res.data
+    assert b'which has this one new line' in res.data
     assert b'CDATA' in res.data
 
-    assert expected_url.encode('utf-8') in res.data
 #
     # Following the 'diff' link, it should no longer display as 'has-unread-changes' even after we recheck it a few times
     res = client.get(url_for("ui.ui_views.diff_history_page", uuid=uuid))
@@ -114,37 +105,19 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
         # It should report nothing found (no new 'has-unread-changes' class)
         res = client.get(url_for("watchlist.index"))
 
-
         assert b'has-unread-changes' not in res.data
         assert b'class="has-unread-changes' not in res.data
         assert b'head title' in res.data  # Should be ON by default
         assert b'test-endpoint' in res.data
 
     # Recheck it but only with a title change, content wasnt changed
-    set_original_response(extra_title=" and more")
+    set_original_response(datastore_path=datastore_path, extra_title=" and more")
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'head title and more' in res.data
 
-    # disable <title> pickup
-    res = client.post(
-        url_for("settings.settings_page"),
-        data={"application-ui-use_page_title_in_list": "", "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("watchlist.index"))
-    assert b'has-unread-changes' in res.data
-    assert b'class="has-unread-changes' in res.data
-    assert b'head title' not in res.data  # should now be off
-
-
     # Be sure the last_viewed is going to be greater than the last snapshot
     time.sleep(1)
 
@@ -165,9 +138,66 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # Cleanup everything
     delete_all_watches(client)
 
+def test_title_scraper(client, live_server, measure_memory_usage, datastore_path):
+
+    set_original_response(datastore_path=datastore_path)
+    uuid = client.application.config.get('DATASTORE').add_watch(url=url_for('test_endpoint', _external=True))
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks()
+
+    # It should report nothing found (no new 'has-unread-changes' class)
+    res = client.get(url_for("watchlist.index"))
+
+    assert b'head title' in res.data  # Should be ON by default
+
+    # Recheck it but only with a title change, content wasnt changed
+    set_original_response(datastore_path=datastore_path, extra_title=" and more")
+
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    res = client.get(url_for("watchlist.index"))
+    assert b'head title and more' in res.data
+
+    # disable <title> pickup
+    res = client.post(
+        url_for("settings.settings_page"),
+        data={"application-ui-use_page_title_in_list": "",
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    set_original_response(datastore_path=datastore_path, extra_title=" SHOULD NOT APPEAR")
+
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    res = client.get(url_for("watchlist.index"))
+    assert b'SHOULD NOT APPEAR' not in res.data
+
+    delete_all_watches(client)
+
+def test_title_scraper_html_only(client, live_server, measure_memory_usage, datastore_path):
+
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+        f.write('"My text document\nWhere I talk about <title>\nwhich should not get registered\n</title>')
+
+    test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
+
+    uuid = client.application.config.get('DATASTORE').add_watch(test_url)
+    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks()
+
+    # It should report nothing found (no new 'has-unread-changes' class)
+    res = client.get(url_for("watchlist.index"))
+
+    assert b'which should not get registered' not in res.data  # Should be ON by default
+    assert not live_server.app.config['DATASTORE'].data['watching'][uuid].get('title')
+
+
+
 
 # Server says its plaintext, we should always treat it as plaintext, and then if they have a filter, try to apply that
-def test_requests_timeout(client, live_server, measure_memory_usage):
+def test_requests_timeout(client, live_server, measure_memory_usage, datastore_path):
     delay = 2
     test_url = url_for('test_endpoint', delay=delay, _external=True)
 
@@ -205,7 +235,7 @@ def test_requests_timeout(client, live_server, measure_memory_usage):
     res = client.get(url_for("watchlist.index"))
     assert b'Read timed out' not in res.data
 
-def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage):
+def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage, datastore_path):
     """
 
     https://github.com/dgtlmoon/changedetection.io/issues/3434
@@ -220,7 +250,7 @@ def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage):
     :param measure_memory_usage:
     :return:
     """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""some random text that should be split by line
 and not parsed with html_to_text
 this way we know that it correctly parsed as plain text
@@ -264,7 +294,7 @@ got it\r\n
     delete_all_watches(client)
 
 
-def test_standard_text_plain(client, live_server, measure_memory_usage):
+def test_standard_text_plain(client, live_server, measure_memory_usage, datastore_path):
     """
 
     https://github.com/dgtlmoon/changedetection.io/issues/3434
@@ -279,7 +309,7 @@ def test_standard_text_plain(client, live_server, measure_memory_usage):
     :param measure_memory_usage:
     :return:
     """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""some random text that should be split by line
 and not parsed with html_to_text
 <title>Even this title should stay because we are just plain text</title>
@@ -325,9 +355,9 @@ got it\r\n
     delete_all_watches(client)
 
 # Server says its plaintext, we should always treat it as plaintext
-def test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage):
+def test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage, datastore_path):
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""<?xml version="1.0" encoding="utf-8"?>
 <resources xmlns:tools="http://schemas.android.com/tools">
     <!--Activity and fragment titles-->
@@ -353,10 +383,10 @@ def test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage
     delete_all_watches(client)
 
 # Server says its plaintext, we should always treat it as plaintext, and then if they have a filter, try to apply that
-def test_plaintext_even_if_xml_content_and_can_apply_filters(client, live_server, measure_memory_usage):
+def test_plaintext_even_if_xml_content_and_can_apply_filters(client, live_server, measure_memory_usage, datastore_path):
 
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""<?xml version="1.0" encoding="utf-8"?>
 <resources xmlns:tools="http://schemas.android.com/tools">
     <!--Activity and fragment titles-->
@@ -375,7 +405,10 @@ def test_plaintext_even_if_xml_content_and_can_apply_filters(client, live_server
         follow_redirects=True
     )
 
-    assert b'&lt;string name=&#34;feed_update_receiver_name&#34;' in res.data
+    # Check that the string element with the correct name attribute is present
+    # Note: namespace declarations may be included when extracting elements, which is correct XML behavior
+    assert b'feed_update_receiver_name' in res.data
+    assert b'Abonnementen bijwerken' in res.data
     assert b'&lt;foobar' not in res.data
 
     res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_backup.py

@@ -8,13 +8,11 @@ import re
 import time
 
 
-def test_backup(client, live_server, measure_memory_usage):
+def test_backup(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
 
     # Add our URL to the import page
     res = client.post(
@@ -31,7 +29,7 @@ def test_backup(client, live_server, measure_memory_usage):
         url_for("backups.request_backup"),
         follow_redirects=True
     )
-    time.sleep(2)
+    time.sleep(4)
 
     res = client.get(
         url_for("backups.index"),

changedetectionio/tests/test_basic_socketio.py

@@ -10,11 +10,12 @@ from .util import (
 )
 from loguru import logger
 
-def run_socketio_watch_update_test(client, live_server, password_mode=""):
+def run_socketio_watch_update_test(client, live_server, password_mode="", datastore_path=""):
     """Test that the socketio emits a watch update event when content changes"""
 
     # Set up the test server
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
+
 
     # Get the SocketIO instance from the app
     from changedetectionio.flask_app import app
@@ -47,7 +48,7 @@ def run_socketio_watch_update_test(client, live_server, password_mode=""):
     socketio_test_client.get_received()
 
     # Make a change to trigger an update
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     # Force recheck
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -105,11 +106,11 @@ def run_socketio_watch_update_test(client, live_server, password_mode=""):
     # Clean up
     client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
 
-def test_everything(live_server, client):
+def test_everything(live_server, client, measure_memory_usage, datastore_path):
 
    #  live_server_setup(live_server) # Setup on conftest per function
 
-    run_socketio_watch_update_test(password_mode="", live_server=live_server, client=client)
+    run_socketio_watch_update_test(password_mode="", live_server=live_server, client=client, datastore_path=datastore_path)
 
     ############################ Password required auth check ##############################
 
@@ -124,7 +125,7 @@ def test_everything(live_server, client):
 
     assert b"Password protection enabled." in res.data
 
-    run_socketio_watch_update_test(password_mode="not logged in, should exit on connect", live_server=live_server, client=client)
+    run_socketio_watch_update_test(password_mode="not logged in, should exit on connect", live_server=live_server, client=client, datastore_path=datastore_path)
     res = client.post(
         url_for("login"),
         data={"password": "foobar"},
@@ -133,4 +134,4 @@ def test_everything(live_server, client):
 
     # Yes we are correctly logged in
     assert b"LOG OUT" in res.data
-    run_socketio_watch_update_test(password_mode="should be like normal", live_server=live_server, client=client)
+    run_socketio_watch_update_test(password_mode="should be like normal", live_server=live_server, client=client, datastore_path=datastore_path)

changedetectionio/tests/test_block_while_text_present.py

@@ -4,8 +4,9 @@ import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 from changedetectionio import html_tools
+import os
 
-def set_original_ignore_response():
+def set_original_ignore_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -17,11 +18,11 @@ def set_original_ignore_response():
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-def set_modified_original_ignore_response():
+def set_modified_original_ignore_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some NEW nice initial text<br>
@@ -36,12 +37,12 @@ def set_modified_original_ignore_response():
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
 # Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
-def set_modified_response_minus_block_text():
+def set_modified_response_minus_block_text(datastore_path):
     test_return_data = """<html>
        <body>
      Some NEW nice initial text<br>
@@ -56,16 +57,16 @@ def set_modified_response_minus_block_text():
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-def test_check_block_changedetection_text_NOT_present(client, live_server, measure_memory_usage):
+def test_check_block_changedetection_text_NOT_present(client, live_server, measure_memory_usage, datastore_path):
 
    #  live_server_setup(live_server) # Setup on conftest per function
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "out of stoCk\r\nfoobar"
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path=datastore_path)
 
 
     # Add our URL to the import page
@@ -109,7 +110,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     assert b'/test-endpoint' in res.data
 
     # The page changed, BUT the text is still there, just the rest of it changes, we should not see a change
-    set_modified_original_ignore_response()
+    set_modified_original_ignore_response(datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -123,7 +124,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
 
     # 2548
     # Going back to the ORIGINAL should NOT trigger a change
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -131,10 +132,11 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
 
 
     # Now we set a change where the text is gone AND its different content, it should now trigger
-    set_modified_response_minus_block_text()
+    set_modified_response_minus_block_text(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
+
     assert b'has-unread-changes' in res.data
 
 

changedetectionio/tests/test_clone.py

@@ -3,12 +3,13 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
+import os
 
 
-def test_clone_functionality(client, live_server, measure_memory_usage):
+def test_clone_functionality(client, live_server, measure_memory_usage, datastore_path):
 
    #  live_server_setup(live_server) # Setup on conftest per function
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("<html><body>Some content</body></html>")
 
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_conditions.py

@@ -1,13 +1,14 @@
 #!/usr/bin/env python3
 import json
 import time
+import os
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 from ..model import CONDITIONS_MATCH_LOGIC_DEFAULT
 
 
-def set_original_response(number="50"):
+def set_original_response(datastore_path, number="50"):
     test_return_data = f"""<html>
        <body>
      <h1>Test Page for Conditions</h1>
@@ -17,10 +18,10 @@ def set_original_response(number="50"):
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
-def set_number_in_range_response(number="75"):
+def set_number_in_range_response(datastore_path, number="75"):
     test_return_data = f"""<html>
        <body>
      <h1>Test Page for Conditions</h1>
@@ -30,10 +31,10 @@ def set_number_in_range_response(number="75"):
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
-def set_number_out_of_range_response(number="150"):
+def set_number_out_of_range_response(datastore_path, number="150"):
     test_return_data = f"""<html>
        <body>
      <h1>Test Page for Conditions</h1>
@@ -43,18 +44,18 @@ def set_number_out_of_range_response(number="150"):
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-# def test_setup(client, live_server, measure_memory_usage):
+# def test_setup(client, live_server, measure_memory_usage, datastore_path):
     """Test that both text and number conditions work together with AND logic."""
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def test_conditions_with_text_and_number(client, live_server, measure_memory_usage):
+def test_conditions_with_text_and_number(client, live_server, measure_memory_usage, datastore_path):
     """Test that both text and number conditions work together with AND logic."""
     
-    set_original_response("50")
+    set_original_response(datastore_path=datastore_path, number="50")
     
 
     test_url = url_for('test_endpoint', _external=True)
@@ -114,7 +115,7 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa
     wait_for_all_checks(client)
 
     # Case 1
-    set_number_in_range_response("70.5")
+    set_number_in_range_response(datastore_path=datastore_path, number="70.5")
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -129,7 +130,7 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa
     client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
     time.sleep(0.2)
 
-    set_number_out_of_range_response("150.5")
+    set_number_out_of_range_response(datastore_path=datastore_path, number="150.5")
 
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -142,9 +143,9 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa
     delete_all_watches(client)
 
 # The 'validate' button next to each rule row
-def test_condition_validate_rule_row(client, live_server, measure_memory_usage):
+def test_condition_validate_rule_row(client, live_server, measure_memory_usage, datastore_path):
 
-    set_original_response("50")
+    set_original_response(datastore_path=datastore_path, number="50")
 
     test_url = url_for('test_endpoint', _external=True)
 
@@ -203,7 +204,7 @@ def test_condition_validate_rule_row(client, live_server, measure_memory_usage):
 
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage):
+def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage, datastore_path):
     
 
     test_return_data = """<html>
@@ -216,7 +217,7 @@ def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage):
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
     # Add our URL to the import page
@@ -242,10 +243,10 @@ def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage):
     )
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
-    
+def test_lev_conditions_plugin(client, live_server, measure_memory_usage, datastore_path):
+    # This should break..
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""<html>
        <body>
      Some initial text<br>
@@ -297,7 +298,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
 
 
     ############### Now change it a LITTLE bit...
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""<html>
        <body>
      Some initial text<br>
@@ -326,7 +327,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     assert b'Queued 1 watch for rechecking.' in res.data

changedetectionio/tests/test_css_selector.py

@@ -3,12 +3,13 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
+import os
 
 from ..html_tools import *
 
 
 
-def set_original_response():
+def set_original_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -21,11 +22,11 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-def set_modified_response():
+def set_modified_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -38,7 +39,7 @@ def set_modified_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
     return None
@@ -69,12 +70,12 @@ def test_include_filters_output():
 
 
 # Tests the whole stack works with the CSS Filter
-def test_check_markup_include_filters_restriction(client, live_server, measure_memory_usage):
+def test_check_markup_include_filters_restriction(client, live_server, measure_memory_usage, datastore_path):
     sleep_time_for_fetch_thread = 3
 
     include_filters = "#sametext"
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -105,7 +106,7 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
     #  Make a change
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -119,11 +120,11 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
 
 
 # Tests the whole stack works with the CSS Filter
-def test_check_multiple_filters(client, live_server, measure_memory_usage):
+def test_check_multiple_filters(client, live_server, measure_memory_usage, datastore_path):
     
     include_filters = "#blob-a\r\nxpath://*[contains(@id,'blob-b')]"
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""<html><body>
      <div id="blob-a">Blob A</div>
      <div id="blob-b">Blob B</div>
@@ -168,12 +169,12 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage):
 # The filter exists, but did not contain anything useful
 # Mainly used when the filter contains just an IMG, this can happen when someone selects an image in the visual-selector
 # Tests fetcher can throw a "ReplyWithContentButNoText" exception after applying filter and extracting text
-def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usage):
+def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usage, datastore_path):
     
 
     include_filters = "#blob-a"
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""<html><body>
          <div id="blob-a">
            <img src="something.jpg">
@@ -216,7 +217,7 @@ def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usa
 
     ### Just an empty selector, no image
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("""<html><body>
          <div id="blob-a">
            <!-- doo doo -->

changedetectionio/tests/test_datastore_isolation.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+"""Test to verify client and live_server share the same datastore"""
+
+def test_client_and_live_server_share_datastore(client, live_server):
+    """Verify that client and live_server use the same app and datastore."""
+
+    # They should be the SAME object
+    assert client.application is live_server.app, "client.application and live_server.app should be the SAME object!"
+
+    # They should share the same datastore
+    client_datastore = client.application.config.get('DATASTORE')
+    server_datastore = live_server.app.config.get('DATASTORE')
+
+    assert client_datastore is server_datastore, \
+        f"Datastores are DIFFERENT objects! client={hex(id(client_datastore))} server={hex(id(server_datastore))}"
+
+    print(f"✓ client.application and live_server.app are the SAME object")
+    print(f"✓ Both use the same DATASTORE at {hex(id(client_datastore))}")
+    print(f"✓ Datastore path: {client_datastore.datastore_path}")

changedetectionio/tests/test_element_removal.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import time
+import os
 
 from flask import url_for
 
@@ -10,7 +11,7 @@ from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 
 
 
-def set_response_with_multiple_index():
+def set_response_with_multiple_index(datastore_path):
     data= """<!DOCTYPE html>
 <html>
 <body>
@@ -36,11 +37,11 @@ def set_response_with_multiple_index():
 </body>
 </html>
 """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(data)
 
 
-def set_original_response():
+def set_original_response(datastore_path):
     test_return_data = """<html>
     <header>
     <h2>Header</h2>
@@ -65,11 +66,11 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-def set_modified_response():
+def set_modified_response(datastore_path):
     test_return_data = """<html>
     <header>
     <h2>Header changed</h2>
@@ -94,7 +95,7 @@ def set_modified_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
@@ -146,10 +147,10 @@ across multiple lines
     )
 
 
-def test_element_removal_full(client, live_server, measure_memory_usage):
+def test_element_removal_full(client, live_server, measure_memory_usage, datastore_path):
     
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
 
     # Add our URL to the import page
@@ -194,7 +195,7 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
     client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
 
     #  Make a change to header/footer/nav
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     # Trigger a check
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -208,9 +209,9 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
     assert b"unviewed" not in res.data
 
 # Re #2752
-def test_element_removal_nth_offset_no_shift(client, live_server, measure_memory_usage):
+def test_element_removal_nth_offset_no_shift(client, live_server, measure_memory_usage, datastore_path):
 
-    set_response_with_multiple_index()
+    set_response_with_multiple_index(datastore_path=datastore_path)
     subtractive_selectors_data = [
 ### css style ###
 """body > table > tr:nth-child(1) > th:nth-child(2)

changedetectionio/tests/test_encoding.py

@@ -5,26 +5,27 @@ import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 import pytest
+import os
 
 
 
 
 
-def set_html_response():
+def set_html_response(datastore_path):
     test_return_data = """
 <html><body><span class="nav_second_img_text">
                   &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;铸大国重器，挺制造脊梁，致力能源未来，赋能美好生活。
                                   </span>
 </body></html>
     """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
 
 # In the case the server does not issue a charset= or doesnt have content_type header set
-def test_check_encoding_detection(client, live_server, measure_memory_usage):
-    set_html_response()
+def test_check_encoding_detection(client, live_server, measure_memory_usage, datastore_path):
+    set_html_response(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', content_type="text/html", _external=True)
@@ -51,8 +52,8 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage):
 
 
 # In the case the server does not issue a charset= or doesnt have content_type header set
-def test_check_encoding_detection_missing_content_type_header(client, live_server, measure_memory_usage):
-    set_html_response()
+def test_check_encoding_detection_missing_content_type_header(client, live_server, measure_memory_usage, datastore_path):
+    set_html_response(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_errorhandling.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import time
+import os
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
@@ -8,9 +9,9 @@ from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 
 
 
-def _runner_test_http_errors(client, live_server, http_code, expected_text):
+def _runner_test_http_errors(client, live_server, http_code, expected_text, datastore_path):
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("Now you going to get a {} error code\n".format(http_code))
 
 
@@ -46,17 +47,15 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text):
     delete_all_watches(client)
 
 
-def test_http_error_handler(client, live_server, measure_memory_usage):
-    _runner_test_http_errors(client, live_server, 403, 'Access denied')
-    _runner_test_http_errors(client, live_server, 404, 'Page not found')
-    _runner_test_http_errors(client, live_server, 500, '(Internal server error) received')
-    _runner_test_http_errors(client, live_server, 400, 'Error - Request returned a HTTP error code 400')
+def test_http_error_handler(client, live_server, measure_memory_usage, datastore_path):
+    _runner_test_http_errors(client, live_server, 403, 'Access denied', datastore_path=datastore_path)
+    _runner_test_http_errors(client, live_server, 404, 'Page not found', datastore_path=datastore_path)
+    _runner_test_http_errors(client, live_server, 500, '(Internal server error) received', datastore_path=datastore_path)
+    _runner_test_http_errors(client, live_server, 400, 'Error - Request returned a HTTP error code 400', datastore_path=datastore_path)
     delete_all_watches(client)
 
 # Just to be sure error text is properly handled
-def test_DNS_errors(client, live_server, measure_memory_usage):
-    # Give the endpoint time to spin up
-    time.sleep(1)
+def test_DNS_errors(client, live_server, measure_memory_usage, datastore_path):
 
     # Add our URL to the import page
     res = client.post(
@@ -84,12 +83,9 @@ def test_DNS_errors(client, live_server, measure_memory_usage):
     delete_all_watches(client)
 
 # Re 1513
-def test_low_level_errors_clear_correctly(client, live_server, measure_memory_usage):
-    
-    # Give the endpoint time to spin up
-    time.sleep(1)
+def test_low_level_errors_clear_correctly(client, live_server, measure_memory_usage, datastore_path):
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("<html><body><div id=here>Hello world</div></body></html>")
 
     # Add our URL to the import page

changedetectionio/tests/test_extract_csv.py

@@ -4,14 +4,15 @@ import time
 from flask import url_for
 from urllib.request import urlopen
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+import os
 
 sleep_time_for_fetch_thread = 3
 
 
 
-def test_check_extract_text_from_diff(client, live_server, measure_memory_usage):
+def test_check_extract_text_from_diff(client, live_server, measure_memory_usage, datastore_path):
     import time
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("Now it's {} seconds since epoch, time flies!".format(str(time.time())))
 
    #  live_server_setup(live_server) # Setup on conftest per function
@@ -33,7 +34,7 @@ def test_check_extract_text_from_diff(client, live_server, measure_memory_usage)
         # Give the thread time to pick it up
         print("Bumping snapshot and checking.. ", n)
         last_date = str(time.time())
-        with open("test-datastore/endpoint-content.txt", "w") as f:
+        with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
             f.write("Now it's {} seconds since epoch, time flies!".format(last_date))
 
         client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_extract_regex.py

@@ -3,11 +3,12 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
+import os
 
 from ..html_tools import *
 
 
-def set_original_response():
+def set_original_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -20,12 +21,12 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
 
-def set_modified_response():
+def set_modified_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -39,13 +40,13 @@ def set_modified_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
     return None
 
 
-def set_multiline_response():
+def set_multiline_response(datastore_path):
     test_return_data = """<html>
        <body>
      
@@ -61,18 +62,18 @@ def set_multiline_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
     return None
 
 
-# def test_setup(client, live_server, measure_memory_usage):
+# def test_setup(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def test_check_filter_multiline(client, live_server, measure_memory_usage):
+def test_check_filter_multiline(client, live_server, measure_memory_usage, datastore_path):
    ##  live_server_setup(live_server) # Setup on conftest per function
-    set_multiline_response()
+    set_multiline_response(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -119,11 +120,11 @@ def test_check_filter_multiline(client, live_server, measure_memory_usage):
     # but the last one, which also says 'lines' shouldnt be here (non-greedy match checking)
     assert b'aaand something lines' not in res.data
 
-def test_check_filter_and_regex_extract(client, live_server, measure_memory_usage):
+def test_check_filter_and_regex_extract(client, live_server, measure_memory_usage, datastore_path):
     
     include_filters = ".changetext"
 
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -159,7 +160,7 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
     assert b'not at the start of the expression' not in res.data
 
     #  Make a change
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -198,7 +199,7 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
 
 
 
-def test_regex_error_handling(client, live_server, measure_memory_usage):
+def test_regex_error_handling(client, live_server, measure_memory_usage, datastore_path):
 
     
 

changedetectionio/tests/test_filter_exist_changes.py

@@ -8,7 +8,7 @@ from .util import set_original_response, live_server_setup, wait_for_notificatio
 from changedetectionio.model import App
 
 
-def set_response_without_filter():
+def set_response_without_filter(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -20,12 +20,12 @@ def set_response_without_filter():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
 
-def set_response_with_filter():
+def set_response_with_filter(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -37,11 +37,11 @@ def set_response_with_filter():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_server, measure_memory_usage):
+def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_server, measure_memory_usage, datastore_path):
 #  Filter knowingly doesn't exist, like someone setting up a known filter to see if some cinema tickets are on sale again
 #  And the page has that filter available
 #  Then I should get a notification
@@ -50,7 +50,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     # Give the endpoint time to spin up
     time.sleep(1)
-    set_response_without_filter()
+    set_response_without_filter(datastore_path=datastore_path)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -105,20 +105,20 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    wait_for_notification_endpoint_output()
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
 
     # Shouldn't exist, shouldn't have fired
-    assert not os.path.isfile("test-datastore/notification.txt")
+    assert not os.path.isfile(os.path.join(datastore_path, "notification.txt"))
     # Now the filter should exist
-    set_response_with_filter()
+    set_response_with_filter(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
-    wait_for_notification_endpoint_output()
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
 
-    assert os.path.isfile("test-datastore/notification.txt")
+    assert os.path.isfile(os.path.join(datastore_path, "notification.txt"))
 
-    with open("test-datastore/notification.txt", 'r') as f:
+    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
         notification = f.read()
 
     assert 'Ticket now on sale' in notification
-    os.unlink("test-datastore/notification.txt")
+    os.unlink(os.path.join(datastore_path, "notification.txt"))

changedetectionio/tests/test_filter_failure_notification.py

@@ -5,7 +5,7 @@ from .util import set_original_response,  wait_for_all_checks, wait_for_notifica
 from ..notification import valid_notification_formats
 
 
-def set_response_with_filter():
+def set_response_with_filter(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -17,14 +17,14 @@ def set_response_with_filter():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-def run_filter_test(client, live_server, content_filter, app_notification_format):
+def run_filter_test(client, live_server, content_filter, app_notification_format, datastore_path):
 
     # Response WITHOUT the filter ID element
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
     live_server.app.config['DATASTORE'].data['settings']['application']['notification_format'] = app_notification_format
 
     # Goto the edit page, add our ignore text
@@ -38,10 +38,16 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
         url_for("ui.form_delete", uuid="all"),
         follow_redirects=True
     )
-    if os.path.isfile("test-datastore/notification.txt"):
-        os.unlink("test-datastore/notification.txt")
+    notification_file = os.path.join(datastore_path, "notification.txt")
+    if os.path.isfile(notification_file):
+        os.unlink(notification_file)
 
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
+    res = client.get(url_for("watchlist.index"))
+
+    assert b'No website watches configured' not in res.data
+
+
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -79,6 +85,7 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
         data=watch_data,
         follow_redirects=True
     )
+
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 0, "No filter = No filter failure"
@@ -95,7 +102,7 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
     # It should have checked once so far and given this error (because we hit SAVE)
 
     wait_for_all_checks(client)
-    assert not os.path.isfile("test-datastore/notification.txt")
+    assert not os.path.isfile(notification_file)
 
     # Hitting [save] would have triggered a recheck, and we have a filter, so this would be ONE failure
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 1, "Should have been checked once"
@@ -110,20 +117,20 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
         wait_for_all_checks(client)
         res = client.get(url_for("watchlist.index"))
         assert b'Warning, no filters were found' in res.data
-        assert not os.path.isfile("test-datastore/notification.txt")
+        assert not os.path.isfile(notification_file)
         time.sleep(1)
-        
+
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 5
 
     time.sleep(2)
     # One more check should trigger the _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT threshold
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    wait_for_notification_endpoint_output()
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
 
     # Now it should exist and contain our "filter not found" alert
-    assert os.path.isfile("test-datastore/notification.txt")
-    with open("test-datastore/notification.txt", 'r') as f:
+    assert os.path.isfile(notification_file)
+    with open(notification_file, 'r') as f:
         notification = f.read()
 
     assert 'Your configured CSS/xPath filters' in notification
@@ -146,19 +153,19 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
 
     # Remove it and prove that it doesn't trigger when not expected
     # It should register a change, but no 'filter not found'
-    os.unlink("test-datastore/notification.txt")
-    set_response_with_filter()
+    os.unlink(notification_file)
+    set_response_with_filter(datastore_path)
 
     # Try several times, it should NOT have 'filter not found'
     for i in range(0, ATTEMPT_THRESHOLD_SETTING + 2):
         client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
         wait_for_all_checks(client)
 
-    wait_for_notification_endpoint_output()
+    wait_for_notification_endpoint_output(datastore_path=datastore_path)
     # It should have sent a notification, but..
-    assert os.path.isfile("test-datastore/notification.txt")
+    assert os.path.isfile(notification_file)
     # but it should not contain the info about a failed filter (because there was none in this case)
-    with open("test-datastore/notification.txt", 'r') as f:
+    with open(notification_file, 'r') as f:
         notification = f.read()
     assert not 'CSS/xPath filter was not present in the page' in notification
 
@@ -170,22 +177,22 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
         url_for("ui.form_delete", uuid="all"),
         follow_redirects=True
     )
-    os.unlink("test-datastore/notification.txt")
+    os.unlink(notification_file)
 
 
-def test_check_include_filters_failure_notification(client, live_server, measure_memory_usage):
+def test_check_include_filters_failure_notification(client, live_server, measure_memory_usage, datastore_path):
     #   #  live_server_setup(live_server) # Setup on conftest per function
-    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('htmlcolor'))
+    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('htmlcolor'), datastore_path=datastore_path)
     # Check markup send conversion didnt affect plaintext preference
-    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('text'))
+    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('text'), datastore_path=datastore_path)
 
-def test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage):
+def test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage, datastore_path):
     #   #  live_server_setup(live_server) # Setup on conftest per function
-    run_filter_test(client=client, live_server=live_server, content_filter='//*[@id="nope-doesnt-exist"]', app_notification_format=valid_notification_formats.get('htmlcolor'))
+    run_filter_test(client=client, live_server=live_server, content_filter='//*[@id="nope-doesnt-exist"]', app_notification_format=valid_notification_formats.get('htmlcolor'), datastore_path=datastore_path)
 
 # Test that notification is never sent
 
-def test_basic_markup_from_text(client, live_server, measure_memory_usage):
+def test_basic_markup_from_text(client, live_server, measure_memory_usage, datastore_path):
     # Test the notification error templates convert to HTML if needed (link activate)
     from ..notification.handler import markup_text_links_to_html
     x = markup_text_links_to_html("hello https://google.com")

changedetectionio/tests/test_group.py

@@ -6,10 +6,10 @@ from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from
 import os
 
 
-# def test_setup(client, live_server, measure_memory_usage):
+# def test_setup(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def set_original_response():
+def set_original_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -20,11 +20,11 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-def set_modified_response():
+def set_modified_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -35,13 +35,13 @@ def set_modified_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
     return None
 
-def test_setup_group_tag(client, live_server, measure_memory_usage):
+def test_setup_group_tag(client, live_server, measure_memory_usage, datastore_path):
     
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
     # Add a tag with some config, import a tag and it should roughly work
     res = client.post(
@@ -116,7 +116,7 @@ def test_setup_group_tag(client, live_server, measure_memory_usage):
     )
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     rss_token = extract_rss_token_from_UI(client)
@@ -129,7 +129,7 @@ def test_setup_group_tag(client, live_server, measure_memory_usage):
     assert b"first-imported=1" in res.data
     delete_all_watches(client)
 
-def test_tag_import_singular(client, live_server, measure_memory_usage):
+def test_tag_import_singular(client, live_server, measure_memory_usage, datastore_path):
     
 
     test_url = url_for('test_endpoint', _external=True)
@@ -148,7 +148,7 @@ def test_tag_import_singular(client, live_server, measure_memory_usage):
     assert res.data.count(b'test-tag') == 1
     delete_all_watches(client)
 
-def test_tag_add_in_ui(client, live_server, measure_memory_usage):
+def test_tag_add_in_ui(client, live_server, measure_memory_usage, datastore_path):
     
 #
     res = client.post(
@@ -164,9 +164,9 @@ def test_tag_add_in_ui(client, live_server, measure_memory_usage):
 
     delete_all_watches(client)
 
-def test_group_tag_notification(client, live_server, measure_memory_usage):
+def test_group_tag_notification(client, live_server, measure_memory_usage, datastore_path):
     
-    set_original_response()
+    set_original_response(datastore_path=datastore_path)
 
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -207,16 +207,16 @@ def test_group_tag_notification(client, live_server, measure_memory_usage):
 
     wait_for_all_checks(client)
 
-    set_modified_response()
+    set_modified_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     time.sleep(3)
 
-    assert os.path.isfile("test-datastore/notification.txt")
+    assert os.path.isfile(os.path.join(datastore_path, "notification.txt"))
 
     # Verify what was sent as a notification, this file should exist
-    with open("test-datastore/notification.txt", "r") as f:
+    with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
         notification_submission = f.read()
-    os.unlink("test-datastore/notification.txt")
+    os.unlink(os.path.join(datastore_path, "notification.txt"))
 
     # Did we see the URL that had a change, in the notification?
     # Diff was correctly executed
@@ -231,7 +231,7 @@ def test_group_tag_notification(client, live_server, measure_memory_usage):
     #@todo Test that each of multiple notifications with different settings
     delete_all_watches(client)
 
-def test_limit_tag_ui(client, live_server, measure_memory_usage):
+def test_limit_tag_ui(client, live_server, measure_memory_usage, datastore_path):
 
     test_url = url_for('test_random_content_endpoint', _external=True)
 
@@ -269,7 +269,7 @@ def test_limit_tag_ui(client, live_server, measure_memory_usage):
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
     assert b'All tags deleted' in res.data
 
-def test_clone_tag_on_import(client, live_server, measure_memory_usage):
+def test_clone_tag_on_import(client, live_server, measure_memory_usage, datastore_path):
     
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -294,7 +294,7 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     assert res.data.count(b'another-tag') == 3
     delete_all_watches(client)
 
-def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usage):
+def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usage, datastore_path):
     
 
     test_url = url_for('test_endpoint', _external=True)
@@ -324,7 +324,7 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
     assert b'All tags deleted' in res.data
 
-def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measure_memory_usage):
+def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measure_memory_usage, datastore_path):
 
     # Add a tag with some config, import a tag and it should roughly work
     res = client.post(
@@ -378,7 +378,7 @@ def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measu
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(d)
 
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_history_consistency.py

@@ -7,7 +7,7 @@ from flask import url_for
 from .util import wait_for_all_checks, delete_all_watches
 from urllib.parse import urlparse, parse_qs
 
-def test_consistent_history(client, live_server, measure_memory_usage):
+def test_consistent_history(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
     workers = int(os.getenv("FETCH_WORKERS", 10))
     r = range(1, 10+workers)
@@ -40,7 +40,7 @@ def test_consistent_history(client, live_server, measure_memory_usage):
     json_db_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, 'url-watches.json')
 
     json_obj = None
-    with open(json_db_file, 'r') as f:
+    with open(json_db_file, 'r', encoding='utf-8') as f:
         json_obj = json.load(f)
 
     # assert the right amount of watches was found in the JSON
@@ -76,13 +76,13 @@ def test_consistent_history(client, live_server, measure_memory_usage):
         assert len(files_in_watch_dir) == 3, "Should be just three files in the dir, html.br snapshot, history.txt and the extracted text snapshot"
 
     json_db_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, 'url-watches.json')
-    with open(json_db_file, 'r') as f:
+    with open(json_db_file, 'r', encoding='utf-8') as f:
         assert '"default"' not in f.read(), "'default' probably shouldnt be here, it came from when the 'default' Watch vars were accidently being saved"
 
 
-def test_check_text_history_view(client, live_server, measure_memory_usage):
+def test_check_text_history_view(client, live_server, measure_memory_usage, datastore_path):
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("<html>test-one</html>")
 
     # Add our URL to the import page
@@ -94,7 +94,7 @@ def test_check_text_history_view(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     # Set second version, Make a change
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("<html>test-two</html>")
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -105,7 +105,7 @@ def test_check_text_history_view(client, live_server, measure_memory_usage):
     assert b'test-two' in res.data
 
     # Set third version, Make a change
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write("<html>test-three</html>")
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_ignore.py

@@ -5,8 +5,9 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 from changedetectionio import html_tools
 from . util import  extract_UUID_from_client
+import os
 
-def set_original_ignore_response():
+def set_original_ignore_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -19,13 +20,13 @@ def set_original_ignore_response():
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-def test_ignore(client, live_server, measure_memory_usage):
+def test_ignore(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path)
     test_url = url_for('test_endpoint', _external=True)
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -55,9 +56,9 @@ def test_ignore(client, live_server, measure_memory_usage):
     assert b'csrftoken' in res.data
 
 
-def test_strip_ignore_lines(client, live_server, measure_memory_usage):
+def test_strip_ignore_lines(client, live_server, measure_memory_usage, datastore_path):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path)
 
 
     # Goto the settings page, add our ignore text

changedetectionio/tests/test_ignore_text.py

@@ -4,6 +4,7 @@ import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 from changedetectionio import html_tools
+import os
 
 
 
@@ -31,7 +32,7 @@ def test_strip_text_func():
     stripped_content = html_tools.strip_ignore_text(test_content, ignore)
     assert stripped_content == "Some initial text\n\nWhich is across multiple lines\n\n\n\nSo let's see what happens."
 
-def set_original_ignore_response(ver_stamp="123"):
+def set_original_ignore_response(datastore_path, ver_stamp="123"):
     test_return_data = f"""<html>
        <body>
      Some initial text<br>
@@ -44,11 +45,11 @@ def set_original_ignore_response(ver_stamp="123"):
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-def set_modified_original_ignore_response(ver_stamp="123"):
+def set_modified_original_ignore_response(datastore_path, ver_stamp="123"):
     test_return_data = f"""<html>
        <body>
      Some NEW nice initial text<br>
@@ -63,12 +64,12 @@ def set_modified_original_ignore_response(ver_stamp="123"):
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
 # Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
-def set_modified_ignore_response(ver_stamp="123"):
+def set_modified_ignore_response(datastore_path, ver_stamp="123"):
     test_return_data = f"""<html>
        <body>
      Some initial text<br>
@@ -82,17 +83,17 @@ def set_modified_ignore_response(ver_stamp="123"):
 
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
 # Ignore text now just removes it entirely, is a LOT more simpler code this way
 
-def test_check_ignore_text_functionality(client, live_server, measure_memory_usage):
+def test_check_ignore_text_functionality(client, live_server, measure_memory_usage, datastore_path):
 
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path=datastore_path)
 
 
     # Add our URL to the import page
@@ -130,7 +131,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     assert b'/test-endpoint' in res.data
 
     #  Make a change
-    set_modified_ignore_response()
+    set_modified_ignore_response(datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -145,7 +146,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
 
 
     # Just to be sure.. set a regular modified change..
-    set_modified_original_ignore_response()
+    set_modified_original_ignore_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -162,10 +163,10 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     delete_all_watches(client)
 
 # When adding some ignore text, it should not trigger a change, even if something else on that line changes
-def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
+def _run_test_global_ignore(client, datastore_path, as_source=False, extra_ignore=""):
     ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ\r\n"+extra_ignore
 
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path=datastore_path)
 
     # Goto the settings page, add our ignore text
     res = client.post(
@@ -222,7 +223,7 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
     # Make a change which includes the ignore text, it should be ignored and no 'change' triggered
     # It adds text with "ZZZZzzzz" and "ZZZZ" is in the ignore list
     # And tweaks the ver_stamp which should be picked up by global regex ignore
-    set_modified_ignore_response(ver_stamp=time.time())
+    set_modified_ignore_response(ver_stamp=time.time(), datastore_path=datastore_path)
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -236,7 +237,7 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
     assert b'/test-endpoint' in res.data
 
     # Just to be sure.. set a regular modified change that will trigger it
-    set_modified_original_ignore_response()
+    set_modified_original_ignore_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -244,10 +245,10 @@ def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
 
     delete_all_watches(client)
 
-def test_check_global_ignore_text_functionality(client, live_server, measure_memory_usage):
+def test_check_global_ignore_text_functionality(client, live_server, measure_memory_usage, datastore_path):
     
-    _run_test_global_ignore(client, as_source=False)
+    _run_test_global_ignore(client, as_source=False, datastore_path=datastore_path)
 
-def test_check_global_ignore_text_functionality_as_source(client, live_server, measure_memory_usage):
+def test_check_global_ignore_text_functionality_as_source(client, live_server, measure_memory_usage, datastore_path):
     
-    _run_test_global_ignore(client, as_source=True, extra_ignore='/\?v=\d/')
+    _run_test_global_ignore(client, as_source=True, extra_ignore='/\?v=\d/', datastore_path=datastore_path)

changedetectionio/tests/test_ignorehyperlinks.py

@@ -4,9 +4,10 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
+import os
 
 
-def set_original_ignore_response():
+def set_original_ignore_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -17,13 +18,13 @@ def set_original_ignore_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
 
-# Should be the same as set_original_ignore_response() but with a different
+# Should be the same as set_original_ignore_response(datastore_path=datastore_path) but with a different
 # link
-def set_modified_ignore_response():
+def set_modified_ignore_response(datastore_path):
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -34,10 +35,10 @@ def set_modified_ignore_response():
      </html>
     """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
+    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
         f.write(test_return_data)
 
-def test_render_anchor_tag_content_true(client, live_server, measure_memory_usage):
+def test_render_anchor_tag_content_true(client, live_server, measure_memory_usage, datastore_path):
     """Testing that the link changes are detected when
     render_anchor_tag_content setting is set to true"""
     sleep_time_for_fetch_thread = 3
@@ -46,7 +47,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
     time.sleep(1)
 
     # set original html text
-    set_original_ignore_response()
+    set_original_ignore_response(datastore_path=datastore_path)
 
     # Goto the settings page, choose to ignore links (dont select/send "application-render_anchor_tag_content")
     res = client.post(
@@ -72,7 +73,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
     # set a new html text with a modified link
-    set_modified_ignore_response()
+    set_modified_ignore_response(datastore_path=datastore_path)
     wait_for_all_checks(client)
 
     # Trigger a check