Add safety timeouts

.github/actions/extract-memory-report/action.yml

@@ -23,29 +23,26 @@ runs:
     - name: Dump container log
       shell: bash
       run: |
-        echo "Disabled for now"
-#        return
-#        docker logs ${{ inputs.container-name }} > ${{ inputs.output-dir }}/${{ inputs.container-name }}-stdout-${{ inputs.python-version }}.txt 2>&1 || echo "Could not get stdout"
-#        docker logs ${{ inputs.container-name }} 2> ${{ inputs.output-dir }}/${{ inputs.container-name }}-stderr-${{ inputs.python-version }}.txt || echo "Could not get stderr"
+        docker logs ${{ inputs.container-name }} > ${{ inputs.output-dir }}/${{ inputs.container-name }}-stdout-${{ inputs.python-version }}.txt 2>&1 || echo "Could not get stdout"
+        docker logs ${{ inputs.container-name }} 2> ${{ inputs.output-dir }}/${{ inputs.container-name }}-stderr-${{ inputs.python-version }}.txt || echo "Could not get stderr"
 
     - name: Extract and display memory test report
       shell: bash
       run: |
-        echo "Disabled for now"
-#        echo "Extracting test-memory.log from container..."
-#        docker cp ${{ inputs.container-name }}:/app/changedetectionio/test-memory.log ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log || echo "test-memory.log not found in container"
-#
-#        echo "=== Top 10 Highest Peak Memory Tests ==="
-#        if [ -f ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log ]; then
-#          grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log | \
-#            sed 's/.*Peak memory: //' | \
-#            paste -d'|' - <(grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log) | \
-#            sort -t'|' -k1 -nr | \
-#            cut -d'|' -f2 | \
-#            head -10
-#          echo ""
-#          echo "=== Full Memory Test Report ==="
-#          cat ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log
-#        else
-#          echo "No memory log available"
-#        fi
+        echo "Extracting test-memory.log from container..."
+        docker cp ${{ inputs.container-name }}:/app/changedetectionio/test-memory.log ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log || echo "test-memory.log not found in container"
+
+        echo "=== Top 10 Highest Peak Memory Tests ==="
+        if [ -f ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log ]; then
+          grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log | \
+            sed 's/.*Peak memory: //' | \
+            paste -d'|' - <(grep "Peak memory:" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log) | \
+            sort -t'|' -k1 -nr | \
+            cut -d'|' -f2 | \
+            head -10
+          echo ""
+          echo "=== Full Memory Test Report ==="
+          cat ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log
+        else
+          echo "No memory log available"
+        fi

.github/workflows/containers.yml

@@ -45,14 +45,6 @@ jobs:
         with:
           python-version: 3.11
 
-      - name: Cache pip packages
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/test-container-build.yml

@@ -50,14 +50,6 @@ jobs:
           with:
             python-version: 3.11
 
-        - name: Cache pip packages
-          uses: actions/cache@v4
-          with:
-            path: ~/.cache/pip
-            key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
-            restore-keys: |
-              ${{ runner.os }}-pip-
-
         # Just test that the build works, some libraries won't compile on ARM/rPi etc
         - name: Set up QEMU
           uses: docker/setup-qemu-action@v3
@@ -82,5 +74,5 @@ jobs:
             file: ${{ matrix.dockerfile }}
             platforms: ${{ matrix.platform }}
             cache-from: type=gha
-            cache-to: type=gha,mode=max
+            cache-to: type=gha,mode=min
 

.github/workflows/test-only.yml

@@ -21,8 +21,6 @@ jobs:
           python3 -c "from openapi_spec_validator import validate_spec; import yaml; validate_spec(yaml.safe_load(open('docs/api-spec.yaml')))"
 
   test-application-3-10:
-    # Only run on push to master (including PR merges)
-    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
     needs: lint-code
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:
@@ -30,15 +28,12 @@ jobs:
 
 
   test-application-3-11:
-    # Always run
     needs: lint-code
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:
       python-version: '3.11'
 
   test-application-3-12:
-    # Only run on push to master (including PR merges)
-    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
     needs: lint-code
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:
@@ -46,8 +41,6 @@ jobs:
       skip-pypuppeteer: true
 
   test-application-3-13:
-    # Only run on push to master (including PR merges)
-    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
     needs: lint-code
     uses: ./.github/workflows/test-stack-reusable-workflow.yml
     with:

.github/workflows/test-stack-reusable-workflow.yml

@@ -28,15 +28,6 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: Cache pip packages
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}-
-            ${{ runner.os }}-pip-
-
       - name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}
         run: |
           echo "---- Building for Python ${{ env.PYTHON_VERSION }} -----"
@@ -69,7 +60,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v5
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -96,7 +87,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v5
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -135,7 +126,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v5
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -177,7 +168,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v5
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -217,7 +208,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v5
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -253,7 +244,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v5
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -282,7 +273,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v5
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -298,18 +289,15 @@ jobs:
           docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest
           docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
 
-      - name: Test proxy Squid style interaction
+      - name: Test proxy squid style interaction
         run: |
           cd changedetectionio
           ./run_proxy_tests.sh
-          docker ps
-          cd ..
 
       - name: Test proxy SOCKS5 style interaction
         run: |
           cd changedetectionio
           ./run_socks_proxy_tests.sh
-          cd ..
 
   # Custom browser URL tests
   custom-browser-tests:
@@ -322,7 +310,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v5
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -353,7 +341,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v5
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp
@@ -398,7 +386,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Download Docker image artifact
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v5
         with:
           name: test-changedetectionio-${{ env.PYTHON_VERSION }}
           path: /tmp

Dockerfile

@@ -34,27 +34,23 @@ ENV OPENSSL_LIB_DIR="/usr/lib/arm-linux-gnueabihf"
 ENV OPENSSL_INCLUDE_DIR="/usr/include/openssl"
 # Additional environment variables for cryptography Rust build
 ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
-RUN --mount=type=cache,id=pip,sharing=locked,target=/tmp/pip-cache \
-  pip install \
-  --prefer-binary \
-  --extra-index-url https://www.piwheels.org/simple \
-  --extra-index-url https://pypi.anaconda.org/ARM-software/simple \
-  --cache-dir=/tmp/pip-cache \
-  --target=/dependencies \
-  -r /requirements.txt
-
+RUN --mount=type=cache,target=/tmp/pip-cache \
+    pip install \
+    --extra-index-url https://www.piwheels.org/simple \
+    --extra-index-url https://pypi.anaconda.org/ARM-software/simple \
+    --cache-dir=/tmp/pip-cache \
+    --target=/dependencies \
+    -r /requirements.txt
 
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN --mount=type=cache,id=pip,sharing=locked,target=/tmp/pip-cache \
-  pip install \
-  --prefer-binary \
-  --cache-dir=/tmp/pip-cache \
-  --target=/dependencies \
-  playwright~=1.48.0 \
-  || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
-
+RUN --mount=type=cache,target=/tmp/pip-cache \
+    pip install \
+    --cache-dir=/tmp/pip-cache \
+    --target=/dependencies \
+    playwright~=1.48.0 \
+    || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
 FROM python:${PYTHON_VERSION}-slim-bookworm

changedetectionio/PLUGIN_README.md

@@ -64,7 +64,7 @@ def count_words_in_history(watch):
             return 0
             
         latest_key = list(watch.history.keys())[-1]
-        latest_content = watch.get_history_snapshot(timestamp=latest_key)
+        latest_content = watch.get_history_snapshot(latest_key)
         return len(latest_content.split())
     except Exception as e:
         logger.error(f"Error counting words: {str(e)}")

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.50.39'
+__version__ = '0.50.33'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/api/Import.py

@@ -1,9 +1,10 @@
+import os
 from changedetectionio.strtobool import strtobool
 from flask_restful import abort, Resource
 from flask import request
+import validators
 from functools import wraps
 from . import auth, validate_openapi_request
-from ..validate_url import is_safe_valid_url
 
 
 def default_content_type(content_type='text/plain'):
@@ -49,13 +50,14 @@ class Import(Resource):
 
         urls = request.get_data().decode('utf8').splitlines()
         added = []
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
         for url in urls:
             url = url.strip()
             if not len(url):
                 continue
 
             # If hosts that only contain alphanumerics are allowed ("localhost" for example)
-            if not is_safe_valid_url(url):
+            if not validators.url(url, simple_host=allow_simplehost):
                 return f"Invalid or unsupported URL - {url}", 400
 
             if dedupe and self.datastore.url_exists(url):

changedetectionio/api/Watch.py

@@ -1,12 +1,14 @@
 import os
 
-from changedetectionio.validate_url import is_safe_valid_url
+from changedetectionio.strtobool import strtobool
+from changedetectionio.html_tools import is_safe_url
 
 from flask_expects_json import expects_json
 from changedetectionio import queuedWatchMetaData
 from changedetectionio import worker_handler
 from flask_restful import abort, Resource
 from flask import request, make_response, send_from_directory
+import validators
 from . import auth
 import copy
 
@@ -122,7 +124,7 @@ class Watch(Resource):
             return validation_error, 400
 
         # XSS etc protection
-        if request.json.get('url') and not is_safe_valid_url(request.json.get('url')):
+        if request.json.get('url') and not is_safe_url(request.json.get('url')):
             return "Invalid URL", 400
 
         watch.update(request.json)
@@ -175,7 +177,7 @@ class WatchSingleHistory(Resource):
                 response = make_response("No content found", 404)
                 response.mimetype = "text/plain"
         else:
-            content = watch.get_history_snapshot(timestamp=timestamp)
+            content = watch.get_history_snapshot(timestamp)
             response = make_response(content, 200)
             response.mimetype = "text/plain"
 
@@ -230,7 +232,9 @@ class CreateWatch(Resource):
         json_data = request.get_json()
         url = json_data['url'].strip()
 
-        if not is_safe_valid_url(url):
+        # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+        if not validators.url(url, simple_host=allow_simplehost):
             return "Invalid or unsupported URL", 400
 
         if json_data.get('proxy'):

changedetectionio/api/api_schema.py

@@ -96,10 +96,7 @@ def build_watch_json_schema(d):
                                                            "enum": ["html_requests", "html_webdriver"]
                                                            })
 
-    schema['properties']['processor'] = {"anyOf": [
-        {"type": "string", "enum": ["restock_diff", "text_json_diff"]},
-        {"type": "null"}
-    ]}
+
 
     # All headers must be key/value type dict
     schema['properties']['headers'] = {

changedetectionio/blueprint/rss/blueprint.py

@@ -118,8 +118,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 fe.title(title=watch_label)
                 try:
 
-                    html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(timestamp=dates[-2]),
-                                                 newest_version_file_contents=watch.get_history_snapshot(timestamp=dates[-1]),
+                    html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]),
+                                                 newest_version_file_contents=watch.get_history_snapshot(dates[-1]),
                                                  include_equal=False,
                                                  line_feed_sep="<br>"
                                                  )

changedetectionio/blueprint/ui/notification.py

@@ -2,7 +2,7 @@ from flask import Blueprint, request, make_response
 import random
 from loguru import logger
 
-from changedetectionio.notification_service import NotificationContextData, set_basic_notification_vars
+from changedetectionio.notification_service import NotificationContextData
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
 
@@ -39,7 +39,11 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             return make_response("Error: You must have atleast one watch configured for 'test notification' to work", 400)
 
         watch = datastore.data['watching'].get(watch_uuid)
-        notification_urls = request.form.get('notification_urls','').strip().splitlines()
+
+        notification_urls = None
+
+        if request.form.get('notification_urls'):
+            notification_urls = request.form['notification_urls'].strip().splitlines()
 
         if not notification_urls:
             logger.debug("Test notification - Trying by group/tag in the edit form if available")
@@ -77,8 +81,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             # Only use if present, if not set in n_object it should use the default system value
             if 'notification_format' in request.form and request.form['notification_format'].strip():
                 n_object['notification_format'] = request.form.get('notification_format', '').strip()
-            else:
-                n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
 
             if 'notification_title' in request.form and request.form['notification_title'].strip():
                 n_object['notification_title'] = request.form.get('notification_title', '').strip()
@@ -95,44 +97,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 n_object['notification_body'] = "Test body"
 
             n_object['as_async'] = False
-
-            #  Same like in notification service, should be refactored
-            dates = []
-            trigger_text = ''
-            snapshot_contents = ''
-            if watch:
-                watch_history = watch.history
-                dates = list(watch_history.keys())
-                trigger_text = watch.get('trigger_text', [])
-                # Add text that was triggered
-                if len(dates):
-                    snapshot_contents = watch.get_history_snapshot(timestamp=dates[-1])
-                else:
-                    snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
-
-                if len(trigger_text):
-                    from . import html_tools
-                    triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
-                    if triggered_text:
-                        triggered_text = '\n'.join(triggered_text)
-
-            # Could be called as a 'test notification' with only 1 snapshot available
-            prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
-            current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
-
-
-
-            if len(dates) > 1:
-                prev_snapshot = watch.get_history_snapshot(timestamp=dates[-2])
-                current_snapshot = watch.get_history_snapshot(timestamp=dates[-1])
-
-            n_object.update(set_basic_notification_vars(snapshot_contents=snapshot_contents,
-                                                        current_snapshot=current_snapshot,
-                                                        prev_snapshot=prev_snapshot,
-                                                        watch=watch,
-                                                        triggered_text=trigger_text))
-
-
+            n_object.update(watch.extra_notification_token_values())
             sent_obj = process_notification(n_object, datastore)
 
         except Exception as e:

changedetectionio/blueprint/ui/views.py

@@ -47,7 +47,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
 
             try:
                 versions = list(watch.history.keys())
-                content = watch.get_history_snapshot(timestamp=timestamp)
+                content = watch.get_history_snapshot(timestamp)
 
                 triggered_line_numbers = html_tools.strip_ignore_text(content=content,
                                                                       wordlist=watch['trigger_text'],

changedetectionio/conditions/plugins/wordcount_plugin.py

@@ -14,7 +14,7 @@ def count_words_in_history(watch, incoming_text=None):
         elif watch.history.keys():
             # When called from UI extras to count latest snapshot
             latest_key = list(watch.history.keys())[-1]
-            latest_content = watch.get_history_snapshot(timestamp=latest_key)
+            latest_content = watch.get_history_snapshot(latest_key)
             return len(latest_content.split())
         return 0
     except Exception as e:

changedetectionio/flask_app.py

@@ -133,10 +133,10 @@ def get_socketio_path():
     # Socket.IO will be available at {prefix}/socket.io/
     return prefix
 
-@app.template_global('is_safe_valid_url')
-def _is_safe_valid_url(test_url):
-    from .validate_url import is_safe_valid_url
-    return is_safe_valid_url(test_url)
+@app.template_global('is_safe_url')
+def _is_safe_url(test_url):
+    from .html_tools import is_safe_url
+    return is_safe_url(test_url)
 
 
 @app.template_filter('format_number_locale')
@@ -387,7 +387,7 @@ def changedetection_app(config=None, datastore_o=None):
             # We would sometimes get login loop errors on sites hosted in sub-paths
 
             # note for the future:
-            #            if not is_safe_valid_url(next):
+            #            if not is_safe_url(next):
             #                return flask.abort(400)
             return redirect(url_for('watchlist.index'))
 
@@ -794,19 +794,15 @@ def ticker_thread_check_time_launch_checks():
 
             # @todo - Maybe make this a hook?
             # Time schedule limit - Decide between watch or global settings
-            scheduler_source = None
             if watch.get('time_between_check_use_default'):
                 time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})
-                scheduler_source = 'system/global settings'
-
+                logger.trace(f"{uuid} Time scheduler - Using system/global settings")
             else:
                 time_schedule_limit = watch.get('time_schedule_limit')
-                scheduler_source = 'watch'
-
+                logger.trace(f"{uuid} Time scheduler - Using watch settings (not global settings)")
             tz_name = datastore.data['settings']['application'].get('scheduler_timezone_default', os.getenv('TZ', 'UTC').strip())
 
             if time_schedule_limit and time_schedule_limit.get('enabled'):
-                logger.trace(f"{uuid} Time scheduler - Using scheduler settings from {scheduler_source}")
                 try:
                     result = is_within_schedule(time_schedule_limit=time_schedule_limit,
                                                 default_tz=tz_name
@@ -818,7 +814,6 @@ def ticker_thread_check_time_launch_checks():
                     logger.error(
                         f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}")
                     return False
-
             # If they supplied an individual entry minutes to threshold.
             threshold = recheck_time_system_seconds if watch.get('time_between_check_use_default') else watch.threshold_seconds()
 

changedetectionio/forms.py

@@ -28,8 +28,11 @@ from wtforms.utils import unset_value
 
 from wtforms.validators import ValidationError
 
+from validators.url import url as url_validator
+
 from changedetectionio.widgets import TernaryNoneBooleanField
 
+
 # default
 # each select <option data-enabled="enabled-0-0"
 from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
@@ -538,10 +541,19 @@ class validateURL(object):
 
 
 def validate_url(test_url):
-    from changedetectionio.validate_url import is_safe_valid_url
-    if not is_safe_valid_url(test_url):
+    # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+    try:
+        url_validator(test_url, simple_host=allow_simplehost)
+    except validators.ValidationError:
+        #@todo check for xss
+        message = f"'{test_url}' is not a valid URL."
         # This should be wtforms.validators.
-        raise ValidationError('Watch protocol is not permitted or invalid URL format')
+        raise ValidationError(message)
+
+    from changedetectionio.html_tools import is_safe_url
+    if not is_safe_url(test_url):
+        # This should be wtforms.validators.
+        raise ValidationError('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX or incorrect URL format')
 
 
 class ValidateSinglePythonRegexString(object):

changedetectionio/html_tools.py

@@ -15,6 +15,8 @@ TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.I | re.S)
 META_CS  = re.compile(r'<meta[^>]+charset=["\']?\s*([a-z0-9_\-:+.]+)', re.I)
 META_CT  = re.compile(r'<meta[^>]+http-equiv=["\']?content-type["\']?[^>]*content=["\'][^>]*charset=([a-z0-9_\-:+.]+)', re.I)
 
+SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):'
+
 # 'price' , 'lowPrice', 'highPrice' are usually under here
 # All of those may or may not appear on different websites - I didnt find a way todo case-insensitive searching here
 LD_JSON_PRODUCT_OFFER_SELECTORS = ["json:$..offers", "json:$..Offers"]
@@ -23,6 +25,22 @@ class JSONNotFound(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
 
+def is_safe_url(test_url):
+    import os
+    # See https://github.com/dgtlmoon/changedetection.io/issues/1358
+
+    # Remove 'source:' prefix so we dont get 'source:javascript:' etc
+    # 'source:' is a valid way to tell us to return the source
+
+    r = re.compile(re.escape('source:'), re.IGNORECASE)
+    test_url = r.sub('', test_url)
+
+    pattern = re.compile(os.getenv('SAFE_PROTOCOL_REGEX', SAFE_PROTOCOL_REGEX), re.IGNORECASE)
+    if not pattern.match(test_url.strip()):
+        return False
+
+    return True
+
 # Doesn't look like python supports forward slash auto enclosure in re.findall
 # So convert it to inline flag "(?i)foobar" type configuration
 @lru_cache(maxsize=100)

changedetectionio/model/App.py

@@ -1,5 +1,4 @@
 from os import getenv
-from copy import deepcopy
 
 from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES
 
@@ -75,8 +74,7 @@ class model(dict):
 
     def __init__(self, *arg, **kw):
         super(model, self).__init__(*arg, **kw)
-        # CRITICAL: deepcopy to avoid sharing mutable objects between instances
-        self.update(deepcopy(self.base_config))
+        self.update(self.base_config)
 
 
 def parse_headers_from_text_file(filepath):

changedetectionio/model/Watch.py

@@ -1,6 +1,5 @@
 from blinker import signal
-from changedetectionio.validate_url import is_safe_valid_url
-
+from changedetectionio.html_tools import is_safe_url
 from changedetectionio.strtobool import strtobool
 from changedetectionio.jinja2_custom import render as jinja_render
 from . import watch_base
@@ -13,6 +12,9 @@ from .. import jinja2_custom as safe_jinja
 from ..diff import ADDED_PLACEMARKER_OPEN
 from ..html_tools import TRANSLATE_WHITESPACE_TABLE
 
+# Allowable protocols, protects against javascript: etc
+# file:// is further checked by ALLOW_FILE_URI
+SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):'
 FAVICON_RESAVE_THRESHOLD_SECONDS=86400
 
 
@@ -61,7 +63,7 @@ class model(watch_base):
     def link(self):
 
         url = self.get('url', '')
-        if not is_safe_valid_url(url):
+        if not is_safe_url(url):
             return 'DISABLED'
 
         ready_url = url
@@ -82,7 +84,7 @@ class model(watch_base):
             ready_url=ready_url.replace('source:', '')
 
         # Also double check it after any Jinja2 formatting just incase
-        if not is_safe_valid_url(ready_url):
+        if not is_safe_url(ready_url):
             return 'DISABLED'
         return ready_url
 
@@ -276,17 +278,9 @@ class model(watch_base):
         # When the 'last viewed' timestamp is less than the oldest snapshot, return oldest
         return sorted_keys[-1]
 
-    def get_history_snapshot(self, timestamp=None, filepath=None):
-        """
-        Accepts either timestamp or filepath
-        :param timestamp:
-        :param filepath:
-        :return:
-        """
+    def get_history_snapshot(self, timestamp):
         import brotli
-
-        if not filepath:
-            filepath = self.history[timestamp]
+        filepath = self.history[timestamp]
 
         # See if a brotli versions exists and switch to that
         if not filepath.endswith('.br') and os.path.isfile(f"{filepath}.br"):
@@ -390,7 +384,7 @@ class model(watch_base):
         # Compare each lines (set) against each history text file (set) looking for something new..
         existing_history = set({})
         for k, v in self.history.items():
-            content = self.get_history_snapshot(filepath=v)
+            content = self.get_history_snapshot(k)
 
             if ignore_whitespace:
                 alist = set([line.translate(TRANSLATE_WHITESPACE_TABLE).lower() for line in content.splitlines()])
@@ -647,7 +641,7 @@ class model(watch_base):
         for k, fname in self.history.items():
             if os.path.isfile(fname):
                 if True:
-                    contents = self.get_history_snapshot(timestamp=k)
+                    contents = self.get_history_snapshot(k)
                     res = re.findall(regex, contents, re.MULTILINE)
                     if res:
                         if not csv_writer:
@@ -740,7 +734,7 @@ class model(watch_base):
             # If a previous attempt doesnt yet exist, just snarf the previous snapshot instead
             dates = list(self.history.keys())
             if len(dates):
-                return self.get_history_snapshot(timestamp=dates[-1])
+                return self.get_history_snapshot(dates[-1])
             else:
                 return ''
 

changedetectionio/notification/email_helpers.py

@@ -35,7 +35,7 @@ def as_monospaced_html_email(content: str, title: str) -> str:
 </head>
 <body style="-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%;">
   <pre role="article" aria-roledescription="email" lang="en"
-       style="font-family: monospace, 'Courier New', Courier; font-size: 0.9rem;
+       style="font-family: monospace, 'Courier New', Courier; font-size: 0.8em;
               white-space: pre-wrap; word-break: break-word;">{content}</pre>
 </body>
 </html>"""

changedetectionio/notification/handler.py

@@ -5,15 +5,13 @@ from apprise import NotifyFormat
 from loguru import logger
 from urllib.parse import urlparse
 from .apprise_plugin.assets import apprise_asset, APPRISE_AVATAR_URL
+from .apprise_plugin.custom_handlers import SUPPORTED_HTTP_METHODS
 from .email_helpers import as_monospaced_html_email
 from ..diff import HTML_REMOVED_STYLE, REMOVED_PLACEMARKER_OPEN, REMOVED_PLACEMARKER_CLOSED, ADDED_PLACEMARKER_OPEN, HTML_ADDED_STYLE, \
     ADDED_PLACEMARKER_CLOSED, CHANGED_INTO_PLACEMARKER_OPEN, CHANGED_INTO_PLACEMARKER_CLOSED, CHANGED_PLACEMARKER_OPEN, \
     CHANGED_PLACEMARKER_CLOSED, HTML_CHANGED_STYLE, HTML_CHANGED_INTO_STYLE
-import re
+from ..notification_service import NotificationContextData, CUSTOM_LINEBREAK_PLACEHOLDER
 
-from ..notification_service import NotificationContextData
-
-newline_re = re.compile(r'\r\n|\r|\n')
 
 
 def markup_text_links_to_html(body):
@@ -129,62 +127,6 @@ def apply_standard_markdown_to_body(n_body):
     return n_body
 
 
-def replace_placemarkers_in_text(text, url, requested_output_format):
-    """
-    Replace diff placemarkers in text based on the URL service type and requested output format.
-    Used for both notification title and body to ensure consistent placeholder replacement.
-
-    :param text: The text to process
-    :param url: The notification URL (to detect service type)
-    :param requested_output_format: The output format (html, htmlcolor, markdown, text, etc.)
-    :return: Processed text with placemarkers replaced
-    """
-    if not text:
-        return text
-
-    if url.startswith('tgram://'):
-        # Telegram only supports a limited subset of HTML
-        # Use strikethrough for removed content, bold for added content
-        text = text.replace(REMOVED_PLACEMARKER_OPEN, '<s>')
-        text = text.replace(REMOVED_PLACEMARKER_CLOSED, '</s>')
-        text = text.replace(ADDED_PLACEMARKER_OPEN, '<b>')
-        text = text.replace(ADDED_PLACEMARKER_CLOSED, '</b>')
-        # Handle changed/replaced lines (old → new)
-        text = text.replace(CHANGED_PLACEMARKER_OPEN, '<s>')
-        text = text.replace(CHANGED_PLACEMARKER_CLOSED, '</s>')
-        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, '<b>')
-        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, '</b>')
-    elif (url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks')
-          or url.startswith('https://discord.com/api')) and requested_output_format == 'html':
-        # Discord doesn't support HTML, use Discord markdown
-        text = apply_discord_markdown_to_body(n_body=text)
-    elif requested_output_format == 'htmlcolor':
-        # https://github.com/dgtlmoon/changedetection.io/issues/821#issuecomment-1241837050
-        text = text.replace(REMOVED_PLACEMARKER_OPEN, f'<span style="{HTML_REMOVED_STYLE}" role="deletion" aria-label="Removed text" title="Removed text">')
-        text = text.replace(REMOVED_PLACEMARKER_CLOSED, f'</span>')
-        text = text.replace(ADDED_PLACEMARKER_OPEN, f'<span style="{HTML_ADDED_STYLE}" role="insertion" aria-label="Added text" title="Added text">')
-        text = text.replace(ADDED_PLACEMARKER_CLOSED, f'</span>')
-        # Handle changed/replaced lines (old → new)
-        text = text.replace(CHANGED_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_STYLE}" role="note" aria-label="Changed text" title="Changed text">')
-        text = text.replace(CHANGED_PLACEMARKER_CLOSED, f'</span>')
-        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_INTO_STYLE}" role="note" aria-label="Changed into" title="Changed into">')
-        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'</span>')
-    elif requested_output_format == 'markdown':
-        # Markdown to HTML - Apprise will convert this to HTML
-        text = apply_standard_markdown_to_body(n_body=text)
-    else:
-        # plaintext, html, and default - use simple text markers
-        text = text.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')
-        text = text.replace(REMOVED_PLACEMARKER_CLOSED, '')
-        text = text.replace(ADDED_PLACEMARKER_OPEN, '(added) ')
-        text = text.replace(ADDED_PLACEMARKER_CLOSED, '')
-        text = text.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')
-        text = text.replace(CHANGED_PLACEMARKER_CLOSED, f'')
-        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')
-        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')
-
-    return text
-
 def apply_service_tweaks(url, n_body, n_title, requested_output_format):
 
     # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
@@ -196,12 +138,6 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
     if not n_body or not n_body.strip():
         return url, n_body, n_title
 
-    # Normalize URL scheme to lowercase to prevent case-sensitivity issues
-    # e.g., "Discord://webhook" -> "discord://webhook", "TGRAM://bot123" -> "tgram://bot123"
-    scheme_separator_pos = url.find('://')
-    if scheme_separator_pos > 0:
-        url = url[:scheme_separator_pos].lower() + url[scheme_separator_pos:]
-
     # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
     parsed = urlparse(url)
     k = '?' if not parsed.query else '&'
@@ -213,22 +149,24 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
             and not url.startswith('put'):
         url += k + f"avatar_url={APPRISE_AVATAR_URL}"
 
-    # Replace placemarkers in title first (this was the missing piece causing the bug)
-    # Titles are ALWAYS plain text across all notification services (Discord embeds, Slack attachments,
-    # email Subject headers, etc.), so we always use 'text' format for title placemarker replacement
-    # Looking over apprise library it seems that all plugins only expect plain-text.
-    n_title = replace_placemarkers_in_text(n_title, url, 'text')
-
     if url.startswith('tgram://'):
         # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
         # re https://github.com/dgtlmoon/changedetection.io/issues/555
         # @todo re-use an existing library we have already imported to strip all non-allowed tags
         n_body = n_body.replace('<br>', '\n')
         n_body = n_body.replace('</br>', '\n')
-        n_body = newline_re.sub('\n', n_body)
+        n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\n')
 
-        # Replace placemarkers for body
-        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
+        # Use strikethrough for removed content, bold for added content
+        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '<s>')
+        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '</s>')
+        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '<b>')
+        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '</b>')
+        # Handle changed/replaced lines (old → new)
+        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, '<s>')
+        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, '</s>')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, '<b>')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, '</b>')
 
         # real limit is 4096, but minus some for extra metadata
         payload_max_size = 3600
@@ -242,7 +180,7 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
         # Discord doesn't support HTML, replace <br> with newlines
         n_body = n_body.strip().replace('<br>', '\n')
         n_body = n_body.replace('</br>', '\n')
-        n_body = newline_re.sub('\n', n_body)
+        n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\n')
 
         # Don't replace placeholders or truncate here - let the custom Discord plugin handle it
         # The plugin will use embeds (6000 char limit across all embeds) if placeholders are present,
@@ -252,7 +190,7 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
         if requested_output_format == 'html':
             # No diff placeholders, use Discord markdown for any other formatting
             # Use Discord markdown: strikethrough for removed, bold for added
-            n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
+            n_body = apply_discord_markdown_to_body(n_body=n_body)
 
             # Apply 2000 char limit for plain content
             payload_max_size = 1700
@@ -263,17 +201,40 @@ def apply_service_tweaks(url, n_body, n_title, requested_output_format):
 
     # Is not discord/tgram and they want htmlcolor
     elif requested_output_format == 'htmlcolor':
-        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
-        n_body = newline_re.sub('<br>\n', n_body)
+        # https://github.com/dgtlmoon/changedetection.io/issues/821#issuecomment-1241837050
+        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, f'<span style="{HTML_REMOVED_STYLE}" role="deletion" aria-label="Removed text" title="Removed text">')
+        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, f'</span>')
+        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, f'<span style="{HTML_ADDED_STYLE}" role="insertion" aria-label="Added text" title="Added text">')
+        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, f'</span>')
+        # Handle changed/replaced lines (old → new)
+        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_STYLE}" role="note" aria-label="Changed text" title="Changed text">')
+        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'</span>')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'<span style="{HTML_CHANGED_INTO_STYLE}" role="note" aria-label="Changed into" title="Changed into">')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'</span>')
+        n_body = n_body.replace('\n', f'{CUSTOM_LINEBREAK_PLACEHOLDER}\n')
     elif requested_output_format == 'html':
-        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
-        n_body = newline_re.sub('<br>\n', n_body)
+        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')
+        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '')
+        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '(added) ')
+        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '')
+        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')
+        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')
+        n_body = n_body.replace('\n', f'{CUSTOM_LINEBREAK_PLACEHOLDER}\n')
     elif requested_output_format == 'markdown':
         # Markdown to HTML - Apprise will convert this to HTML
-        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
+        n_body = apply_standard_markdown_to_body(n_body=n_body)
 
     else: #plaintext etc default
-        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)
+        n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')
+        n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, '')
+        n_body = n_body.replace(ADDED_PLACEMARKER_OPEN, '(added) ')
+        n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, '')
+        n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')
+        n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')
+        n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')
 
     return url, n_body, n_title
 
@@ -334,18 +295,24 @@ def process_notification(n_object: NotificationContextData, datastore):
     with (apprise.LogCapture(level=apprise.logging.DEBUG) as logs):
         for url in n_object['notification_urls']:
 
+            # Get the notification body from datastore
             n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)
-            n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
 
             if n_object.get('markup_text_links_to_html_links'):
                 n_body = markup_text_links_to_html(body=n_body)
 
+            n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)
+
             url = url.strip()
-            if not url or url.startswith('#'):
-                logger.debug(f"Skipping commented out or empty notification URL - '{url}'")
+            if url.startswith('#'):
+                logger.trace(f"Skipping commented out notification URL - {url}")
                 continue
 
-            logger.info(f">> Process Notification: AppRise start notifying '{url}'")
+            if not url:
+                logger.warning(f"Process Notification: skipping empty notification URL.")
+                continue
+
+            logger.info(f">> Process Notification: AppRise notifying {url}")
             url = jinja_render(template_str=url, **notification_parameters)
 
             # If it's a plaintext document, and they want HTML type email/alerts, so it needs to be escaped
@@ -386,18 +353,25 @@ def process_notification(n_object: NotificationContextData, datastore):
                     requested_output_format = NotifyFormat.HTML.value
                     apprise_input_format = NotifyFormat.HTML.value  # Changed from MARKDOWN to HTML
 
+                # Could have arrived at any stage, so we dont end up running .escape on it
+                if 'html' in requested_output_format:
+                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '<br>\r\n')
+                else:
+                    # texty types
+                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\r\n')
+
             else:
                 # ?format was IN the apprise URL, they are kind of on their own here, we will try our best
                 if 'format=html' in url:
-                    n_body = newline_re.sub('<br>\r\n', n_body)
+                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '<br>\r\n')
                     # This will also prevent apprise from doing conversion
                     apprise_input_format = NotifyFormat.HTML.value
                     requested_output_format = NotifyFormat.HTML.value
                 elif 'format=text' in url:
+                    n_body = n_body.replace(CUSTOM_LINEBREAK_PLACEHOLDER, '\r\n')
                     apprise_input_format = NotifyFormat.TEXT.value
                     requested_output_format = NotifyFormat.TEXT.value
 
-
             sent_objs.append({'title': n_title,
                               'body': n_body,
                               'url': url})

changedetectionio/notification_service.py

@@ -9,8 +9,11 @@ for both sync and async workers
 from loguru import logger
 import time
 
+from changedetectionio.model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 from changedetectionio.notification import default_notification_format, valid_notification_formats
 
+# This gets modified on notification time (handler.py) depending on the required notification output
+CUSTOM_LINEBREAK_PLACEHOLDER='@BR@'
 
 
 # What is passed around as notification context, also used as the complete list of valid {{ tokens }}
@@ -20,14 +23,10 @@ class NotificationContextData(dict):
             'base_url': None,
             'current_snapshot': None,
             'diff': None,
-            'diff_clean': None,
             'diff_added': None,
-            'diff_added_clean': None,
             'diff_full': None,
-            'diff_full_clean': None,
             'diff_patch': None,
             'diff_removed': None,
-            'diff_removed_clean': None,
             'diff_url': None,
             'markup_text_links_to_html_links': False, # If automatic conversion of plaintext to HTML should happen
             'notification_timestamp': time.time(),
@@ -72,38 +71,6 @@ class NotificationContextData(dict):
 
         super().__setitem__(key, value)
 
-
-def set_basic_notification_vars(snapshot_contents, current_snapshot, prev_snapshot, watch, triggered_text):
-    now = time.time()
-    from changedetectionio import diff
-
-    n_object = {
-        'current_snapshot': snapshot_contents,
-        'diff': diff.render_diff(prev_snapshot, current_snapshot),
-        'diff_clean': diff.render_diff(prev_snapshot, current_snapshot, include_change_type_prefix=False),
-        'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False),
-        'diff_added_clean': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, include_change_type_prefix=False),
-        'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True),
-        'diff_full_clean': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, include_change_type_prefix=False),
-        'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, patch_format=True),
-        'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False),
-        'diff_removed_clean': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, include_change_type_prefix=False),
-        'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
-        'triggered_text': triggered_text,
-        'uuid': watch.get('uuid') if watch else None,
-        'watch_url': watch.get('url') if watch else None,
-        'watch_uuid': watch.get('uuid') if watch else None,
-        'watch_mime_type': watch.get('content-type')
-    }
-
-    # The \n's in the content from the above will get converted to <br> etc depending on the notification format
-
-    if watch:
-        n_object.update(watch.extra_notification_token_values())
-
-    logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time() - now:.3f}s")
-    return n_object
-
 class NotificationService:
     """
     Standalone notification service that handles all notification functionality
@@ -118,6 +85,7 @@ class NotificationService:
         """
         Queue a notification for a watch with full diff rendering and template variables
         """
+        from changedetectionio import diff
         from changedetectionio.notification import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 
         if not isinstance(n_object, NotificationContextData):
@@ -126,6 +94,8 @@ class NotificationService:
         dates = []
         trigger_text = ''
 
+        now = time.time()
+
         if watch:
             watch_history = watch.history
             dates = list(watch_history.keys())
@@ -133,7 +103,7 @@ class NotificationService:
 
         # Add text that was triggered
         if len(dates):
-            snapshot_contents = watch.get_history_snapshot(timestamp=dates[-1])
+            snapshot_contents = watch.get_history_snapshot(dates[-1])
         else:
             snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
 
@@ -147,23 +117,35 @@ class NotificationService:
             from . import html_tools
             triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)
             if triggered_text:
-                triggered_text = '\n'.join(triggered_text)
+                triggered_text = CUSTOM_LINEBREAK_PLACEHOLDER.join(triggered_text)
 
         # Could be called as a 'test notification' with only 1 snapshot available
         prev_snapshot = "Example text: example test\nExample text: change detection is cool\nExample text: some more examples\n"
         current_snapshot = "Example text: example test\nExample text: change detection is fantastic\nExample text: even more examples\nExample text: a lot more examples"
 
         if len(dates) > 1:
-            prev_snapshot = watch.get_history_snapshot(timestamp=dates[-2])
-            current_snapshot = watch.get_history_snapshot(timestamp=dates[-1])
+            prev_snapshot = watch.get_history_snapshot(dates[-2])
+            current_snapshot = watch.get_history_snapshot(dates[-1])
 
+        n_object.update({
+            'current_snapshot': snapshot_contents,
+            'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
+            'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
+            'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
+            'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER, patch_format=True),
+            'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=CUSTOM_LINEBREAK_PLACEHOLDER),
+            'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
+            'triggered_text': triggered_text,
+            'uuid': watch.get('uuid') if watch else None,
+            'watch_url': watch.get('url') if watch else None,
+            'watch_uuid': watch.get('uuid') if watch else None,
+            'watch_mime_type': watch.get('content-type')
+        })
 
-        n_object.update(set_basic_notification_vars(snapshot_contents=snapshot_contents,
-                                                    current_snapshot=current_snapshot,
-                                                    prev_snapshot=prev_snapshot,
-                                                    watch=watch,
-                                                    triggered_text=triggered_text))
+        if watch:
+            n_object.update(watch.extra_notification_token_values())
 
+        logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
         logger.debug("Queued notification for sending")
         self.notification_q.put(n_object)
 

changedetectionio/processors/__init__.py

@@ -91,8 +91,6 @@ class difference_detection_processor():
             else:
                 logger.debug("Skipping adding proxy data when custom Browser endpoint is specified. ")
 
-        logger.debug(f"Using proxy '{proxy_url}' for {self.watch['uuid']}")
-
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
         self.fetcher = fetcher_obj(proxy_override=proxy_url,

changedetectionio/processors/magic.py

@@ -88,7 +88,7 @@ class guess_stream_type():
                         magic_content_header = mime
 
         except Exception as e:
-            logger.warning(f"Error getting a more precise mime type from 'puremagic' library ({str(e)}), using content-based detection")
+            logger.error(f"Error getting a more precise mime type from 'puremagic' library ({str(e)}), using content-based detection")
 
         # Content-based detection (most reliable for text formats)
         # Check for HTML patterns first - if found, override magic's text/plain

changedetectionio/processors/text_json_diff/__init__.py

@@ -32,7 +32,7 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
     '''Used by @app.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])'''
     from changedetectionio import forms, html_tools
     from changedetectionio.model.Watch import model as watch_model
-    from concurrent.futures import ThreadPoolExecutor
+    from concurrent.futures import ProcessPoolExecutor
     from copy import deepcopy
     from flask import request
     import brotli
@@ -76,16 +76,13 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data):
             update_handler.fetcher.headers['content-type'] = tmp_watch.get('content-type')
 
             # Process our watch with filters and the HTML from disk, and also a blank watch with no filters but also with the same HTML from disk
-            # Do this as parallel threads (not processes) to avoid pickle issues with Lock objects
-            try:
-                with ThreadPoolExecutor(max_workers=2) as executor:
-                    future1 = executor.submit(_task, tmp_watch, update_handler)
-                    future2 = executor.submit(_task, blank_watch_no_filters, update_handler)
+            # Do this as a parallel process because it could take some time
+            with ProcessPoolExecutor(max_workers=2) as executor:
+                future1 = executor.submit(_task, tmp_watch, update_handler)
+                future2 = executor.submit(_task, blank_watch_no_filters, update_handler)
 
-                    text_after_filter = future1.result()
-                    text_before_filter = future2.result()
-            except Exception as e:
-                x=1
+                text_after_filter = future1.result()
+                text_before_filter = future2.result()
 
     try:
         trigger_line_numbers = html_tools.strip_ignore_text(content=text_after_filter,

changedetectionio/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-addopts = --no-start-live-server --live-server-port=0
+addopts = --no-start-live-server --live-server-port=5005
 #testpaths = tests pytest_invenio
 #live_server_scope = function
 

changedetectionio/run_basic_tests.sh

@@ -11,16 +11,19 @@ set -e
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-# REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser
-REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -n 30 --dist load  tests/test_*.py
+find tests/test_*py -type f|while read test_name
+do
+  echo "TEST RUNNING $test_name"
+  # REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser
+  REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 --tb=long $test_name
+done
 
-#time pytest -n auto --dist loadfile -vv --tb=long tests/test_*.py
 echo "RUNNING WITH BASE_URL SET"
 
 # Now re-run some tests with BASE_URL enabled
 # Re #65 - Ability to include a link back to the installation, in the notification.
 export BASE_URL="https://really-unique-domain.io"
-REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv --maxfail=1 tests/test_notification.py
+REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 tests/test_notification.py
 
 
 # Re-run with HIDE_REFERER set - could affect login

changedetectionio/run_proxy_tests.sh

@@ -5,6 +5,8 @@ set -e
 # enable debug
 set -x
 
+docker network inspect changedet-network >/dev/null 2>&1 || docker network create changedet-network
+
 # Test proxy list handling, starting two squids on different ports
 # Each squid adds a different header to the response, which is the main thing we test for.
 docker run --network changedet-network -d --name squid-one --hostname squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
@@ -19,13 +21,12 @@ docker run --network changedet-network -d \
   -v `pwd`/tests/proxy_list/squid-passwords.txt:/etc/squid3/passwords \
   ubuntu/squid:4.13-21.10_edge
 
-sleep 5
+
 ## 2nd test actually choose the preferred proxy from proxies.json
-# This will force a request via "proxy-two"
 docker run --network changedet-network \
-  -v `pwd`/tests/proxy_list/proxies.json-example:/tmp/proxies.json \
+  -v `pwd`/tests/proxy_list/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest -s tests/proxy_list/test_multiple_proxy.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_multiple_proxy.py'
 
 set +e
 echo "- Looking for chosen.changedetection.io request in squid-one - it should NOT be here"
@@ -49,10 +50,8 @@ fi
 # Test the UI configurable proxies
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py'
 
-# Give squid proxies a moment to flush their logs
-sleep 2
 
 # Should see a request for one.changedetection.io in there
 echo "- Looking for .changedetection.io request in squid-custom"
@@ -66,10 +65,7 @@ fi
 # Test "no-proxy" option
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py --datastore-path /tmp'
-
-# Give squid proxies a moment to flush their logs
-sleep 2
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py'
 
 # We need to handle grep returning 1
 set +e
@@ -86,8 +82,6 @@ for c in $(echo "squid-one squid-two squid-custom"); do
   fi
 done
 
-echo "docker ps output"
-docker ps
 
 docker kill squid-one squid-two squid-custom
 
@@ -96,19 +90,19 @@ docker kill squid-one squid-two squid-custom
 # Requests
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_proxy_noconnect.py'
 
 # Playwright
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py'
 
 # Puppeteer fast
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && FAST_PUPPETEER_CHROME_FETCHER=1 PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && FAST_PUPPETEER_CHROME_FETCHER=1 PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000 pytest tests/proxy_list/test_proxy_noconnect.py'
 
 # Selenium
 docker run --network changedet-network \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && WEBDRIVER_URL=http://selenium:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && WEBDRIVER_URL=http://selenium:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py'

changedetectionio/run_socks_proxy_tests.sh

@@ -15,13 +15,13 @@ docker run --network changedet-network -d --hostname socks5proxy-noauth --rm -p
 echo "---------------------------------- SOCKS5 -------------------"
 # SOCKS5 related - test from proxies.json
 docker run --network changedet-network \
-  -v `pwd`/tests/proxy_socks5/proxies.json-example:/tmp/proxies.json \
+  -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
   --rm \
   -e "FLASK_SERVER_NAME=cdio" \
   --hostname cdio \
   -e "SOCKSTEST=proxiesjson" \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py  --datastore-path /tmp'
+  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py'
 
 # SOCKS5 related - by manually entering in UI
 docker run --network changedet-network \
@@ -30,18 +30,18 @@ docker run --network changedet-network \
   --hostname cdio \
   -e "SOCKSTEST=manual" \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy.py'
 
 # SOCKS5 related - test from proxies.json via playwright - NOTE- PLAYWRIGHT DOESNT SUPPORT AUTHENTICATING PROXY
 docker run --network changedet-network \
   -e "SOCKSTEST=manual-playwright" \
   --hostname cdio \
   -e "FLASK_SERVER_NAME=cdio" \
-  -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/tmp/proxies.json \
+  -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \
   -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" \
   --rm \
   test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py --datastore-path /tmp'
+  bash -c 'cd changedetectionio && pytest --live-server-host=0.0.0.0 --live-server-port=5004  -s tests/proxy_socks5/test_socks5_proxy_sources.py'
 
 echo "socks5 server logs"
 docker logs socks5proxy

changedetectionio/static/js/notifications.js

@@ -14,10 +14,10 @@ $(document).ready(function () {
         e.preventDefault();
 
         data = {
-            notification_urls: $('textarea.notification-urls').val(),
-            notification_title: $('input.notification-title').val(),
-            notification_body: $('textarea.notification-body').val(),
-            notification_format: $('select.notification-format').val(),
+            notification_body: $('#notification_body').val(),
+            notification_format: $('#notification_format').val(),
+            notification_title: $('#notification_title').val(),
+            notification_urls: $('.notification-urls').val(),
             tags: $('#tags').val(),
             window_url: window.location.href,
         }

changedetectionio/static/styles/scss/styles.scss

@@ -329,18 +329,12 @@ a.pure-button-selected {
 .notifications-wrapper {
   padding-top: 0.5rem;
   #notification-test-log {
-    margin-top: 1rem;
-    padding: 1rem;
+    padding-top: 1rem;
     white-space: pre-wrap;
     word-break: break-word;
     overflow-wrap: break-word;
     max-width: 100%;
     box-sizing: border-box;
-    max-height: 12rem;
-    overflow-y: scroll;
-    border: 1px solid var(--color-border-notification);
-    border-radius: 5px;
-
   }
 }
 

changedetectionio/store.py

@@ -1,6 +1,5 @@
 from changedetectionio.strtobool import strtobool
-
-from changedetectionio.validate_url import is_safe_valid_url
+from changedetectionio.html_tools import is_safe_url
 
 from flask import (
     flash
@@ -42,24 +41,17 @@ class ChangeDetectionStore:
     needs_write_urgent = False
 
     __version_check = True
-    save_data_thread = None
 
     def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
         # Should only be active for docker
         # logging.basicConfig(filename='/dev/stdout', level=logging.INFO)
-
-        self.needs_write = False
-        self.start_time = time.time()
-        self.stop_thread = False
-        self.reload_state(datastore_path=datastore_path, include_default_watches=include_default_watches, version_tag=version_tag)
-
-
-    def reload_state(self, datastore_path, include_default_watches, version_tag):
-        logger.info(f"Datastore path is '{datastore_path}'")
-
         self.__data = App.model()
         self.datastore_path = datastore_path
         self.json_store_path = os.path.join(self.datastore_path, "url-watches.json")
+        logger.info(f"Datastore path is '{self.json_store_path}'")
+        self.needs_write = False
+        self.start_time = time.time()
+        self.stop_thread = False
         # Base definition for all watchers
         # deepcopy part of #569 - not sure why its needed exactly
         self.generic_definition = deepcopy(Watch.model(datastore_path = datastore_path, default={}))
@@ -152,10 +144,7 @@ class ChangeDetectionStore:
         self.needs_write = True
 
         # Finally start the thread that will manage periodic data saves to JSON
-        # Only start if thread is not already running (reload_state might be called multiple times)
-        if not self.save_data_thread or not self.save_data_thread.is_alive():
-            self.save_data_thread = threading.Thread(target=self.save_datastore)
-            self.save_data_thread.start()
+        save_data_thread = threading.Thread(target=self.save_datastore).start()
 
     def rehydrate_entity(self, uuid, entity, processor_override=None):
         """Set the dict back to the dict Watch object"""
@@ -261,8 +250,7 @@ class ChangeDetectionStore:
                 self.__data['watching'] = {}
                 time.sleep(1) # Mainly used for testing to allow all items to flush before running next test
                 for uuid in self.data['watching']:
-                    path = pathlib.Path(
-                        os.path.join(self.datastore_path, uuid))
+                    path = pathlib.Path(os.path.join(self.datastore_path, uuid))
                     if os.path.exists(path):
                         self.delete(uuid)
 
@@ -353,10 +341,8 @@ class ChangeDetectionStore:
                 logger.error(f"Error fetching metadata for shared watch link {url} {str(e)}")
                 flash("Error fetching metadata for {}".format(url), 'error')
                 return False
-
-        if not is_safe_valid_url(url):
-            flash('Watch protocol is not permitted or invalid URL format', 'error')
-
+        if not is_safe_url(url):
+            flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error')
             return None
 
         if tag and type(tag) == str:
@@ -422,6 +408,7 @@ class ChangeDetectionStore:
             self.sync_to_json()
             return
         else:
+
             try:
                 # Re #286  - First write to a temp file, then confirm it looks OK and rename it
                 # This is a fairly basic strategy to deal with the case that the file is corrupted,
@@ -451,7 +438,7 @@ class ChangeDetectionStore:
                 logger.remove()
                 logger.add(sys.stderr)
 
-                logger.info(f"Shutting down datastore '{self.datastore_path}' thread")
+                logger.critical("Shutting down datastore thread")
                 return
 
             if self.needs_write or self.needs_write_urgent:
@@ -1006,38 +993,28 @@ class ChangeDetectionStore:
 
 
     # Some notification formats got the wrong name type
-    def update_23(self):
-
-        def re_run(formats):
-            sys_n_format = self.data['settings']['application'].get('notification_format')
-            key_exists_as_value = next((k for k, v in formats.items() if v == sys_n_format), None)
-            if key_exists_as_value:  # key of "Plain text"
-                logger.success(f"['settings']['application']['notification_format'] '{sys_n_format}' -> '{key_exists_as_value}'")
-                self.data['settings']['application']['notification_format'] = key_exists_as_value
-
-            for uuid, watch in self.data['watching'].items():
-                n_format = self.data['watching'][uuid].get('notification_format')
-                key_exists_as_value = next((k for k, v in formats.items() if v == n_format), None)
-                if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text"
-                    logger.success(f"['watching'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'")
-                    self.data['watching'][uuid]['notification_format'] = key_exists_as_value  # should be 'text' or whatever
-
-            for uuid, tag in self.data['settings']['application']['tags'].items():
-                n_format = self.data['settings']['application']['tags'][uuid].get('notification_format')
-                key_exists_as_value = next((k for k, v in formats.items() if v == n_format), None)
-                if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text"
-                    logger.success(
-                        f"['settings']['application']['tags'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'")
-                    self.data['settings']['application']['tags'][uuid][
-                        'notification_format'] = key_exists_as_value  # should be 'text' or whatever
-
+    def update_22(self):
         from .notification import valid_notification_formats
-        formats = deepcopy(valid_notification_formats)
-        re_run(formats)
-        # And in previous versions, it was "text" instead of Plain text, Markdown instead of "Markdown to HTML"
-        formats['text'] = 'Text'
-        formats['markdown'] = 'Markdown'
-        re_run(formats)
+
+        sys_n_format = self.data['settings']['application'].get('notification_format')
+        key_exists_as_value = next((k for k, v in valid_notification_formats.items() if v == sys_n_format), None)
+        if key_exists_as_value: # key of "Plain text"
+            logger.success(f"['settings']['application']['notification_format'] '{sys_n_format}' -> '{key_exists_as_value}'")
+            self.data['settings']['application']['notification_format'] = key_exists_as_value
+
+        for uuid, watch in self.data['watching'].items():
+            n_format = self.data['watching'][uuid].get('notification_format')
+            key_exists_as_value = next((k for k, v in valid_notification_formats.items() if v == n_format), None)
+            if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text"
+                logger.success(f"['watching'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'")
+                self.data['watching'][uuid]['notification_format'] = key_exists_as_value # should be 'text' or whatever
+
+        for uuid, tag in self.data['settings']['application']['tags'].items():
+            n_format = self.data['settings']['application']['tags'][uuid].get('notification_format')
+            key_exists_as_value = next((k for k, v in valid_notification_formats.items() if v == n_format), None)
+            if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of "Plain text"
+                logger.success(f"['settings']['application']['tags'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'")
+                self.data['settings']['application']['tags'][uuid]['notification_format'] = key_exists_as_value # should be 'text' or whatever
 
     def add_notification_url(self, notification_url):
         

changedetectionio/templates/_common_fields.html

@@ -87,35 +87,19 @@
 									<tr>
                                         <td><code>{{ '{{diff}}' }}</code></td>
                                         <td>The diff output - only changes, additions, and removals</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff_clean}}' }}</code></td>
-                                        <td>The diff output - only changes, additions, and removals &dash; <i>Without (added) prefix or colors</i></td>
                                     </tr>
 									<tr>
                                         <td><code>{{ '{{diff_added}}' }}</code></td>
                                         <td>The diff output - only changes and additions</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff_added_clean}}' }}</code></td>
-                                        <td>The diff output - only changes and additions &dash; <i>Without (added) prefix or colors</i></td>
                                     </tr>
 									<tr>
                                         <td><code>{{ '{{diff_removed}}' }}</code></td>
                                         <td>The diff output - only changes and removals</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff_removed_clean}}' }}</code></td>
-                                        <td>The diff output - only changes and removals &dash; <i>Without (added) prefix or colors</i></td>
                                     </tr>
                                     <tr>
                                         <td><code>{{ '{{diff_full}}' }}</code></td>
                                         <td>The diff output - full difference output</td>
                                     </tr>
-                                    <tr>
-                                        <td><code>{{ '{{diff_full_clean}}' }}</code></td>
-                                        <td>The diff output - full difference output &dash; <i>Without (added) prefix or colors</i></td>
-                                    </tr>
                                     <tr>
                                         <td><code>{{ '{{diff_patch}}' }}</code></td>
                                         <td>The diff output - patch in unified format</td>

changedetectionio/templates/base.html

@@ -53,7 +53,7 @@
           <a class="pure-menu-heading" href="{{url_for('watchlist.index')}}">
             <strong>Change</strong>Detection.io</a>
         {% endif %}
-        {% if current_diff_url and is_safe_valid_url(current_diff_url) %}
+        {% if current_diff_url and is_safe_url(current_diff_url) %}
           <a class="current-diff-url" href="{{ current_diff_url }}">
             <span style="max-width: 30%; overflow: hidden">{{ current_diff_url }}</span></a>
         {% else %}

changedetectionio/tests/conftest.py

@@ -11,7 +11,6 @@ import os
 import sys
 from loguru import logger
 
-from changedetectionio.flask_app import init_app_secret
 from changedetectionio.tests.util import live_server_setup, new_live_server_setup
 
 # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
@@ -88,6 +87,7 @@ def measure_memory_usage(request):
 def cleanup(datastore_path):
     import glob
     # Unlink test output files
+
     for g in ["*.txt", "*.json", "*.pdf"]:
         files = glob.glob(os.path.join(datastore_path, g))
         for f in files:
@@ -97,121 +97,34 @@ def cleanup(datastore_path):
             if os.path.isfile(f):
                 os.unlink(f)
 
-def pytest_addoption(parser):
-    """Add custom command-line options for pytest.
-
-    Provides --datastore-path option for specifying custom datastore location.
-    Note: Cannot use -d short option as it's reserved by pytest for debug mode.
-    """
-    parser.addoption(
-        "--datastore-path",
-        action="store",
-        default=None,
-        help="Custom datastore path for tests"
-    )
-
-@pytest.fixture(scope='session')
-def datastore_path(tmp_path_factory, request):
-    """Provide datastore path unique to this worker.
-
-    Supports custom path via --datastore-path/-d flag (mirrors main app).
-
-    CRITICAL for xdist isolation:
-    - Each WORKER gets its own directory
-    - Tests on same worker run SEQUENTIALLY and cleanup between tests
-    - No subdirectories needed since tests don't overlap on same worker
-    - Example: /tmp/test-datastore-gw0/ for worker gw0
-    """
-    # Check for custom path first (mirrors main app's -d flag)
-    custom_path = request.config.getoption("--datastore-path")
-    if custom_path:
-        # Ensure the directory exists
-        os.makedirs(custom_path, exist_ok=True)
-        logger.info(f"Using custom datastore path: {custom_path}")
-        return custom_path
-
-    # Otherwise use default tmp_path_factory logic
-    worker_id = getattr(request.config, 'workerinput', {}).get('workerid', 'master')
-    if worker_id == 'master':
-        path = tmp_path_factory.mktemp("test-datastore")
-    else:
-        path = tmp_path_factory.mktemp(f"test-datastore-{worker_id}")
-    return str(path)
-
-
 @pytest.fixture(scope='function', autouse=True)
-def prepare_test_function(live_server, datastore_path):
-    """Prepare each test with complete isolation.
+def prepare_test_function(live_server):
 
-    CRITICAL for xdist per-test isolation:
-    - Reuses the SAME datastore instance (so blueprint references stay valid)
-    - Clears all watches and state for a clean slate
-    - First watch will get uuid="first"
-    """
     routes = [rule.rule for rule in live_server.app.url_map.iter_rules()]
     if '/test-random-content-endpoint' not in routes:
         logger.debug("Setting up test URL routes")
         new_live_server_setup(live_server)
 
-    # CRITICAL: Point app to THIS test's unique datastore directory
-    live_server.app.config['TEST_DATASTORE_PATH'] = datastore_path
-
-    # CRITICAL: Get datastore and stop it from writing stale data
-    datastore = live_server.app.config.get('DATASTORE')
-
-    # Prevent background thread from writing during cleanup/reload
-    datastore.needs_write = False
-    datastore.needs_write_urgent = False
-
-    # CRITICAL: Clean up any files from previous tests
-    # This ensures a completely clean directory
-    cleanup(datastore_path)
-
-    # CRITICAL: Reload the EXISTING datastore instead of creating a new one
-    # This keeps blueprint references valid (they capture datastore at construction)
-    # reload_state() completely resets the datastore to a clean state
-
-    # Reload state with clean data (no default watches)
-    datastore.reload_state(
-        datastore_path=datastore_path,
-        include_default_watches=False,
-        version_tag=datastore.data.get('version_tag', '0.0.0')
-    )
-    live_server.app.secret_key = init_app_secret(datastore_path)
-    logger.debug(f"prepare_test_function: Reloaded datastore at {hex(id(datastore))}")
-    logger.debug(f"prepare_test_function: Path {datastore.datastore_path}")
 
     yield
-
-    # Cleanup: Clear watches again after test
-    try:
-        datastore.data['watching'] = {}
-        datastore.needs_write = True
-    except Exception as e:
-        logger.warning(f"Error during datastore cleanup: {e}")
-
-
-# So the app can also know which test name it was
-@pytest.fixture(autouse=True)
-def set_test_name(request):
-  """Automatically set TEST_NAME env var for every test"""
-  test_name = request.node.name
-  os.environ['PYTEST_CURRENT_TEST'] = test_name
-  yield
-  # Cleanup if needed
+    # Then cleanup/shutdown
+    live_server.app.config['DATASTORE'].data['watching']={}
+    time.sleep(0.3)
+    live_server.app.config['DATASTORE'].data['watching']={}
 
 
 @pytest.fixture(scope='session')
-def app(request, datastore_path):
-    """Create application once per worker (session).
+def app(request):
+    """Create application for the tests."""
+    datastore_path = "./test-datastore"
 
-    Note: Actual per-test isolation is handled by:
-    - prepare_test_function() recreates datastore and cleans directory
-    - All tests on same worker use same directory (cleaned between tests)
-    """
     # So they don't delay in fetching
     os.environ["MINIMUM_SECONDS_RECHECK_TIME"] = "0"
-    logger.debug(f"Testing with datastore_path={datastore_path}")
+    try:
+        os.mkdir(datastore_path)
+    except FileExistsError:
+        pass
+
     cleanup(datastore_path)
 
     app_config = {'datastore_path': datastore_path, 'disable_checkver' : True}
@@ -234,8 +147,6 @@ def app(request, datastore_path):
     # Disable CSRF while running tests
     app.config['WTF_CSRF_ENABLED'] = False
     app.config['STOP_THREADS'] = True
-    # Store datastore_path so Flask routes can access it
-    app.config['TEST_DATASTORE_PATH'] = datastore_path
 
     def teardown():
         # Stop all threads and services

changedetectionio/tests/custom_browser_url/test_custom_browser_url.py

@@ -73,13 +73,13 @@ def do_test(client, live_server, make_test_use_extra_browser=False):
 
 
 # Requires playwright to be installed
-def test_request_via_custom_browser_url(client, live_server, measure_memory_usage, datastore_path):
+def test_request_via_custom_browser_url(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     # We do this so we can grep the logs of the custom container and see if the request actually went through that container
     do_test(client, live_server, make_test_use_extra_browser=True)
 
 
-def test_request_not_via_custom_browser_url(client, live_server, measure_memory_usage, datastore_path):
+def test_request_not_via_custom_browser_url(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     # We do this so we can grep the logs of the custom container and see if the request actually went through that container
     do_test(client, live_server, make_test_use_extra_browser=False)

changedetectionio/tests/fetchers/test_content.py

@@ -8,7 +8,7 @@ import logging
 
 
 # Requires playwright to be installed
-def test_fetch_webdriver_content(client, live_server, measure_memory_usage, datastore_path):
+def test_fetch_webdriver_content(client, live_server, measure_memory_usage):
     #  live_server_setup(live_server) # Setup on conftest per function
 
     #####################

changedetectionio/tests/fetchers/test_custom_js_before_content.py

@@ -3,7 +3,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 
-def test_execute_custom_js(client, live_server, measure_memory_usage, datastore_path):
+def test_execute_custom_js(client, live_server, measure_memory_usage):
 
    #  live_server_setup(live_server) # Setup on conftest per function
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks
 
 
-def test_preferred_proxy(client, live_server, measure_memory_usage, datastore_path):
+def test_preferred_proxy(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     url = "http://chosen.changedetection.io"
 

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 
-def test_noproxy_option(client, live_server, measure_memory_usage, datastore_path):
+def test_noproxy_option(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     # Run by run_proxy_tests.sh
     # Call this URL then scan the containers that it never went through them

changedetectionio/tests/proxy_list/test_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
 # just make a request, we will grep in the docker logs to see it actually got called
-def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path):
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     res = client.post(
         url_for("imports.import_page"),

changedetectionio/tests/proxy_list/test_proxy_noconnect.py

@@ -12,7 +12,7 @@ from ... import strtobool
 # FAST_PUPPETEER_CHROME_FETCHER=True PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000 pytest tests/proxy_list/test_proxy_noconnect.py
 # WEBDRIVER_URL=http://127.0.0.1:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py
 
-def test_proxy_noconnect_custom(client, live_server, measure_memory_usage, datastore_path):
+def test_proxy_noconnect_custom(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
     # Goto settings, add our custom one

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -6,7 +6,7 @@ from ..util import live_server_setup, wait_for_all_checks
 import os
 
 # just make a request, we will grep in the docker logs to see it actually got called
-def test_select_custom(client, live_server, measure_memory_usage, datastore_path):
+def test_select_custom(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
     # Goto settings, add our custom one
@@ -50,7 +50,7 @@ def test_select_custom(client, live_server, measure_memory_usage, datastore_path
     # Now we should see the request in the container logs for "squid-squid-custom" because it will be the only default
 
 
-def test_custom_proxy_validation(client, live_server, measure_memory_usage, datastore_path):
+def test_custom_proxy_validation(client, live_server, measure_memory_usage):
     #  live_server_setup(live_server) # Setup on conftest per function
 
     # Goto settings, add our custom one

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -5,7 +5,7 @@ from flask import url_for
 from changedetectionio.tests.util import live_server_setup, wait_for_all_checks, extract_UUID_from_client, delete_all_watches
 
 
-def set_response(datastore_path):
+def set_response():
     import time
     data = """<html>
        <body>
@@ -15,13 +15,13 @@ def set_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(data)
     time.sleep(1)
 
-def test_socks5(client, live_server, measure_memory_usage, datastore_path):
+def test_socks5(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_response(datastore_path)
+    set_response()
 
     # Setup a proxy
     res = client.post(

changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py

@@ -4,7 +4,7 @@ from flask import url_for
 from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
 
 
-def set_response(datastore_path):
+def set_response():
     import time
     data = """<html>
        <body>
@@ -14,15 +14,15 @@ def set_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(data)
     time.sleep(1)
 
 # should be proxies.json mounted from run_proxy_tests.sh already
 # -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json
-def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage, datastore_path):
+def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_response(datastore_path)
+    set_response()
     # Because the socks server should connect back to us
     test_url = url_for('test_endpoint', _external=True) + f"?socks-test-tag={os.getenv('SOCKSTEST', '')}"
     test_url = test_url.replace('localhost.localdomain', 'cdio')

changedetectionio/tests/restock/test_restock.py

@@ -11,7 +11,7 @@ from changedetectionio.notification import (
 )
 
 
-def set_original_response(datastore_path):
+def set_original_response():
     test_return_data = """<html>
        <body>
        <section id=header style="padding: 50px; height: 350px">This is the header which should be ignored always - <span>add to cart</span></section>
@@ -26,13 +26,13 @@ def set_original_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
 
 
-def set_back_in_stock_response(datastore_path):
+def set_back_in_stock_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -45,14 +45,14 @@ def set_back_in_stock_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
-def test_restock_detection(client, live_server, measure_memory_usage, datastore_path):
+def test_restock_detection(client, live_server, measure_memory_usage):
 
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
     #assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
    #  live_server_setup(live_server) # Setup on conftest per function
     #####################
@@ -88,25 +88,24 @@ def test_restock_detection(client, live_server, measure_memory_usage, datastore_
     assert b'not-in-stock' in res.data # should be out of stock
 
     # Is it correctly shown as in stock
-    set_back_in_stock_response(datastore_path)
+    set_back_in_stock_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
     assert b'not-in-stock' not in res.data
 
     # We should have a notification
-    notification_file = os.path.join(datastore_path, "notification.txt")
-    wait_for_notification_endpoint_output(datastore_path=datastore_path)
-    assert os.path.isfile(notification_file), "Notification received"
-    os.unlink(notification_file)
+    wait_for_notification_endpoint_output()
+    assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
+    os.unlink("test-datastore/notification.txt")
 
     # Default behaviour is to only fire notification when it goes OUT OF STOCK -> IN STOCK
     # So here there should be no file, because we go IN STOCK -> OUT OF STOCK
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     time.sleep(5)
-    assert not os.path.isfile(notification_file), "No notification should have fired when it went OUT OF STOCK by default"
+    assert not os.path.isfile("test-datastore/notification.txt"), "No notification should have fired when it went OUT OF STOCK by default"
 
     # BUT we should see that it correctly shows "not in stock"
     res = client.get(url_for("watchlist.index"))

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -3,9 +3,8 @@ from flask import url_for
 from email import message_from_string
 from email.policy import default as email_policy
 
-from changedetectionio.diff import HTML_REMOVED_STYLE, HTML_ADDED_STYLE, HTML_CHANGED_STYLE, REMOVED_PLACEMARKER_OPEN, \
-    CHANGED_PLACEMARKER_OPEN, ADDED_PLACEMARKER_OPEN
-from changedetectionio.notification_service import NotificationContextData
+from changedetectionio.diff import HTML_REMOVED_STYLE, HTML_ADDED_STYLE, HTML_CHANGED_STYLE
+from changedetectionio.notification_service import NotificationContextData, CUSTOM_LINEBREAK_PLACEHOLDER
 from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
     wait_for_all_checks, \
     set_longer_modified_response, delete_all_watches
@@ -41,10 +40,9 @@ def get_last_message_from_smtp_server():
 
 # Requires running the test SMTP server
 
-def test_check_notification_email_formats_default_HTML(client, live_server, measure_memory_usage, datastore_path):
+def test_check_notification_email_formats_default_HTML(client, live_server, measure_memory_usage):
     ##  live_server_setup(live_server) # Setup on conftest per function
-    set_original_response(datastore_path=datastore_path)
-
+    set_original_response()
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
 
@@ -73,7 +71,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     assert b"Watch added" in res.data
 
     wait_for_all_checks(client)
-    set_longer_modified_response(datastore_path=datastore_path)
+    set_longer_modified_response()
     time.sleep(2)
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -101,6 +99,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     text_content = text_part.get_content()
     assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
     assert 'fallback-body\r\n' in text_content  # The plaintext part
+    assert CUSTOM_LINEBREAK_PLACEHOLDER not in text_content
 
     # Second part should be text/html
     html_part = parts[1]
@@ -109,12 +108,12 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas
     assert 'some text<br>' in html_content  # We converted \n from the notification body
     assert 'fallback-body<br>' in html_content  # kept the original <br>
     assert '(added) So let\'s see what happens.<br>' in html_content  # the html part
+    assert CUSTOM_LINEBREAK_PLACEHOLDER not in html_content
     delete_all_watches(client)
 
 
-def test_check_notification_plaintext_format(client, live_server, measure_memory_usage, datastore_path):
-    set_original_response(datastore_path=datastore_path)
-
+def test_check_notification_plaintext_format(client, live_server, measure_memory_usage):
+    set_original_response()
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
 
@@ -123,8 +122,8 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
     res = client.post(
         url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title {{watch_title}}  {{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}  " + default_notification_title,
-              "application-notification_body": f"some text\n" + default_notification_body + f"\nMore output test\n{ALL_MARKUP_TOKENS}",
+              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_body": "some text\n" + default_notification_body,
               "application-notification_format": 'text',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
@@ -139,7 +138,7 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     time.sleep(2)
 
-    set_longer_modified_response(datastore_path=datastore_path)
+    set_longer_modified_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -147,18 +146,9 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
 
     msg_raw = get_last_message_from_smtp_server()
     assert len(msg_raw) >= 1
-    #time.sleep(60)
+
     # Parse the email properly using Python's email library
     msg = message_from_string(msg_raw, policy=email_policy)
-    # Subject/title got marked up
-    subject = msg['subject']
-    # Subject should always be plaintext and never marked up to anything else
-    assert REMOVED_PLACEMARKER_OPEN not in subject
-    assert CHANGED_PLACEMARKER_OPEN not in subject
-    assert ADDED_PLACEMARKER_OPEN not in subject
-    assert 'diff added didnt split' not in subject
-    assert '(changed) Which is across' in subject
-    assert 'PLACEMARKER' not in subject
 
     # The email should be plain text only (not multipart)
     assert not msg.is_multipart()
@@ -174,9 +164,8 @@ def test_check_notification_plaintext_format(client, live_server, measure_memory
 
 
 
-def test_check_notification_html_color_format(client, live_server, measure_memory_usage, datastore_path):
-    set_original_response(datastore_path=datastore_path)
-
+def test_check_notification_html_color_format(client, live_server, measure_memory_usage):
+    set_original_response()
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
 
@@ -185,7 +174,7 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
     res = client.post(
         url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title {{watch_title}} - diff_added_lines_test : '{{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}' " + default_notification_title,
+              "application-notification_title": "fallback-title " + default_notification_title,
               "application-notification_body": f"some text\n{default_notification_body}\nMore output test\n{ALL_MARKUP_TOKENS}",
               "application-notification_format": 'htmlcolor',
               "requests-time_between_check-minutes": 180,
@@ -206,7 +195,7 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
     assert b"Watch added" in res.data
 
     wait_for_all_checks(client)
-    set_longer_modified_response(datastore_path=datastore_path)
+    set_longer_modified_response()
     time.sleep(2)
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -219,19 +208,6 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
 
     # Parse the email properly using Python's email library
     msg = message_from_string(msg_raw, policy=email_policy)
-    # Subject/title got marked up
-    subject = msg['subject']
-    # Subject should always be plaintext and never marked up to anything else
-    assert REMOVED_PLACEMARKER_OPEN not in subject
-    assert CHANGED_PLACEMARKER_OPEN not in subject
-    assert ADDED_PLACEMARKER_OPEN not in subject
-    assert 'diff added didnt split' not in subject
-    assert '(changed) Which is across' in subject
-    assert 'PLACEMARKER' not in subject
-    assert 'head title' in subject
-    assert "span" not in subject
-    assert 'background-color' not in subject
-
 
     # The email should have two bodies (multipart/alternative with text/plain and text/html)
     assert msg.is_multipart()
@@ -259,9 +235,8 @@ def test_check_notification_html_color_format(client, live_server, measure_memor
     assert 'some text<br>' in html_content
     delete_all_watches(client)
 
-def test_check_notification_markdown_format(client, live_server, measure_memory_usage, datastore_path):
-    set_original_response(datastore_path=datastore_path)
-
+def test_check_notification_markdown_format(client, live_server, measure_memory_usage):
+    set_original_response()
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
 
@@ -270,7 +245,7 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_
     res = client.post(
         url_for("settings.settings_page"),
         data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title  diff_added_lines_test : '{{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}' " + default_notification_title,
+              "application-notification_title": "fallback-title " + default_notification_title,
               "application-notification_body": "*header*\n\nsome text\n" + default_notification_body,
               "application-notification_format": 'markdown',
               "requests-time_between_check-minutes": 180,
@@ -291,7 +266,7 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_
     assert b"Watch added" in res.data
 
     wait_for_all_checks(client)
-    set_longer_modified_response(datastore_path=datastore_path)
+    set_longer_modified_response()
     time.sleep(2)
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -308,14 +283,6 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_
     # The email should have two bodies (multipart/alternative with text/plain and text/html)
     assert msg.is_multipart()
     assert msg.get_content_type() == 'multipart/alternative'
-    subject = msg['subject']
-    # Subject should always be plaintext and never marked up to anything else
-    assert REMOVED_PLACEMARKER_OPEN not in subject
-    assert CHANGED_PLACEMARKER_OPEN not in subject
-    assert ADDED_PLACEMARKER_OPEN not in subject
-    assert 'diff added didnt split' not in subject
-    assert '(changed) Which is across' in subject
-
 
     # Get the parts
     parts = list(msg.iter_parts())
@@ -334,20 +301,16 @@ def test_check_notification_markdown_format(client, live_server, measure_memory_
     assert html_part.get_content_type() == 'text/html'
     html_content = html_part.get_content()
     assert '<p><em>header</em></p>' in html_content
-    assert '<strong>So let\'s see what happens.</strong><br />' in html_content # Additions are <strong> in markdown
-    # the '<br />' will come from apprises conversion, not from our code, we would rather use '<br>' correctly
-    # the '<br />' is actually a nice way to know if apprise done the conversion.
-
+    assert '<strong>So let\'s see what happens.</strong><br>' in html_content # Additions are <strong> in markdown
     delete_all_watches(client)
 
 # Custom notification body with HTML, that is either sent as HTML or rendered to plaintext and sent
-def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage, datastore_path):
+def test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage):
 
     # HTML problems? see this
     # https://github.com/caronc/apprise/issues/633
 
-    set_original_response(datastore_path=datastore_path)
-
+    set_original_response()
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
     notification_body = f"""<!DOCTYPE html>
 <html lang="en">
@@ -387,7 +350,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     #################################### FIRST SITUATION, PLAIN TEXT NOTIFICATION IS WANTED BUT WE HAVE HTML IN OUR TEMPLATE AND CONTENT ##########
     wait_for_all_checks(client)
-    set_longer_modified_response(datastore_path=datastore_path)
+    set_longer_modified_response()
     time.sleep(2)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
@@ -412,8 +375,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
 
     #################################### SECOND SITUATION, HTML IS CORRECTLY PASSED THROUGH TO THE EMAIL ####################
-    set_original_response(datastore_path=datastore_path)
-
+    set_original_response()
     # Now override as HTML format
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
@@ -462,11 +424,10 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
 
     delete_all_watches(client)
 
-def test_check_plaintext_document_plaintext_notification_smtp(client, live_server, measure_memory_usage, datastore_path):
+def test_check_plaintext_document_plaintext_notification_smtp(client, live_server, measure_memory_usage):
     """When following a plaintext document, notification in Plain Text format is sent correctly"""
-    import os
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("Some nice plain text\nwhich we add some extra data\nover here\n")
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
@@ -493,7 +454,7 @@ def test_check_plaintext_document_plaintext_notification_smtp(client, live_serve
     wait_for_all_checks(client)
 
     # Change the content
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("Some nice plain text\nwhich we add some extra data\nAnd let's talk about <title> tags\nover here\n")
 
 
@@ -515,11 +476,10 @@ def test_check_plaintext_document_plaintext_notification_smtp(client, live_serve
     assert '<pre' not in body
     delete_all_watches(client)
 
-def test_check_plaintext_document_html_notifications(client, live_server, measure_memory_usage, datastore_path):
+def test_check_plaintext_document_html_notifications(client, live_server, measure_memory_usage):
     """When following a plaintext document, notification in Plain Text format is sent correctly"""
-    import os
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("    Some nice plain text\nwhich we add some extra data\nover here\n")
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
@@ -546,7 +506,7 @@ def test_check_plaintext_document_html_notifications(client, live_server, measur
     wait_for_all_checks(client)
 
     # Change the content
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("    Some nice plain text\nwhich we add some extra data\nAnd let's talk about <title> tags\nover here\n")
 
 
@@ -585,7 +545,6 @@ def test_check_plaintext_document_html_notifications(client, live_server, measur
     # Should be the HTML, but not HTML Color
     assert 'background-color' not in html_content
     assert '<br>(added) And let&#39;s talk about &lt;title&gt; tags<br>' in html_content
-    assert 'PLACEMARKER' not in html_content
     assert '&lt;br' not in html_content
     assert '<pre role="article"' in html_content # Should have got wrapped nicely in email_helpers.py
 
@@ -595,11 +554,10 @@ def test_check_plaintext_document_html_notifications(client, live_server, measur
     delete_all_watches(client)
 
 
-def test_check_plaintext_document_html_color_notifications(client, live_server, measure_memory_usage, datastore_path):
+def test_check_plaintext_document_html_color_notifications(client, live_server, measure_memory_usage):
     """When following a plaintext document, notification in Plain Text format is sent correctly"""
-    import os
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("Some nice plain text\nwhich we add some extra data\nover here\n")
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
@@ -627,7 +585,7 @@ def test_check_plaintext_document_html_color_notifications(client, live_server,
     wait_for_all_checks(client)
 
     # Change the content
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("Some nice plain text\nwhich we add some extra data\nAnd let's talk about <title> tags\nover here\n")
 
     time.sleep(1)
@@ -668,11 +626,10 @@ def test_check_plaintext_document_html_color_notifications(client, live_server,
     assert '<pre role="article"' in html_content # Should have got wrapped nicely in email_helpers.py
     delete_all_watches(client)
 
-def test_check_html_document_plaintext_notification(client, live_server, measure_memory_usage, datastore_path):
+def test_check_html_document_plaintext_notification(client, live_server, measure_memory_usage):
     """When following a HTML document, notification in Plain Text format is sent correctly"""
-    import os
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html><body>some stuff<br>and more stuff<br>and even more stuff<br></body></html>")
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
@@ -699,7 +656,7 @@ def test_check_html_document_plaintext_notification(client, live_server, measure
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html><body>sxome stuff<br>and more stuff<br>lets slip this in<br>and this in<br>and even more stuff<br>&lt;tag&gt;</body></html>")
 
     time.sleep(0.1)
@@ -716,7 +673,6 @@ def test_check_html_document_plaintext_notification(client, live_server, measure
 
     assert '<tag>' in body # Should have got converted from original HTML to plaintext
     assert '(changed) some stuff\r\n' in body
-    assert 'PLACEMARKER' not in body
     assert '(into) sxome stuff\r\n' in body
     assert '(added) lets slip this in\r\n' in body
     assert '(added) and this in\r\n' in body
@@ -726,10 +682,9 @@ def test_check_html_document_plaintext_notification(client, live_server, measure
     delete_all_watches(client)
 
 
-def test_check_html_notification_with_apprise_format_is_html(client, live_server, measure_memory_usage, datastore_path):
+def test_check_html_notification_with_apprise_format_is_html(client, live_server, measure_memory_usage):
     ##  live_server_setup(live_server) # Setup on conftest per function
-    set_original_response(datastore_path=datastore_path)
-
+    set_original_response()
 
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com&format=html'
 
@@ -758,7 +713,7 @@ def test_check_html_notification_with_apprise_format_is_html(client, live_server
     assert b"Watch added" in res.data
 
     wait_for_all_checks(client)
-    set_longer_modified_response(datastore_path=datastore_path)
+    set_longer_modified_response()
     time.sleep(2)
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -786,6 +741,7 @@ def test_check_html_notification_with_apprise_format_is_html(client, live_server
     text_content = text_part.get_content()
     assert '(added) So let\'s see what happens.\r\n' in text_content  # The plaintext part
     assert 'fallback-body\r\n' in text_content  # The plaintext part
+    assert CUSTOM_LINEBREAK_PLACEHOLDER not in text_content
 
     # Second part should be text/html
     html_part = parts[1]
@@ -794,4 +750,5 @@ def test_check_html_notification_with_apprise_format_is_html(client, live_server
     assert 'some text<br>' in html_content  # We converted \n from the notification body
     assert 'fallback-body<br>' in html_content  # kept the original <br>
     assert '(added) So let\'s see what happens.<br>' in html_content  # the html part
+    assert CUSTOM_LINEBREAK_PLACEHOLDER not in html_content
     delete_all_watches(client)

changedetectionio/tests/test_access_control.py

@@ -2,7 +2,7 @@ from .util import live_server_setup, wait_for_all_checks
 from flask import url_for
 import time
 
-def test_check_access_control(app, client, live_server, measure_memory_usage, datastore_path):
+def test_check_access_control(app, client, live_server, measure_memory_usage):
     # Still doesnt work, but this is closer.
    #  live_server_setup(live_server) # Setup on conftest per function
 

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import os.path
-import os
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output, delete_all_watches
@@ -10,7 +9,7 @@ import time
 from ..diff import ADDED_PLACEMARKER_OPEN
 
 
-def set_original(datastore_path, excluding=None, add_line=None):
+def set_original(excluding=None, add_line=None):
     test_return_data = """<html>
      <body>
      <p>Some initial text</p>
@@ -36,16 +35,16 @@ def set_original(datastore_path, excluding=None, add_line=None):
 
         test_return_data = output
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
-# def test_setup(client, live_server, measure_memory_usage, datastore_path):
+# def test_setup(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage, datastore_path):
+def test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage):
 
     # Give the endpoint time to spin up
-    set_original(datastore_path=datastore_path)
+    set_original()
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
@@ -65,10 +64,9 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
               "time_between_check_use_default": "y"},
         follow_redirects=True
     )
-
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
-    set_original(excluding='Something irrelevant', datastore_path=datastore_path)
+    set_original(excluding='Something irrelevant')
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -79,7 +77,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     assert b'has-unread-changes' not in res.data
 
     # The trigger line is REMOVED,  this should trigger
-    set_original(excluding='The golden line', datastore_path=datastore_path)
+    set_original(excluding='The golden line')
 
     # Check in the processor here what's going on, its triggering empty-reply and no change.
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -94,7 +92,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     time.sleep(0.2)
 
     time.sleep(1)
-    set_original(excluding=None, datastore_path=datastore_path)
+    set_original(excluding=None)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     time.sleep(1)
@@ -102,7 +100,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     assert b'has-unread-changes' not in res.data
 
     # Remove it again, and we should get a trigger
-    set_original(excluding='The golden line', datastore_path=datastore_path)
+    set_original(excluding='The golden line')
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -111,7 +109,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory
     delete_all_watches(client)
 
 
-def test_check_add_line_contains_trigger(client, live_server, measure_memory_usage, datastore_path):
+def test_check_add_line_contains_trigger(client, live_server, measure_memory_usage):
     
     delete_all_watches(client)
     time.sleep(1)
@@ -134,7 +132,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     )
     assert b'Settings updated' in res.data
 
-    set_original(datastore_path=datastore_path)
+    set_original()
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
@@ -157,7 +155,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     )
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
-    set_original(excluding='Something irrelevant', datastore_path=datastore_path)
+    set_original(excluding='Something irrelevant')
 
     # A line thats not the trigger should not trigger anything
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -168,7 +166,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     assert b'has-unread-changes' not in res.data
 
     # The trigger line is ADDED,  this should trigger
-    set_original(add_line='<p>Oh yes please</p>', datastore_path=datastore_path)
+    set_original(add_line='<p>Oh yes please</p>')
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -176,9 +174,9 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     assert b'has-unread-changes' in res.data
 
     # Takes a moment for apprise to fire
-    wait_for_notification_endpoint_output(datastore_path=datastore_path)
-    assert os.path.isfile(os.path.join(datastore_path, "notification.txt")), "Notification fired because I can see the output file"
-    with open(os.path.join(datastore_path, "notification.txt"), 'rb') as f:
+    wait_for_notification_endpoint_output()
+    assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file"
+    with open("test-datastore/notification.txt", 'rb') as f:
         response = f.read()
         assert ADDED_PLACEMARKER_OPEN.encode('utf-8') not in response #  _apply_diff_filtering shouldnt add something here
         assert b'-Oh yes please' in response

changedetectionio/tests/test_api.py

@@ -3,13 +3,12 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
-import os
 
 import json
 import uuid
 
 
-def set_original_response(datastore_path):
+def set_original_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -22,12 +21,12 @@ def set_original_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
 
-def set_modified_response(datastore_path):
+def set_modified_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -40,7 +39,7 @@ def set_modified_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
     return None
@@ -53,17 +52,17 @@ def is_valid_uuid(val):
         return False
 
 
-# def test_setup(client, live_server, measure_memory_usage, datastore_path):
+# def test_setup(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
 
-def test_api_simple(client, live_server, measure_memory_usage, datastore_path):
+def test_api_simple(client, live_server, measure_memory_usage):
     
 
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
     # Create a watch
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
 
     # Validate bad URL
     test_url = url_for('test_endpoint', _external=True )
@@ -112,7 +111,7 @@ def test_api_simple(client, live_server, measure_memory_usage, datastore_path):
     time.sleep(1)
     wait_for_all_checks(client)
 
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
     # Trigger recheck of all ?recheck_all=1
     client.get(
         url_for("createwatch", recheck_all='1'),
@@ -245,7 +244,7 @@ def test_api_simple(client, live_server, measure_memory_usage, datastore_path):
     )
     assert len(res.json) == 0, "Watch list should be empty"
 
-def test_access_denied(client, live_server, measure_memory_usage, datastore_path):
+def test_access_denied(client, live_server, measure_memory_usage):
     # `config_api_token_enabled` Should be On by default
     res = client.get(
         url_for("createwatch")
@@ -290,11 +289,11 @@ def test_access_denied(client, live_server, measure_memory_usage, datastore_path
     )
     assert b"Settings updated." in res.data
 
-def test_api_watch_PUT_update(client, live_server, measure_memory_usage, datastore_path):
+def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
 
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
     # Create a watch
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
     test_url = url_for('test_endpoint', _external=True)
 
     # Create new
@@ -399,7 +398,7 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage, datasto
     delete_all_watches(client)
 
 
-def test_api_import(client, live_server, measure_memory_usage, datastore_path):
+def test_api_import(client, live_server, measure_memory_usage):
 
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -421,7 +420,7 @@ def test_api_import(client, live_server, measure_memory_usage, datastore_path):
     res = client.get(url_for('tags.tags_overview_page'))
     assert b'import-test' in res.data
 
-def test_api_conflict_UI_password(client, live_server, measure_memory_usage, datastore_path):
+def test_api_conflict_UI_password(client, live_server, measure_memory_usage):
 
     
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
@@ -439,7 +438,7 @@ def test_api_conflict_UI_password(client, live_server, measure_memory_usage, dat
     assert b"Password protection enabled." in res.data
 
     # Create a watch
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
     test_url = url_for('test_endpoint', _external=True)
 
     # Create new

changedetectionio/tests/test_api_notifications.py

@@ -4,7 +4,7 @@ from flask import url_for
 from .util import live_server_setup
 import json
 
-def test_api_notifications_crud(client, live_server, measure_memory_usage, datastore_path):
+def test_api_notifications_crud(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 

changedetectionio/tests/test_api_openapi.py

@@ -12,7 +12,7 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 
 
-def test_openapi_validation_invalid_content_type_on_create_watch(client, live_server, measure_memory_usage, datastore_path):
+def test_openapi_validation_invalid_content_type_on_create_watch(client, live_server, measure_memory_usage):
     """Test that creating a watch with invalid content-type triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -29,7 +29,7 @@ def test_openapi_validation_invalid_content_type_on_create_watch(client, live_se
     assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
 
 
-def test_openapi_validation_missing_required_field_create_watch(client, live_server, measure_memory_usage, datastore_path):
+def test_openapi_validation_missing_required_field_create_watch(client, live_server, measure_memory_usage):
     """Test that creating a watch without required URL field triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -46,7 +46,7 @@ def test_openapi_validation_missing_required_field_create_watch(client, live_ser
     assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
 
 
-def test_openapi_validation_invalid_field_in_request_body(client, live_server, measure_memory_usage, datastore_path):
+def test_openapi_validation_invalid_field_in_request_body(client, live_server, measure_memory_usage):
     """Test that including invalid fields triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -83,7 +83,7 @@ def test_openapi_validation_invalid_field_in_request_body(client, live_server, m
     assert b"Additional properties are not allowed" in res.data, "Should contain validation error about additional properties"
 
 
-def test_openapi_validation_import_wrong_content_type(client, live_server, measure_memory_usage, datastore_path):
+def test_openapi_validation_import_wrong_content_type(client, live_server, measure_memory_usage):
     """Test that import endpoint with wrong content-type triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -100,7 +100,7 @@ def test_openapi_validation_import_wrong_content_type(client, live_server, measu
     assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
 
 
-def test_openapi_validation_import_correct_content_type_succeeds(client, live_server, measure_memory_usage, datastore_path):
+def test_openapi_validation_import_correct_content_type_succeeds(client, live_server, measure_memory_usage):
     """Test that import endpoint with correct content-type succeeds (positive test)."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -117,7 +117,7 @@ def test_openapi_validation_import_correct_content_type_succeeds(client, live_se
     assert len(res.json) == 2, "Should import 2 URLs"
 
 
-def test_openapi_validation_get_requests_bypass_validation(client, live_server, measure_memory_usage, datastore_path):
+def test_openapi_validation_get_requests_bypass_validation(client, live_server, measure_memory_usage):
     """Test that GET requests bypass OpenAPI validation entirely."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -141,7 +141,7 @@ def test_openapi_validation_get_requests_bypass_validation(client, live_server,
     assert isinstance(res.json, dict), "Should return JSON dictionary for watch list"
 
 
-def test_openapi_validation_create_tag_missing_required_title(client, live_server, measure_memory_usage, datastore_path):
+def test_openapi_validation_create_tag_missing_required_title(client, live_server, measure_memory_usage):
     """Test that creating a tag without required title triggers OpenAPI validation error."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
@@ -158,7 +158,7 @@ def test_openapi_validation_create_tag_missing_required_title(client, live_serve
     assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
 
 
-def test_openapi_validation_watch_update_allows_partial_updates(client, live_server, measure_memory_usage, datastore_path):
+def test_openapi_validation_watch_update_allows_partial_updates(client, live_server, measure_memory_usage):
     """Test that watch updates allow partial updates without requiring all fields (positive test)."""
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 

changedetectionio/tests/test_api_search.py

@@ -6,7 +6,7 @@ import time
 from .util import live_server_setup, wait_for_all_checks
 
 
-def test_api_search(client, live_server, measure_memory_usage, datastore_path):
+def test_api_search(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 

changedetectionio/tests/test_api_tags.py

@@ -5,14 +5,13 @@ from .util import live_server_setup, wait_for_all_checks, set_original_response
 import json
 import time
 
-def test_api_tags_listing(client, live_server, measure_memory_usage, datastore_path):
+def test_api_tags_listing(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
     tag_title = 'Test Tag'
 
 
-    set_original_response(datastore_path=datastore_path)
-
+    set_original_response()
 
     res = client.get(
         url_for("tags"),

changedetectionio/tests/test_auth.py

@@ -5,7 +5,7 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 
 # test pages with http://username@password:foobar.com/ work
-def test_basic_auth(client, live_server, measure_memory_usage, datastore_path):
+def test_basic_auth(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
 

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -3,10 +3,9 @@
 import time
 from flask import url_for
 from .util import live_server_setup, extract_UUID_from_client, wait_for_all_checks
-import os
 
 
-def set_response_with_ldjson(datastore_path):
+def set_response_with_ldjson():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -56,11 +55,11 @@ def set_response_with_ldjson(datastore_path):
      </html>
 """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
-def set_response_without_ldjson(datastore_path):
+def set_response_without_ldjson():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -73,17 +72,17 @@ def set_response_without_ldjson(datastore_path):
      </html>
 """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
-# def test_setup(client, live_server, measure_memory_usage, datastore_path):
+# def test_setup(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
 # actually only really used by the distll.io importer, but could be handy too
-def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage, datastore_path):
+def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage):
     
-    set_response_with_ldjson(datastore_path=datastore_path)
+    set_response_with_ldjson()
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -122,7 +121,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage
 
     ##########################################################################################
     # And we shouldnt see the offer
-    set_response_without_ldjson(datastore_path=datastore_path)
+    set_response_without_ldjson()
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -152,7 +151,7 @@ def _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_
     client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
 
 
-def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usage, datastore_path):
+def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usage):
     
     test_return_data = """
             <html>
@@ -182,7 +181,7 @@ def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usa
             <div class="yes">Some extra stuff</div>
             </body></html>
      """
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
     _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=True)
@@ -216,7 +215,7 @@ def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usa
     #         <div class="yes">Some extra stuff</div>
     #         </body></html>
     #  """
-    # with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    # with open("test-datastore/endpoint-content.txt", "w") as f:
     #     f.write(test_return_data)
     #
     # _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=False)

changedetectionio/tests/test_backend.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-import os
 
 import time
 from flask import url_for
@@ -17,8 +16,8 @@ def test_inscriptus():
     assert stripped_text_from_html == 'test!\nok man'
 
 
-def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path):
-    set_original_response(datastore_path=datastore_path)
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
+    set_original_response()
    #  live_server_setup(live_server) # Setup on conftest per function
 
     # Add our URL to the import page
@@ -61,7 +60,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     assert b'foobar-detection' not in res.data
 
     # Make a change
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
 
     # Force recheck
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -122,7 +121,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
         assert b'test-endpoint' in res.data
 
     # Recheck it but only with a title change, content wasnt changed
-    set_original_response(datastore_path=datastore_path, extra_title=" and more")
+    set_original_response(extra_title=" and more")
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
@@ -168,7 +167,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
 
 # Server says its plaintext, we should always treat it as plaintext, and then if they have a filter, try to apply that
-def test_requests_timeout(client, live_server, measure_memory_usage, datastore_path):
+def test_requests_timeout(client, live_server, measure_memory_usage):
     delay = 2
     test_url = url_for('test_endpoint', delay=delay, _external=True)
 
@@ -206,7 +205,7 @@ def test_requests_timeout(client, live_server, measure_memory_usage, datastore_p
     res = client.get(url_for("watchlist.index"))
     assert b'Read timed out' not in res.data
 
-def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage, datastore_path):
+def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage):
     """
 
     https://github.com/dgtlmoon/changedetection.io/issues/3434
@@ -221,7 +220,7 @@ def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage, d
     :param measure_memory_usage:
     :return:
     """
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("""some random text that should be split by line
 and not parsed with html_to_text
 this way we know that it correctly parsed as plain text
@@ -265,7 +264,7 @@ got it\r\n
     delete_all_watches(client)
 
 
-def test_standard_text_plain(client, live_server, measure_memory_usage, datastore_path):
+def test_standard_text_plain(client, live_server, measure_memory_usage):
     """
 
     https://github.com/dgtlmoon/changedetection.io/issues/3434
@@ -280,7 +279,7 @@ def test_standard_text_plain(client, live_server, measure_memory_usage, datastor
     :param measure_memory_usage:
     :return:
     """
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("""some random text that should be split by line
 and not parsed with html_to_text
 <title>Even this title should stay because we are just plain text</title>
@@ -326,9 +325,9 @@ got it\r\n
     delete_all_watches(client)
 
 # Server says its plaintext, we should always treat it as plaintext
-def test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage, datastore_path):
+def test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage):
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("""<?xml version="1.0" encoding="utf-8"?>
 <resources xmlns:tools="http://schemas.android.com/tools">
     <!--Activity and fragment titles-->
@@ -354,10 +353,10 @@ def test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage
     delete_all_watches(client)
 
 # Server says its plaintext, we should always treat it as plaintext, and then if they have a filter, try to apply that
-def test_plaintext_even_if_xml_content_and_can_apply_filters(client, live_server, measure_memory_usage, datastore_path):
+def test_plaintext_even_if_xml_content_and_can_apply_filters(client, live_server, measure_memory_usage):
 
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("""<?xml version="1.0" encoding="utf-8"?>
 <resources xmlns:tools="http://schemas.android.com/tools">
     <!--Activity and fragment titles-->

changedetectionio/tests/test_backup.py

@@ -8,11 +8,13 @@ import re
 import time
 
 
-def test_backup(client, live_server, measure_memory_usage, datastore_path):
+def test_backup(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     # Add our URL to the import page
     res = client.post(
@@ -29,7 +31,7 @@ def test_backup(client, live_server, measure_memory_usage, datastore_path):
         url_for("backups.request_backup"),
         follow_redirects=True
     )
-    time.sleep(4)
+    time.sleep(2)
 
     res = client.get(
         url_for("backups.index"),

changedetectionio/tests/test_basic_socketio.py

@@ -10,12 +10,11 @@ from .util import (
 )
 from loguru import logger
 
-def run_socketio_watch_update_test(client, live_server, password_mode="", datastore_path=""):
+def run_socketio_watch_update_test(client, live_server, password_mode=""):
     """Test that the socketio emits a watch update event when content changes"""
 
     # Set up the test server
-    set_original_response(datastore_path=datastore_path)
-
+    set_original_response()
 
     # Get the SocketIO instance from the app
     from changedetectionio.flask_app import app
@@ -48,7 +47,7 @@ def run_socketio_watch_update_test(client, live_server, password_mode="", datast
     socketio_test_client.get_received()
 
     # Make a change to trigger an update
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
 
     # Force recheck
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -106,11 +105,11 @@ def run_socketio_watch_update_test(client, live_server, password_mode="", datast
     # Clean up
     client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
 
-def test_everything(live_server, client, measure_memory_usage, datastore_path):
+def test_everything(live_server, client):
 
    #  live_server_setup(live_server) # Setup on conftest per function
 
-    run_socketio_watch_update_test(password_mode="", live_server=live_server, client=client, datastore_path=datastore_path)
+    run_socketio_watch_update_test(password_mode="", live_server=live_server, client=client)
 
     ############################ Password required auth check ##############################
 
@@ -125,7 +124,7 @@ def test_everything(live_server, client, measure_memory_usage, datastore_path):
 
     assert b"Password protection enabled." in res.data
 
-    run_socketio_watch_update_test(password_mode="not logged in, should exit on connect", live_server=live_server, client=client, datastore_path=datastore_path)
+    run_socketio_watch_update_test(password_mode="not logged in, should exit on connect", live_server=live_server, client=client)
     res = client.post(
         url_for("login"),
         data={"password": "foobar"},
@@ -134,4 +133,4 @@ def test_everything(live_server, client, measure_memory_usage, datastore_path):
 
     # Yes we are correctly logged in
     assert b"LOG OUT" in res.data
-    run_socketio_watch_update_test(password_mode="should be like normal", live_server=live_server, client=client, datastore_path=datastore_path)
+    run_socketio_watch_update_test(password_mode="should be like normal", live_server=live_server, client=client)

changedetectionio/tests/test_block_while_text_present.py

@@ -4,9 +4,8 @@ import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 from changedetectionio import html_tools
-import os
 
-def set_original_ignore_response(datastore_path):
+def set_original_ignore_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -18,11 +17,11 @@ def set_original_ignore_response(datastore_path):
 
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-def set_modified_original_ignore_response(datastore_path):
+def set_modified_original_ignore_response():
     test_return_data = """<html>
        <body>
      Some NEW nice initial text<br>
@@ -37,12 +36,12 @@ def set_modified_original_ignore_response(datastore_path):
 
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
 # Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
-def set_modified_response_minus_block_text(datastore_path):
+def set_modified_response_minus_block_text():
     test_return_data = """<html>
        <body>
      Some NEW nice initial text<br>
@@ -57,16 +56,16 @@ def set_modified_response_minus_block_text(datastore_path):
 
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-def test_check_block_changedetection_text_NOT_present(client, live_server, measure_memory_usage, datastore_path):
+def test_check_block_changedetection_text_NOT_present(client, live_server, measure_memory_usage):
 
    #  live_server_setup(live_server) # Setup on conftest per function
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "out of stoCk\r\nfoobar"
-    set_original_ignore_response(datastore_path=datastore_path)
+    set_original_ignore_response()
 
 
     # Add our URL to the import page
@@ -110,7 +109,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     assert b'/test-endpoint' in res.data
 
     # The page changed, BUT the text is still there, just the rest of it changes, we should not see a change
-    set_modified_original_ignore_response(datastore_path=datastore_path)
+    set_modified_original_ignore_response()
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -124,7 +123,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
 
     # 2548
     # Going back to the ORIGINAL should NOT trigger a change
-    set_original_ignore_response(datastore_path=datastore_path)
+    set_original_ignore_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -132,11 +131,10 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
 
 
     # Now we set a change where the text is gone AND its different content, it should now trigger
-    set_modified_response_minus_block_text(datastore_path=datastore_path)
+    set_modified_response_minus_block_text()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
-
     assert b'has-unread-changes' in res.data
 
 

changedetectionio/tests/test_clone.py

@@ -3,13 +3,12 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
-import os
 
 
-def test_clone_functionality(client, live_server, measure_memory_usage, datastore_path):
+def test_clone_functionality(client, live_server, measure_memory_usage):
 
    #  live_server_setup(live_server) # Setup on conftest per function
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html><body>Some content</body></html>")
 
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_conditions.py

@@ -1,14 +1,13 @@
 #!/usr/bin/env python3
 import json
 import time
-import os
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 from ..model import CONDITIONS_MATCH_LOGIC_DEFAULT
 
 
-def set_original_response(datastore_path, number="50"):
+def set_original_response(number="50"):
     test_return_data = f"""<html>
        <body>
      <h1>Test Page for Conditions</h1>
@@ -18,10 +17,10 @@ def set_original_response(datastore_path, number="50"):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
-def set_number_in_range_response(datastore_path, number="75"):
+def set_number_in_range_response(number="75"):
     test_return_data = f"""<html>
        <body>
      <h1>Test Page for Conditions</h1>
@@ -31,10 +30,10 @@ def set_number_in_range_response(datastore_path, number="75"):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
-def set_number_out_of_range_response(datastore_path, number="150"):
+def set_number_out_of_range_response(number="150"):
     test_return_data = f"""<html>
        <body>
      <h1>Test Page for Conditions</h1>
@@ -44,18 +43,18 @@ def set_number_out_of_range_response(datastore_path, number="150"):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-# def test_setup(client, live_server, measure_memory_usage, datastore_path):
+# def test_setup(client, live_server, measure_memory_usage):
     """Test that both text and number conditions work together with AND logic."""
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def test_conditions_with_text_and_number(client, live_server, measure_memory_usage, datastore_path):
+def test_conditions_with_text_and_number(client, live_server, measure_memory_usage):
     """Test that both text and number conditions work together with AND logic."""
     
-    set_original_response(datastore_path=datastore_path, number="50")
+    set_original_response("50")
     
 
     test_url = url_for('test_endpoint', _external=True)
@@ -115,7 +114,7 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa
     wait_for_all_checks(client)
 
     # Case 1
-    set_number_in_range_response(datastore_path=datastore_path, number="70.5")
+    set_number_in_range_response("70.5")
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -130,7 +129,7 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa
     client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
     time.sleep(0.2)
 
-    set_number_out_of_range_response(datastore_path=datastore_path, number="150.5")
+    set_number_out_of_range_response("150.5")
 
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -143,9 +142,9 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa
     delete_all_watches(client)
 
 # The 'validate' button next to each rule row
-def test_condition_validate_rule_row(client, live_server, measure_memory_usage, datastore_path):
+def test_condition_validate_rule_row(client, live_server, measure_memory_usage):
 
-    set_original_response(datastore_path=datastore_path, number="50")
+    set_original_response("50")
 
     test_url = url_for('test_endpoint', _external=True)
 
@@ -204,7 +203,7 @@ def test_condition_validate_rule_row(client, live_server, measure_memory_usage,
 
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage, datastore_path):
+def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage):
     
 
     test_return_data = """<html>
@@ -217,7 +216,7 @@ def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage,
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
     # Add our URL to the import page
@@ -243,10 +242,10 @@ def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage,
     )
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_lev_conditions_plugin(client, live_server, measure_memory_usage, datastore_path):
-    # This should break..
+def test_lev_conditions_plugin(client, live_server, measure_memory_usage):
+    
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("""<html>
        <body>
      Some initial text<br>
@@ -298,7 +297,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage, datast
 
 
     ############### Now change it a LITTLE bit...
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("""<html>
        <body>
      Some initial text<br>
@@ -327,7 +326,7 @@ def test_lev_conditions_plugin(client, live_server, measure_memory_usage, datast
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     assert b'Queued 1 watch for rechecking.' in res.data

changedetectionio/tests/test_css_selector.py

@@ -3,13 +3,12 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
-import os
 
 from ..html_tools import *
 
 
 
-def set_original_response(datastore_path):
+def set_original_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -22,11 +21,11 @@ def set_original_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
-def set_modified_response(datastore_path):
+def set_modified_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -39,7 +38,7 @@ def set_modified_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
     return None
@@ -70,12 +69,12 @@ def test_include_filters_output():
 
 
 # Tests the whole stack works with the CSS Filter
-def test_check_markup_include_filters_restriction(client, live_server, measure_memory_usage, datastore_path):
+def test_check_markup_include_filters_restriction(client, live_server, measure_memory_usage):
     sleep_time_for_fetch_thread = 3
 
     include_filters = "#sametext"
 
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -106,7 +105,7 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
     #  Make a change
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -120,11 +119,11 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m
 
 
 # Tests the whole stack works with the CSS Filter
-def test_check_multiple_filters(client, live_server, measure_memory_usage, datastore_path):
+def test_check_multiple_filters(client, live_server, measure_memory_usage):
     
     include_filters = "#blob-a\r\nxpath://*[contains(@id,'blob-b')]"
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("""<html><body>
      <div id="blob-a">Blob A</div>
      <div id="blob-b">Blob B</div>
@@ -169,12 +168,12 @@ def test_check_multiple_filters(client, live_server, measure_memory_usage, datas
 # The filter exists, but did not contain anything useful
 # Mainly used when the filter contains just an IMG, this can happen when someone selects an image in the visual-selector
 # Tests fetcher can throw a "ReplyWithContentButNoText" exception after applying filter and extracting text
-def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usage, datastore_path):
+def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usage):
     
 
     include_filters = "#blob-a"
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("""<html><body>
          <div id="blob-a">
            <img src="something.jpg">
@@ -217,7 +216,7 @@ def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usa
 
     ### Just an empty selector, no image
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("""<html><body>
          <div id="blob-a">
            <!-- doo doo -->

changedetectionio/tests/test_datastore_isolation.py

@@ -1,19 +0,0 @@
-#!/usr/bin/env python3
-"""Test to verify client and live_server share the same datastore"""
-
-def test_client_and_live_server_share_datastore(client, live_server):
-    """Verify that client and live_server use the same app and datastore."""
-
-    # They should be the SAME object
-    assert client.application is live_server.app, "client.application and live_server.app should be the SAME object!"
-
-    # They should share the same datastore
-    client_datastore = client.application.config.get('DATASTORE')
-    server_datastore = live_server.app.config.get('DATASTORE')
-
-    assert client_datastore is server_datastore, \
-        f"Datastores are DIFFERENT objects! client={hex(id(client_datastore))} server={hex(id(server_datastore))}"
-
-    print(f"✓ client.application and live_server.app are the SAME object")
-    print(f"✓ Both use the same DATASTORE at {hex(id(client_datastore))}")
-    print(f"✓ Datastore path: {client_datastore.datastore_path}")

changedetectionio/tests/test_element_removal.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import time
-import os
 
 from flask import url_for
 
@@ -11,7 +10,7 @@ from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 
 
 
-def set_response_with_multiple_index(datastore_path):
+def set_response_with_multiple_index():
     data= """<!DOCTYPE html>
 <html>
 <body>
@@ -37,11 +36,11 @@ def set_response_with_multiple_index(datastore_path):
 </body>
 </html>
 """
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(data)
 
 
-def set_original_response(datastore_path):
+def set_original_response():
     test_return_data = """<html>
     <header>
     <h2>Header</h2>
@@ -66,11 +65,11 @@ def set_original_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-def set_modified_response(datastore_path):
+def set_modified_response():
     test_return_data = """<html>
     <header>
     <h2>Header changed</h2>
@@ -95,7 +94,7 @@ def set_modified_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
@@ -147,10 +146,10 @@ across multiple lines
     )
 
 
-def test_element_removal_full(client, live_server, measure_memory_usage, datastore_path):
+def test_element_removal_full(client, live_server, measure_memory_usage):
     
 
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
 
 
     # Add our URL to the import page
@@ -195,7 +194,7 @@ def test_element_removal_full(client, live_server, measure_memory_usage, datasto
     client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
 
     #  Make a change to header/footer/nav
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
 
     # Trigger a check
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -209,9 +208,9 @@ def test_element_removal_full(client, live_server, measure_memory_usage, datasto
     assert b"unviewed" not in res.data
 
 # Re #2752
-def test_element_removal_nth_offset_no_shift(client, live_server, measure_memory_usage, datastore_path):
+def test_element_removal_nth_offset_no_shift(client, live_server, measure_memory_usage):
 
-    set_response_with_multiple_index(datastore_path=datastore_path)
+    set_response_with_multiple_index()
     subtractive_selectors_data = [
 ### css style ###
 """body > table > tr:nth-child(1) > th:nth-child(2)

changedetectionio/tests/test_encoding.py

@@ -5,27 +5,26 @@ import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 import pytest
-import os
 
 
 
 
 
-def set_html_response(datastore_path):
+def set_html_response():
     test_return_data = """
 <html><body><span class="nav_second_img_text">
                   &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;铸大国重器，挺制造脊梁，致力能源未来，赋能美好生活。
                                   </span>
 </body></html>
     """
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
 
 # In the case the server does not issue a charset= or doesnt have content_type header set
-def test_check_encoding_detection(client, live_server, measure_memory_usage, datastore_path):
-    set_html_response(datastore_path=datastore_path)
+def test_check_encoding_detection(client, live_server, measure_memory_usage):
+    set_html_response()
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', content_type="text/html", _external=True)
@@ -52,8 +51,8 @@ def test_check_encoding_detection(client, live_server, measure_memory_usage, dat
 
 
 # In the case the server does not issue a charset= or doesnt have content_type header set
-def test_check_encoding_detection_missing_content_type_header(client, live_server, measure_memory_usage, datastore_path):
-    set_html_response(datastore_path=datastore_path)
+def test_check_encoding_detection_missing_content_type_header(client, live_server, measure_memory_usage):
+    set_html_response()
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_errorhandling.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import time
-import os
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
@@ -9,9 +8,9 @@ from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 
 
 
-def _runner_test_http_errors(client, live_server, http_code, expected_text, datastore_path):
+def _runner_test_http_errors(client, live_server, http_code, expected_text):
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("Now you going to get a {} error code\n".format(http_code))
 
 
@@ -47,15 +46,17 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text, data
     delete_all_watches(client)
 
 
-def test_http_error_handler(client, live_server, measure_memory_usage, datastore_path):
-    _runner_test_http_errors(client, live_server, 403, 'Access denied', datastore_path=datastore_path)
-    _runner_test_http_errors(client, live_server, 404, 'Page not found', datastore_path=datastore_path)
-    _runner_test_http_errors(client, live_server, 500, '(Internal server error) received', datastore_path=datastore_path)
-    _runner_test_http_errors(client, live_server, 400, 'Error - Request returned a HTTP error code 400', datastore_path=datastore_path)
+def test_http_error_handler(client, live_server, measure_memory_usage):
+    _runner_test_http_errors(client, live_server, 403, 'Access denied')
+    _runner_test_http_errors(client, live_server, 404, 'Page not found')
+    _runner_test_http_errors(client, live_server, 500, '(Internal server error) received')
+    _runner_test_http_errors(client, live_server, 400, 'Error - Request returned a HTTP error code 400')
     delete_all_watches(client)
 
 # Just to be sure error text is properly handled
-def test_DNS_errors(client, live_server, measure_memory_usage, datastore_path):
+def test_DNS_errors(client, live_server, measure_memory_usage):
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     # Add our URL to the import page
     res = client.post(
@@ -83,9 +84,12 @@ def test_DNS_errors(client, live_server, measure_memory_usage, datastore_path):
     delete_all_watches(client)
 
 # Re 1513
-def test_low_level_errors_clear_correctly(client, live_server, measure_memory_usage, datastore_path):
+def test_low_level_errors_clear_correctly(client, live_server, measure_memory_usage):
+    
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html><body><div id=here>Hello world</div></body></html>")
 
     # Add our URL to the import page

changedetectionio/tests/test_extract_csv.py

@@ -4,15 +4,14 @@ import time
 from flask import url_for
 from urllib.request import urlopen
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
-import os
 
 sleep_time_for_fetch_thread = 3
 
 
 
-def test_check_extract_text_from_diff(client, live_server, measure_memory_usage, datastore_path):
+def test_check_extract_text_from_diff(client, live_server, measure_memory_usage):
     import time
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("Now it's {} seconds since epoch, time flies!".format(str(time.time())))
 
    #  live_server_setup(live_server) # Setup on conftest per function
@@ -34,7 +33,7 @@ def test_check_extract_text_from_diff(client, live_server, measure_memory_usage,
         # Give the thread time to pick it up
         print("Bumping snapshot and checking.. ", n)
         last_date = str(time.time())
-        with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+        with open("test-datastore/endpoint-content.txt", "w") as f:
             f.write("Now it's {} seconds since epoch, time flies!".format(last_date))
 
         client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_extract_regex.py

@@ -3,12 +3,11 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
-import os
 
 from ..html_tools import *
 
 
-def set_original_response(datastore_path):
+def set_original_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -21,12 +20,12 @@ def set_original_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
 
-def set_modified_response(datastore_path):
+def set_modified_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -40,13 +39,13 @@ def set_modified_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
     return None
 
 
-def set_multiline_response(datastore_path):
+def set_multiline_response():
     test_return_data = """<html>
        <body>
      
@@ -62,18 +61,18 @@ def set_multiline_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
     return None
 
 
-# def test_setup(client, live_server, measure_memory_usage, datastore_path):
+# def test_setup(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def test_check_filter_multiline(client, live_server, measure_memory_usage, datastore_path):
+def test_check_filter_multiline(client, live_server, measure_memory_usage):
    ##  live_server_setup(live_server) # Setup on conftest per function
-    set_multiline_response(datastore_path=datastore_path)
+    set_multiline_response()
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -120,11 +119,11 @@ def test_check_filter_multiline(client, live_server, measure_memory_usage, datas
     # but the last one, which also says 'lines' shouldnt be here (non-greedy match checking)
     assert b'aaand something lines' not in res.data
 
-def test_check_filter_and_regex_extract(client, live_server, measure_memory_usage, datastore_path):
+def test_check_filter_and_regex_extract(client, live_server, measure_memory_usage):
     
     include_filters = ".changetext"
 
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -160,7 +159,7 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
     assert b'not at the start of the expression' not in res.data
 
     #  Make a change
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -199,7 +198,7 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag
 
 
 
-def test_regex_error_handling(client, live_server, measure_memory_usage, datastore_path):
+def test_regex_error_handling(client, live_server, measure_memory_usage):
 
     
 

changedetectionio/tests/test_filter_exist_changes.py

@@ -8,7 +8,7 @@ from .util import set_original_response, live_server_setup, wait_for_notificatio
 from changedetectionio.model import App
 
 
-def set_response_without_filter(datastore_path):
+def set_response_without_filter():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -20,12 +20,12 @@ def set_response_without_filter(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
 
-def set_response_with_filter(datastore_path):
+def set_response_with_filter():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -37,11 +37,11 @@ def set_response_with_filter(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
-def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_server, measure_memory_usage, datastore_path):
+def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_server, measure_memory_usage):
 #  Filter knowingly doesn't exist, like someone setting up a known filter to see if some cinema tickets are on sale again
 #  And the page has that filter available
 #  Then I should get a notification
@@ -50,7 +50,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     # Give the endpoint time to spin up
     time.sleep(1)
-    set_response_without_filter(datastore_path=datastore_path)
+    set_response_without_filter()
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -105,20 +105,20 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    wait_for_notification_endpoint_output(datastore_path=datastore_path)
+    wait_for_notification_endpoint_output()
 
     # Shouldn't exist, shouldn't have fired
-    assert not os.path.isfile(os.path.join(datastore_path, "notification.txt"))
+    assert not os.path.isfile("test-datastore/notification.txt")
     # Now the filter should exist
-    set_response_with_filter(datastore_path=datastore_path)
+    set_response_with_filter()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
-    wait_for_notification_endpoint_output(datastore_path=datastore_path)
+    wait_for_notification_endpoint_output()
 
-    assert os.path.isfile(os.path.join(datastore_path, "notification.txt"))
+    assert os.path.isfile("test-datastore/notification.txt")
 
-    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
+    with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
 
     assert 'Ticket now on sale' in notification
-    os.unlink(os.path.join(datastore_path, "notification.txt"))
+    os.unlink("test-datastore/notification.txt")

changedetectionio/tests/test_filter_failure_notification.py

@@ -5,7 +5,7 @@ from .util import set_original_response,  wait_for_all_checks, wait_for_notifica
 from ..notification import valid_notification_formats
 
 
-def set_response_with_filter(datastore_path):
+def set_response_with_filter():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -17,14 +17,14 @@ def set_response_with_filter(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
-def run_filter_test(client, live_server, content_filter, app_notification_format, datastore_path):
+def run_filter_test(client, live_server, content_filter, app_notification_format):
 
     # Response WITHOUT the filter ID element
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
     live_server.app.config['DATASTORE'].data['settings']['application']['notification_format'] = app_notification_format
 
     # Goto the edit page, add our ignore text
@@ -38,16 +38,10 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
         url_for("ui.form_delete", uuid="all"),
         follow_redirects=True
     )
-    notification_file = os.path.join(datastore_path, "notification.txt")
-    if os.path.isfile(notification_file):
-        os.unlink(notification_file)
+    if os.path.isfile("test-datastore/notification.txt"):
+        os.unlink("test-datastore/notification.txt")
 
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
-    res = client.get(url_for("watchlist.index"))
-
-    assert b'No website watches configured' not in res.data
-
-
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -85,7 +79,6 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
         data=watch_data,
         follow_redirects=True
     )
-
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 0, "No filter = No filter failure"
@@ -102,7 +95,7 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
     # It should have checked once so far and given this error (because we hit SAVE)
 
     wait_for_all_checks(client)
-    assert not os.path.isfile(notification_file)
+    assert not os.path.isfile("test-datastore/notification.txt")
 
     # Hitting [save] would have triggered a recheck, and we have a filter, so this would be ONE failure
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 1, "Should have been checked once"
@@ -117,20 +110,20 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
         wait_for_all_checks(client)
         res = client.get(url_for("watchlist.index"))
         assert b'Warning, no filters were found' in res.data
-        assert not os.path.isfile(notification_file)
+        assert not os.path.isfile("test-datastore/notification.txt")
         time.sleep(1)
-
+        
     assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 5
 
     time.sleep(2)
     # One more check should trigger the _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT threshold
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    wait_for_notification_endpoint_output(datastore_path=datastore_path)
+    wait_for_notification_endpoint_output()
 
     # Now it should exist and contain our "filter not found" alert
-    assert os.path.isfile(notification_file)
-    with open(notification_file, 'r') as f:
+    assert os.path.isfile("test-datastore/notification.txt")
+    with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
 
     assert 'Your configured CSS/xPath filters' in notification
@@ -153,19 +146,19 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
 
     # Remove it and prove that it doesn't trigger when not expected
     # It should register a change, but no 'filter not found'
-    os.unlink(notification_file)
-    set_response_with_filter(datastore_path)
+    os.unlink("test-datastore/notification.txt")
+    set_response_with_filter()
 
     # Try several times, it should NOT have 'filter not found'
     for i in range(0, ATTEMPT_THRESHOLD_SETTING + 2):
         client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
         wait_for_all_checks(client)
 
-    wait_for_notification_endpoint_output(datastore_path=datastore_path)
+    wait_for_notification_endpoint_output()
     # It should have sent a notification, but..
-    assert os.path.isfile(notification_file)
+    assert os.path.isfile("test-datastore/notification.txt")
     # but it should not contain the info about a failed filter (because there was none in this case)
-    with open(notification_file, 'r') as f:
+    with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
     assert not 'CSS/xPath filter was not present in the page' in notification
 
@@ -177,22 +170,22 @@ def run_filter_test(client, live_server, content_filter, app_notification_format
         url_for("ui.form_delete", uuid="all"),
         follow_redirects=True
     )
-    os.unlink(notification_file)
+    os.unlink("test-datastore/notification.txt")
 
 
-def test_check_include_filters_failure_notification(client, live_server, measure_memory_usage, datastore_path):
+def test_check_include_filters_failure_notification(client, live_server, measure_memory_usage):
     #   #  live_server_setup(live_server) # Setup on conftest per function
-    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('htmlcolor'), datastore_path=datastore_path)
+    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('htmlcolor'))
     # Check markup send conversion didnt affect plaintext preference
-    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('text'), datastore_path=datastore_path)
+    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('text'))
 
-def test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage, datastore_path):
+def test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage):
     #   #  live_server_setup(live_server) # Setup on conftest per function
-    run_filter_test(client=client, live_server=live_server, content_filter='//*[@id="nope-doesnt-exist"]', app_notification_format=valid_notification_formats.get('htmlcolor'), datastore_path=datastore_path)
+    run_filter_test(client=client, live_server=live_server, content_filter='//*[@id="nope-doesnt-exist"]', app_notification_format=valid_notification_formats.get('htmlcolor'))
 
 # Test that notification is never sent
 
-def test_basic_markup_from_text(client, live_server, measure_memory_usage, datastore_path):
+def test_basic_markup_from_text(client, live_server, measure_memory_usage):
     # Test the notification error templates convert to HTML if needed (link activate)
     from ..notification.handler import markup_text_links_to_html
     x = markup_text_links_to_html("hello https://google.com")

changedetectionio/tests/test_group.py

@@ -6,10 +6,10 @@ from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from
 import os
 
 
-# def test_setup(client, live_server, measure_memory_usage, datastore_path):
+# def test_setup(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def set_original_response(datastore_path):
+def set_original_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -20,11 +20,11 @@ def set_original_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
-def set_modified_response(datastore_path):
+def set_modified_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -35,13 +35,13 @@ def set_modified_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
-def test_setup_group_tag(client, live_server, measure_memory_usage, datastore_path):
+def test_setup_group_tag(client, live_server, measure_memory_usage):
     
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
 
     # Add a tag with some config, import a tag and it should roughly work
     res = client.post(
@@ -116,7 +116,7 @@ def test_setup_group_tag(client, live_server, measure_memory_usage, datastore_pa
     )
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     rss_token = extract_rss_token_from_UI(client)
@@ -129,7 +129,7 @@ def test_setup_group_tag(client, live_server, measure_memory_usage, datastore_pa
     assert b"first-imported=1" in res.data
     delete_all_watches(client)
 
-def test_tag_import_singular(client, live_server, measure_memory_usage, datastore_path):
+def test_tag_import_singular(client, live_server, measure_memory_usage):
     
 
     test_url = url_for('test_endpoint', _external=True)
@@ -148,7 +148,7 @@ def test_tag_import_singular(client, live_server, measure_memory_usage, datastor
     assert res.data.count(b'test-tag') == 1
     delete_all_watches(client)
 
-def test_tag_add_in_ui(client, live_server, measure_memory_usage, datastore_path):
+def test_tag_add_in_ui(client, live_server, measure_memory_usage):
     
 #
     res = client.post(
@@ -164,9 +164,9 @@ def test_tag_add_in_ui(client, live_server, measure_memory_usage, datastore_path
 
     delete_all_watches(client)
 
-def test_group_tag_notification(client, live_server, measure_memory_usage, datastore_path):
+def test_group_tag_notification(client, live_server, measure_memory_usage):
     
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
 
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -207,16 +207,16 @@ def test_group_tag_notification(client, live_server, measure_memory_usage, datas
 
     wait_for_all_checks(client)
 
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     time.sleep(3)
 
-    assert os.path.isfile(os.path.join(datastore_path, "notification.txt"))
+    assert os.path.isfile("test-datastore/notification.txt")
 
     # Verify what was sent as a notification, this file should exist
-    with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
+    with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
-    os.unlink(os.path.join(datastore_path, "notification.txt"))
+    os.unlink("test-datastore/notification.txt")
 
     # Did we see the URL that had a change, in the notification?
     # Diff was correctly executed
@@ -231,7 +231,7 @@ def test_group_tag_notification(client, live_server, measure_memory_usage, datas
     #@todo Test that each of multiple notifications with different settings
     delete_all_watches(client)
 
-def test_limit_tag_ui(client, live_server, measure_memory_usage, datastore_path):
+def test_limit_tag_ui(client, live_server, measure_memory_usage):
 
     test_url = url_for('test_random_content_endpoint', _external=True)
 
@@ -269,7 +269,7 @@ def test_limit_tag_ui(client, live_server, measure_memory_usage, datastore_path)
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
     assert b'All tags deleted' in res.data
 
-def test_clone_tag_on_import(client, live_server, measure_memory_usage, datastore_path):
+def test_clone_tag_on_import(client, live_server, measure_memory_usage):
     
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -294,7 +294,7 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage, datastor
     assert res.data.count(b'another-tag') == 3
     delete_all_watches(client)
 
-def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usage, datastore_path):
+def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usage):
     
 
     test_url = url_for('test_endpoint', _external=True)
@@ -324,7 +324,7 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
     assert b'All tags deleted' in res.data
 
-def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measure_memory_usage, datastore_path):
+def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measure_memory_usage):
 
     # Add a tag with some config, import a tag and it should roughly work
     res = client.post(
@@ -378,7 +378,7 @@ def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measu
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(d)
 
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_history_consistency.py

@@ -7,7 +7,7 @@ from flask import url_for
 from .util import wait_for_all_checks, delete_all_watches
 from urllib.parse import urlparse, parse_qs
 
-def test_consistent_history(client, live_server, measure_memory_usage, datastore_path):
+def test_consistent_history(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
     workers = int(os.getenv("FETCH_WORKERS", 10))
     r = range(1, 10+workers)
@@ -80,9 +80,9 @@ def test_consistent_history(client, live_server, measure_memory_usage, datastore
         assert '"default"' not in f.read(), "'default' probably shouldnt be here, it came from when the 'default' Watch vars were accidently being saved"
 
 
-def test_check_text_history_view(client, live_server, measure_memory_usage, datastore_path):
+def test_check_text_history_view(client, live_server, measure_memory_usage):
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html>test-one</html>")
 
     # Add our URL to the import page
@@ -94,7 +94,7 @@ def test_check_text_history_view(client, live_server, measure_memory_usage, data
     wait_for_all_checks(client)
 
     # Set second version, Make a change
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html>test-two</html>")
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -105,7 +105,7 @@ def test_check_text_history_view(client, live_server, measure_memory_usage, data
     assert b'test-two' in res.data
 
     # Set third version, Make a change
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("<html>test-three</html>")
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_ignore.py

@@ -5,9 +5,8 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
 from changedetectionio import html_tools
 from . util import  extract_UUID_from_client
-import os
 
-def set_original_ignore_response(datastore_path):
+def set_original_ignore_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -20,13 +19,13 @@ def set_original_ignore_response(datastore_path):
 
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-def test_ignore(client, live_server, measure_memory_usage, datastore_path):
+def test_ignore(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_original_ignore_response(datastore_path)
+    set_original_ignore_response()
     test_url = url_for('test_endpoint', _external=True)
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -56,9 +55,9 @@ def test_ignore(client, live_server, measure_memory_usage, datastore_path):
     assert b'csrftoken' in res.data
 
 
-def test_strip_ignore_lines(client, live_server, measure_memory_usage, datastore_path):
+def test_strip_ignore_lines(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_original_ignore_response(datastore_path)
+    set_original_ignore_response()
 
 
     # Goto the settings page, add our ignore text

changedetectionio/tests/test_ignore_text.py

@@ -4,7 +4,6 @@ import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 from changedetectionio import html_tools
-import os
 
 
 
@@ -32,7 +31,7 @@ def test_strip_text_func():
     stripped_content = html_tools.strip_ignore_text(test_content, ignore)
     assert stripped_content == "Some initial text\n\nWhich is across multiple lines\n\n\n\nSo let's see what happens."
 
-def set_original_ignore_response(datastore_path, ver_stamp="123"):
+def set_original_ignore_response(ver_stamp="123"):
     test_return_data = f"""<html>
        <body>
      Some initial text<br>
@@ -45,11 +44,11 @@ def set_original_ignore_response(datastore_path, ver_stamp="123"):
 
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-def set_modified_original_ignore_response(datastore_path, ver_stamp="123"):
+def set_modified_original_ignore_response(ver_stamp="123"):
     test_return_data = f"""<html>
        <body>
      Some NEW nice initial text<br>
@@ -64,12 +63,12 @@ def set_modified_original_ignore_response(datastore_path, ver_stamp="123"):
 
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
 # Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
-def set_modified_ignore_response(datastore_path, ver_stamp="123"):
+def set_modified_ignore_response(ver_stamp="123"):
     test_return_data = f"""<html>
        <body>
      Some initial text<br>
@@ -83,17 +82,17 @@ def set_modified_ignore_response(datastore_path, ver_stamp="123"):
 
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
 # Ignore text now just removes it entirely, is a LOT more simpler code this way
 
-def test_check_ignore_text_functionality(client, live_server, measure_memory_usage, datastore_path):
+def test_check_ignore_text_functionality(client, live_server, measure_memory_usage):
 
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
-    set_original_ignore_response(datastore_path=datastore_path)
+    set_original_ignore_response()
 
 
     # Add our URL to the import page
@@ -131,7 +130,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     assert b'/test-endpoint' in res.data
 
     #  Make a change
-    set_modified_ignore_response(datastore_path=datastore_path)
+    set_modified_ignore_response()
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -146,7 +145,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
 
 
     # Just to be sure.. set a regular modified change..
-    set_modified_original_ignore_response(datastore_path=datastore_path)
+    set_modified_original_ignore_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -163,10 +162,10 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa
     delete_all_watches(client)
 
 # When adding some ignore text, it should not trigger a change, even if something else on that line changes
-def _run_test_global_ignore(client, datastore_path, as_source=False, extra_ignore=""):
+def _run_test_global_ignore(client, as_source=False, extra_ignore=""):
     ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ\r\n"+extra_ignore
 
-    set_original_ignore_response(datastore_path=datastore_path)
+    set_original_ignore_response()
 
     # Goto the settings page, add our ignore text
     res = client.post(
@@ -223,7 +222,7 @@ def _run_test_global_ignore(client, datastore_path, as_source=False, extra_ignor
     # Make a change which includes the ignore text, it should be ignored and no 'change' triggered
     # It adds text with "ZZZZzzzz" and "ZZZZ" is in the ignore list
     # And tweaks the ver_stamp which should be picked up by global regex ignore
-    set_modified_ignore_response(ver_stamp=time.time(), datastore_path=datastore_path)
+    set_modified_ignore_response(ver_stamp=time.time())
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -237,7 +236,7 @@ def _run_test_global_ignore(client, datastore_path, as_source=False, extra_ignor
     assert b'/test-endpoint' in res.data
 
     # Just to be sure.. set a regular modified change that will trigger it
-    set_modified_original_ignore_response(datastore_path=datastore_path)
+    set_modified_original_ignore_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -245,10 +244,10 @@ def _run_test_global_ignore(client, datastore_path, as_source=False, extra_ignor
 
     delete_all_watches(client)
 
-def test_check_global_ignore_text_functionality(client, live_server, measure_memory_usage, datastore_path):
+def test_check_global_ignore_text_functionality(client, live_server, measure_memory_usage):
     
-    _run_test_global_ignore(client, as_source=False, datastore_path=datastore_path)
+    _run_test_global_ignore(client, as_source=False)
 
-def test_check_global_ignore_text_functionality_as_source(client, live_server, measure_memory_usage, datastore_path):
+def test_check_global_ignore_text_functionality_as_source(client, live_server, measure_memory_usage):
     
-    _run_test_global_ignore(client, as_source=True, extra_ignore='/\?v=\d/', datastore_path=datastore_path)
+    _run_test_global_ignore(client, as_source=True, extra_ignore='/\?v=\d/')

changedetectionio/tests/test_ignorehyperlinks.py

@@ -4,10 +4,9 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
-import os
 
 
-def set_original_ignore_response(datastore_path):
+def set_original_ignore_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -18,13 +17,13 @@ def set_original_ignore_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-# Should be the same as set_original_ignore_response(datastore_path=datastore_path) but with a different
+# Should be the same as set_original_ignore_response() but with a different
 # link
-def set_modified_ignore_response(datastore_path):
+def set_modified_ignore_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -35,10 +34,10 @@ def set_modified_ignore_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
-def test_render_anchor_tag_content_true(client, live_server, measure_memory_usage, datastore_path):
+def test_render_anchor_tag_content_true(client, live_server, measure_memory_usage):
     """Testing that the link changes are detected when
     render_anchor_tag_content setting is set to true"""
     sleep_time_for_fetch_thread = 3
@@ -47,7 +46,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
     time.sleep(1)
 
     # set original html text
-    set_original_ignore_response(datastore_path=datastore_path)
+    set_original_ignore_response()
 
     # Goto the settings page, choose to ignore links (dont select/send "application-render_anchor_tag_content")
     res = client.post(
@@ -73,7 +72,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
     # set a new html text with a modified link
-    set_modified_ignore_response(datastore_path=datastore_path)
+    set_modified_ignore_response()
     wait_for_all_checks(client)
 
     # Trigger a check

changedetectionio/tests/test_ignorestatuscode.py

@@ -3,13 +3,12 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
-import os
 
 
 
 
 
-def set_original_response(datastore_path):
+def set_original_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -20,11 +19,11 @@ def set_original_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-def set_some_changed_response(datastore_path):
+def set_some_changed_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -35,17 +34,17 @@ def set_some_changed_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-def test_normal_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage, datastore_path):
+def test_normal_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage):
 
 
     # Give the endpoint time to spin up
     time.sleep(1)
 
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
 
     # Goto the settings page, add our ignore text
     res = client.post(
@@ -66,7 +65,7 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me
 
     wait_for_all_checks(client)
 
-    set_some_changed_response(datastore_path=datastore_path)
+    set_some_changed_response()
     wait_for_all_checks(client)
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -81,10 +80,10 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me
 
 
 # Tests the whole stack works with staus codes ignored
-def test_403_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage, datastore_path):
+def test_403_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage):
     sleep_time_for_fetch_thread = 3
 
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -110,7 +109,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server, measu
     wait_for_all_checks(client)
 
     #  Make a change
-    set_some_changed_response(datastore_path=datastore_path)
+    set_some_changed_response()
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_ignorewhitespace.py

@@ -3,13 +3,12 @@
 import time
 from flask import url_for
 from . util import live_server_setup
-import os
 
 
 
 
-# Should be the same as set_original_ignore_response(datastore_path=datastore_path) but with a little more whitespacing
-def set_original_ignore_response_but_with_whitespace(datastore_path):
+# Should be the same as set_original_ignore_response() but with a little more whitespacing
+def set_original_ignore_response_but_with_whitespace():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -27,11 +26,11 @@ def set_original_ignore_response_but_with_whitespace(datastore_path):
      </html>
 
     """
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-def set_original_ignore_response(datastore_path):
+def set_original_ignore_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -43,19 +42,19 @@ def set_original_ignore_response(datastore_path):
 
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_check_ignore_whitespace(client, live_server, measure_memory_usage, datastore_path):
+def test_check_ignore_whitespace(client, live_server, measure_memory_usage):
     sleep_time_for_fetch_thread = 3
 
     # Give the endpoint time to spin up
     time.sleep(1)
 
-    set_original_ignore_response(datastore_path=datastore_path)
+    set_original_ignore_response()
 
     # Goto the settings page, add our ignore text
     res = client.post(
@@ -78,7 +77,7 @@ def test_check_ignore_whitespace(client, live_server, measure_memory_usage, data
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
-    set_original_ignore_response_but_with_whitespace(datastore_path)
+    set_original_ignore_response_but_with_whitespace()
     time.sleep(sleep_time_for_fetch_thread)
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_import.py

@@ -8,10 +8,10 @@ from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 
 
-# def test_setup(client, live_server, measure_memory_usage, datastore_path):
+# def test_setup(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def test_import(client, live_server, measure_memory_usage, datastore_path):
+def test_import(client, live_server, measure_memory_usage):
     # Give the endpoint time to spin up
     wait_for_all_checks(client)
 
@@ -34,7 +34,7 @@ https://example.com tag1, other tag"""
     res = client.get( url_for("watchlist.index"))
     res = client.get( url_for("watchlist.index"))
 
-def xtest_import_skip_url(client, live_server, measure_memory_usage, datastore_path):
+def xtest_import_skip_url(client, live_server, measure_memory_usage):
 
 
     # Give the endpoint time to spin up
@@ -57,7 +57,7 @@ def xtest_import_skip_url(client, live_server, measure_memory_usage, datastore_p
     # Clear flask alerts
     res = client.get( url_for("watchlist.index"))
 
-def test_import_distillio(client, live_server, measure_memory_usage, datastore_path):
+def test_import_distillio(client, live_server, measure_memory_usage):
 
     distill_data='''
 {
@@ -123,7 +123,7 @@ def test_import_distillio(client, live_server, measure_memory_usage, datastore_p
     # Clear flask alerts
     res = client.get(url_for("watchlist.index"))
 
-def test_import_custom_xlsx(client, live_server, measure_memory_usage, datastore_path):
+def test_import_custom_xlsx(client, live_server, measure_memory_usage):
     """Test can upload a excel spreadsheet and the watches are created correctly"""
 
     
@@ -171,7 +171,7 @@ def test_import_custom_xlsx(client, live_server, measure_memory_usage, datastore
 
     delete_all_watches(client)
 
-def test_import_watchete_xlsx(client, live_server, measure_memory_usage, datastore_path):
+def test_import_watchete_xlsx(client, live_server, measure_memory_usage):
     """Test can upload a excel spreadsheet and the watches are created correctly"""
 
     

changedetectionio/tests/test_jinja2.py

@@ -7,11 +7,11 @@ from .util import live_server_setup, wait_for_all_checks
 from ..jinja2_custom import render
 
 
-# def test_setup(client, live_server, measure_memory_usage, datastore_path):
+# def test_setup(client, live_server, measure_memory_usage):
    # #  live_server_setup(live_server) # Setup on conftest per function
 
 # If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_jinja2_in_url_query(client, live_server, measure_memory_usage, datastore_path):
+def test_jinja2_in_url_query(client, live_server, measure_memory_usage):
     
 
     # Add our URL to the import page
@@ -36,7 +36,7 @@ def test_jinja2_in_url_query(client, live_server, measure_memory_usage, datastor
     assert b'date=2' in res.data
 
 # Test for issue #1493 - jinja2-time offset functionality
-def test_jinja2_time_offset_in_url_query(client, live_server, measure_memory_usage, datastore_path):
+def test_jinja2_time_offset_in_url_query(client, live_server, measure_memory_usage):
     """Test that jinja2 time offset expressions work in watch URLs (issue #1493)."""
 
     # Add our URL to the import page with time offset expression
@@ -64,21 +64,29 @@ def test_jinja2_time_offset_in_url_query(client, live_server, measure_memory_usa
     # Should not have template error
     assert b'Invalid template' not in res.data
 
-
 # https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456
-def test_jinja2_security_url_query(client, live_server, measure_memory_usage, datastore_path):
+def test_jinja2_security_url_query(client, live_server, measure_memory_usage):
+    
+
     # Add our URL to the import page
     test_url = url_for('test_return_query', _external=True)
 
-    full_url = test_url + "?date={{ ''.__class__.__mro__[1].__subclasses__()}}"
-
+    # because url_for() will URL-encode the var, but we dont here
+    full_url = "{}?{}".format(test_url,
+                              "date={{ ''.__class__.__mro__[1].__subclasses__()}}", )
     res = client.post(
         url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": full_url, "tags": "test"},
         follow_redirects=True
     )
-    assert b"Watch added" not in res.data
+    assert b"Watch added" in res.data
+    wait_for_all_checks(client)
 
+    # It should report nothing found (no new 'has-unread-changes' class)
+    res = client.get(url_for("watchlist.index"))
+    assert b'is invalid and cannot be used' in res.data
+    # Some of the spewed output from the subclasses
+    assert b'dict_values' not in res.data
 
 def test_timezone(mocker):
     """Verify that timezone is parsed."""

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -6,7 +6,6 @@ from flask import url_for
 from markupsafe import escape
 from . util import live_server_setup, wait_for_all_checks, delete_all_watches
 import pytest
-import os
 jq_support = True
 
 try:
@@ -93,7 +92,7 @@ def test_unittest_inline_extract_body():
     text = html_tools.extract_json_as_string(content, "json:$.testKey")
     assert text == '42'
 
-def set_original_ext_response(datastore_path):
+def set_original_ext_response():
     data = """
         [
         {
@@ -110,11 +109,11 @@ def set_original_ext_response(datastore_path):
     ]
         """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(data)
     return None
 
-def set_modified_ext_response(datastore_path):
+def set_modified_ext_response():
     # This should get reformatted
     data = """ [ { "isPriceLowered": false,  "status": "Sold",  "statusOrig": "sold" }, {
         "_id": "5e7b3e1fb3262d306323ff1e",
@@ -125,11 +124,11 @@ def set_modified_ext_response(datastore_path):
 ]
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(data)
     return None
 
-def set_original_response(datastore_path):
+def set_original_response():
     test_return_data = """
     {
       "employees": [
@@ -150,12 +149,12 @@ def set_original_response(datastore_path):
       "available": true
     }
     """
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
 
-def set_json_response_with_html(datastore_path):
+def set_json_response_with_html():
     test_return_data = """
     {
       "test": [
@@ -165,11 +164,11 @@ def set_json_response_with_html(datastore_path):
       ]
     }
     """
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
-def set_modified_response(datastore_path):
+def set_modified_response():
     test_return_data = """
     {
       "employees": [
@@ -191,15 +190,15 @@ def set_modified_response(datastore_path):
     }
         """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
     return None
 
-def test_check_json_without_filter(client, live_server, measure_memory_usage, datastore_path):
+def test_check_json_without_filter(client, live_server, measure_memory_usage):
     # Request a JSON document from a application/json source containing HTML
     # and be sure it doesn't get chewed up by instriptis
-    set_json_response_with_html(datastore_path=datastore_path)
+    set_json_response_with_html()
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', content_type="application/json", _external=True)
@@ -220,8 +219,8 @@ def test_check_json_without_filter(client, live_server, measure_memory_usage, da
 
     delete_all_watches(client)
 
-def check_json_filter(json_filter, client, live_server, datastore_path):
-    set_original_response(datastore_path=datastore_path)
+def check_json_filter(json_filter, client, live_server):
+    set_original_response()
 
 
     # Add our URL to the import page
@@ -241,7 +240,7 @@ def check_json_filter(json_filter, client, live_server, datastore_path):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
     #  Make a change
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -261,19 +260,19 @@ def check_json_filter(json_filter, client, live_server, datastore_path):
 
     delete_all_watches(client)
 
-def test_check_jsonpath_filter(client, live_server, measure_memory_usage, datastore_path):
-    check_json_filter('json:boss.name', client, live_server, datastore_path=datastore_path)
+def test_check_jsonpath_filter(client, live_server, measure_memory_usage):
+    check_json_filter('json:boss.name', client, live_server)
 
-def test_check_jq_filter(client, live_server, measure_memory_usage, datastore_path):
+def test_check_jq_filter(client, live_server, measure_memory_usage):
     if jq_support:
-        check_json_filter('jq:.boss.name', client, live_server, datastore_path=datastore_path)
+        check_json_filter('jq:.boss.name', client, live_server)
 
-def test_check_jqraw_filter(client, live_server, measure_memory_usage, datastore_path):
+def test_check_jqraw_filter(client, live_server, measure_memory_usage):
     if jq_support:
-        check_json_filter('jqraw:.boss.name', client, live_server, datastore_path=datastore_path)
+        check_json_filter('jqraw:.boss.name', client, live_server)
 
-def check_json_filter_bool_val(json_filter, client, live_server, datastore_path):
-    set_original_response(datastore_path=datastore_path)
+def check_json_filter_bool_val(json_filter, client, live_server):
+    set_original_response()
 
     test_url = url_for('test_endpoint', content_type="application/json", _external=True)
 
@@ -282,7 +281,7 @@ def check_json_filter_bool_val(json_filter, client, live_server, datastore_path)
     wait_for_all_checks(client)
 
     #  Make a change
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -295,24 +294,24 @@ def check_json_filter_bool_val(json_filter, client, live_server, datastore_path)
 
     delete_all_watches(client)
 
-def test_check_jsonpath_filter_bool_val(client, live_server, measure_memory_usage, datastore_path):
-    check_json_filter_bool_val("json:$['available']", client, live_server, datastore_path=datastore_path)
+def test_check_jsonpath_filter_bool_val(client, live_server, measure_memory_usage):
+    check_json_filter_bool_val("json:$['available']", client, live_server)
 
-def test_check_jq_filter_bool_val(client, live_server, measure_memory_usage, datastore_path):
+def test_check_jq_filter_bool_val(client, live_server, measure_memory_usage):
     if jq_support:
-        check_json_filter_bool_val("jq:.available", client, live_server, datastore_path=datastore_path)
+        check_json_filter_bool_val("jq:.available", client, live_server)
 
-def test_check_jqraw_filter_bool_val(client, live_server, measure_memory_usage, datastore_path):
+def test_check_jqraw_filter_bool_val(client, live_server, measure_memory_usage):
     if jq_support:
-        check_json_filter_bool_val("jq:.available", client, live_server, datastore_path=datastore_path)
+        check_json_filter_bool_val("jq:.available", client, live_server)
 
 # Re #265 - Extended JSON selector test
 # Stuff to consider here
 # - Selector should be allowed to return empty when it doesnt match (people might wait for some condition)
 # - The 'diff' tab could show the old and new content
 # - Form should let us enter a selector that doesnt (yet) match anything
-def check_json_ext_filter(json_filter, client, live_server, datastore_path):
-    set_original_ext_response(datastore_path=datastore_path)
+def check_json_ext_filter(json_filter, client, live_server):
+    set_original_ext_response()
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', content_type="application/json", _external=True)
@@ -344,7 +343,7 @@ def check_json_ext_filter(json_filter, client, live_server, datastore_path):
     # Give the thread time to pick it up
     wait_for_all_checks(client)
     #  Make a change
-    set_modified_ext_response(datastore_path=datastore_path)
+    set_modified_ext_response()
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -353,7 +352,7 @@ def check_json_ext_filter(json_filter, client, live_server, datastore_path):
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
+    snapshot_contents = watch.get_history_snapshot(dates[0])
 
     assert snapshot_contents[0] == '['
 
@@ -377,10 +376,10 @@ def check_json_ext_filter(json_filter, client, live_server, datastore_path):
 
     delete_all_watches(client)
 
-def test_ignore_json_order(client, live_server, measure_memory_usage, datastore_path):
+def test_ignore_json_order(client, live_server, measure_memory_usage):
     # A change in order shouldn't trigger a notification
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write('{"hello" : 123, "world": 123}')
 
 
@@ -391,7 +390,7 @@ def test_ignore_json_order(client, live_server, measure_memory_usage, datastore_
 
     wait_for_all_checks(client)
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write('{"world" : 123, "hello": 123}')
 
     # Trigger a check
@@ -402,7 +401,7 @@ def test_ignore_json_order(client, live_server, measure_memory_usage, datastore_
     assert b'has-unread-changes' not in res.data
 
     # Just to be sure it still works
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write('{"world" : 123, "hello": 124}')
 
     # Trigger a check
@@ -414,10 +413,10 @@ def test_ignore_json_order(client, live_server, measure_memory_usage, datastore_
 
     delete_all_watches(client)
 
-def test_correct_header_detect(client, live_server, measure_memory_usage, datastore_path):
+def test_correct_header_detect(client, live_server, measure_memory_usage):
     # Like in https://github.com/dgtlmoon/changedetection.io/pull/1593
     # Specify extra html that JSON is sometimes wrapped in - when using SockpuppetBrowser / Puppeteer / Playwrightetc
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write('<html><body>{ "world": 123, "hello" : 123}')
 
     # Add our URL to the import page
@@ -439,7 +438,7 @@ def test_correct_header_detect(client, live_server, measure_memory_usage, datast
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
+    snapshot_contents = watch.get_history_snapshot(dates[0])
 
     assert b'&#34;hello&#34;: 123,' in res.data # properly html escaped in the front end
 
@@ -451,18 +450,18 @@ def test_correct_header_detect(client, live_server, measure_memory_usage, datast
 
     delete_all_watches(client)
 
-def test_check_jsonpath_ext_filter(client, live_server, measure_memory_usage, datastore_path):
-    check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server, datastore_path=datastore_path)
+def test_check_jsonpath_ext_filter(client, live_server, measure_memory_usage):
+    check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server)
 
-def test_check_jq_ext_filter(client, live_server, measure_memory_usage, datastore_path):
+def test_check_jq_ext_filter(client, live_server, measure_memory_usage):
     if jq_support:
-        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server, datastore_path=datastore_path)
+        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
 
-def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage, datastore_path):
+def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage):
     if jq_support:
-        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server, datastore_path=datastore_path)
+        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
 
-def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage, datastore_path):
+def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage):
     from .. import html_tools
 
     # JSON string with BOM and correct double-quoted keys

changedetectionio/tests/test_live_preview.py

@@ -2,10 +2,9 @@
 
 from flask import url_for
 from changedetectionio.tests.util import live_server_setup, wait_for_all_checks, extract_UUID_from_client, delete_all_watches
-import os
 
 
-def set_response(datastore_path):
+def set_response():
 
     data = """<html>
        <body>Awesome, you made it<br>
@@ -16,12 +15,12 @@ something to trigger<br>
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(data)
 
-def test_content_filter_live_preview(client, live_server, measure_memory_usage, datastore_path):
+def test_content_filter_live_preview(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_response(datastore_path=datastore_path)
+    set_response()
 
     test_url = url_for('test_endpoint', _external=True)
 

changedetectionio/tests/test_nonrenderable_pages.py

@@ -3,10 +3,9 @@
 from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, delete_all_watches
 import time
-import os
 
 
-def set_nonrenderable_response(datastore_path):
+def set_nonrenderable_response():
     test_return_data = """<html>
     <head><title>modified head title</title></head>
     <!-- like when some angular app was broken and doesnt render or whatever -->
@@ -14,20 +13,20 @@ def set_nonrenderable_response(datastore_path):
      </body>
      </html>
     """
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     time.sleep(1)
 
     return None
 
-def set_zero_byte_response(datastore_path):
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+def set_zero_byte_response():
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write("")
     time.sleep(1)
     return None
 
-def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path):
-    set_original_response(datastore_path=datastore_path)
+def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
+    set_original_response()
    #  live_server_setup(live_server) # Setup on conftest per function
 
     # Add our URL to the import page
@@ -56,7 +55,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     )
 
     # this should not trigger a change, because no good text could be converted from the HTML
-    set_nonrenderable_response(datastore_path)
+    set_nonrenderable_response()
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
 
@@ -85,7 +84,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
               'application-fetch_backend': "html_requests"},
         follow_redirects=True
     )
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
 
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -101,7 +100,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
 
     # A totally zero byte (#2528) response should also not trigger an error
-    set_zero_byte_response(datastore_path=datastore_path)
+    set_zero_byte_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     # 2877

changedetectionio/tests/test_notification.py

@@ -21,9 +21,9 @@ from ..model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH
 
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
-def test_check_notification(client, live_server, measure_memory_usage, datastore_path):
+def test_check_notification(client, live_server, measure_memory_usage):
     
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
 
     # Re 360 - new install should have defaults set
     res = client.get(url_for("settings.settings_page"))
@@ -83,7 +83,8 @@ def test_check_notification(client, live_server, measure_memory_usage, datastore
 
 
     uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    with open(os.path.join(datastore_path, str(uuid), 'last-screenshot.png'), 'wb') as f:
+    datastore = 'test-datastore'
+    with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f:
         f.write(base64.b64decode(testimage_png))
 
     # Goto the edit page, add our ignore text
@@ -137,7 +138,7 @@ def test_check_notification(client, live_server, measure_memory_usage, datastore
 
     ## Now recheck, and it should have sent the notification
     wait_for_all_checks(client)
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -150,9 +151,9 @@ def test_check_notification(client, live_server, measure_memory_usage, datastore
 
 
     # Verify what was sent as a notification, this file should exist
-    with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
+    with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
-    os.unlink(os.path.join(datastore_path, "notification.txt"))
+    os.unlink("test-datastore/notification.txt")
 
     # Did we see the URL that had a change, in the notification?
     # Diff was correctly executed
@@ -196,18 +197,18 @@ def test_check_notification(client, live_server, measure_memory_usage, datastore
 
 
     # This should insert the {current_snapshot}
-    set_more_modified_response(datastore_path=datastore_path)
+    set_more_modified_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     time.sleep(3)
     # Verify what was sent as a notification, this file should exist
-    with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
+    with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
     assert "Ohh yeah awesome" in notification_submission
 
 
     # Prove that "content constantly being marked as Changed with no Updating causes notification" is not a thing
     # https://github.com/dgtlmoon/changedetection.io/discussions/192
-    os.unlink(os.path.join(datastore_path, "notification.txt"))
+    os.unlink("test-datastore/notification.txt")
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -216,13 +217,13 @@ def test_check_notification(client, live_server, measure_memory_usage, datastore
     wait_for_all_checks(client)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    assert os.path.exists(os.path.join(datastore_path, "notification.txt")) == False
+    assert os.path.exists("test-datastore/notification.txt") == False
 
     res = client.get(url_for("settings.notification_logs"))
     # be sure we see it in the output log
     assert b'New ChangeDetection.io Notification - ' + test_url.encode('utf-8') in res.data
 
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={
@@ -242,7 +243,7 @@ def test_check_notification(client, live_server, measure_memory_usage, datastore
     time.sleep(2)
 
     # Verify what was sent as a notification, this file should exist
-    with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
+    with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
     assert "fallback-title" in notification_submission
     assert "fallback-body" in notification_submission
@@ -253,7 +254,7 @@ def test_check_notification(client, live_server, measure_memory_usage, datastore
         follow_redirects=True
     )
 
-def test_notification_validation(client, live_server, measure_memory_usage, datastore_path):
+def test_notification_validation(client, live_server, measure_memory_usage):
 
     time.sleep(1)
 
@@ -291,7 +292,7 @@ def test_notification_validation(client, live_server, measure_memory_usage, data
     )
 
 
-def test_notification_urls_jinja2_apprise_integration(client, live_server, measure_memory_usage, datastore_path):
+def test_notification_urls_jinja2_apprise_integration(client, live_server, measure_memory_usage):
 
     #
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
@@ -313,14 +314,14 @@ def test_notification_urls_jinja2_apprise_integration(client, live_server, measu
     assert b'Settings updated' in res.data
 
 
-def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_memory_usage, datastore_path):
+def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_memory_usage):
     
 
     # test_endpoint - that sends the contents of a file
     # test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt)
 
     # CUSTOM JSON BODY CHECK for POST://
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
     test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&watch_uuid={{ watch_uuid }}&xxx={{ watch_url }}&now={% now 'Europe/London', '%Y-%m-%d' %}&+custom-header=123&+second=hello+world%20%22space%22"
 
@@ -351,7 +352,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
 
     wait_for_all_checks(client)
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
@@ -363,7 +364,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     res = client.get(url_for("watchlist.index"))
     assert b'notification-error' not in res.data
 
-    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
+    with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
         j = json.loads(x)
         assert j['url'].startswith('http://localhost')
@@ -372,8 +373,8 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
 
 
     # URL check, this will always be converted to lowercase
-    assert os.path.isfile(os.path.join(datastore_path, "notification-url.txt"))
-    with open(os.path.join(datastore_path, "notification-url.txt"), 'r') as f:
+    assert os.path.isfile("test-datastore/notification-url.txt")
+    with open("test-datastore/notification-url.txt", 'r') as f:
         notification_url = f.read()
         assert 'xxx=http' in notification_url
         # apprise style headers should be stripped
@@ -384,18 +385,18 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
         assert f'watch_uuid={watch_uuid}' in notification_url
 
 
-    with open(os.path.join(datastore_path, "notification-headers.txt"), 'r') as f:
+    with open("test-datastore/notification-headers.txt", 'r') as f:
         notification_headers = f.read()
         assert 'custom-header: 123' in notification_headers.lower()
         assert 'second: hello world "space"' in notification_headers.lower()
 
 
     # Should always be automatically detected as JSON content type even when we set it as 'Plain Text' (default)
-    assert os.path.isfile(os.path.join(datastore_path, "notification-content-type.txt"))
-    with open(os.path.join(datastore_path, "notification-content-type.txt"), 'r') as f:
+    assert os.path.isfile("test-datastore/notification-content-type.txt")
+    with open("test-datastore/notification-content-type.txt", 'r') as f:
         assert 'application/json' in f.read()
 
-    os.unlink(os.path.join(datastore_path, "notification-url.txt"))
+    os.unlink("test-datastore/notification-url.txt")
 
     client.get(
         url_for("ui.form_delete", uuid="all"),
@@ -404,15 +405,15 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
 
 
 #2510
-#@todo run it again as text, html, htmlcolor
-def test_global_send_test_notification(client, live_server, measure_memory_usage, datastore_path):
+def test_global_send_test_notification(client, live_server, measure_memory_usage):
 
-    set_original_response(datastore_path=datastore_path)
-    if os.path.isfile(os.path.join(datastore_path, "notification.txt")):
-        os.unlink(os.path.join(datastore_path, "notification.txt")) \
+    
+    set_original_response()
+    if os.path.isfile("test-datastore/notification.txt"):
+        os.unlink("test-datastore/notification.txt") \
 
     # 1995 UTF-8 content should be encoded
-    test_body = 'change detection is cool 网站监测 内容更新了 - {{diff_full}}'
+    test_body = 'change detection is cool 网站监测 内容更新了'
 
     # otherwise other settings would have already existed from previous tests in this file
     res = client.post(
@@ -450,18 +451,11 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert res.status_code != 400
     assert res.status_code != 500
 
-    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
+    with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
-        assert 'change detection is cool 网站监测 内容更新了' in x
-        if 'html' in default_notification_format:
-            # this should come from default text when in global/system mode here changedetectionio/notification_service.py
-            assert 'title="Changed into">Example text:' in x
-        else:
-            assert 'title="Changed into">Example text:' not in x
-            assert 'span' not in x
-            assert 'Example text:' in x
+        assert test_body in x
 
-    os.unlink(os.path.join(datastore_path, "notification.txt"))
+    os.unlink("test-datastore/notification.txt")
 
     ######### Test group/tag settings
     res = client.post(
@@ -476,7 +470,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     # Give apprise time to fire
     time.sleep(4)
 
-    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
+    with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
         # Should come from notification.py default handler when there is no notification body to pull from
         assert 'change detection is cool 网站监测 内容更新了' in x
@@ -516,54 +510,13 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
     assert b"Error: You must have atleast one watch configured for 'test notification' to work" in res.data
 
 
-#2510
-def test_single_send_test_notification_on_watch(client, live_server, measure_memory_usage, datastore_path):
 
-    set_original_response(datastore_path=datastore_path)
-    if os.path.isfile(os.path.join(datastore_path, "notification.txt")):
-        os.unlink(os.path.join(datastore_path, "notification.txt")) \
+def _test_color_notifications(client, notification_body_token):
 
+    set_original_response()
 
-    test_url = url_for('test_endpoint', _external=True)
-    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
-    # 1995 UTF-8 content should be encoded
-    test_body = 'change detection is cool 网站监测 内容更新了 - {{diff_full}}'
-    ######### Test global/system settings
-    res = client.post(
-        url_for("ui.ui_notification.ajax_callback_send_notification_test")+f"/{uuid}",
-        data={"notification_urls": test_notification_url,
-              "notification_body": test_body,
-              "notification_format": default_notification_format,
-              "notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
-              },
-        follow_redirects=True
-    )
-
-    assert res.status_code != 400
-    assert res.status_code != 500
-
-    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
-        x = f.read()
-        assert 'change detection is cool 网站监测 内容更新了' in x
-        if 'html' in default_notification_format:
-            # this should come from default text when in global/system mode here changedetectionio/notification_service.py
-            assert 'title="Changed into">Example text:' in x
-        else:
-            assert 'title="Changed into">Example text:' not in x
-            assert 'span' not in x
-            assert 'Example text:' in x
-
-    os.unlink(os.path.join(datastore_path, "notification.txt"))
-
-def _test_color_notifications(client, notification_body_token, datastore_path):
-
-    set_original_response(datastore_path=datastore_path)
-
-    if os.path.isfile(os.path.join(datastore_path, "notification.txt")):
-        os.unlink(os.path.join(datastore_path, "notification.txt"))
+    if os.path.isfile("test-datastore/notification.txt"):
+        os.unlink("test-datastore/notification.txt")
 
 
     test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
@@ -595,7 +548,7 @@ def _test_color_notifications(client, notification_body_token, datastore_path):
 
     wait_for_all_checks(client)
 
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
 
 
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -604,7 +557,7 @@ def _test_color_notifications(client, notification_body_token, datastore_path):
     wait_for_all_checks(client)
     time.sleep(3)
 
-    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
+    with open("test-datastore/notification.txt", 'r') as f:
         x = f.read()
         s =  f'<span style="{HTML_CHANGED_STYLE}" role="note" aria-label="Changed text" title="Changed text">Which is across multiple lines'
         assert s in x
@@ -616,7 +569,6 @@ def _test_color_notifications(client, notification_body_token, datastore_path):
     )
 
 # Just checks the format of the colour notifications was correct
-def test_html_color_notifications(client, live_server, measure_memory_usage, datastore_path):
-    _test_color_notifications(client, '{{diff}}',datastore_path=datastore_path)
-    _test_color_notifications(client, '{{diff_full}}',datastore_path=datastore_path)
-
+def test_html_color_notifications(client, live_server, measure_memory_usage):
+    _test_color_notifications(client, '{{diff}}')
+    _test_color_notifications(client, '{{diff_full}}')

changedetectionio/tests/test_notification_errors.py

@@ -4,10 +4,10 @@ from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 import logging
 
-def test_check_notification_error_handling(client, live_server, measure_memory_usage, datastore_path):
+def test_check_notification_error_handling(client, live_server, measure_memory_usage):
 
    #  live_server_setup(live_server) # Setup on conftest per function
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
 
     # Set a URL and fetch it, then set a notification URL which is going to give errors
     test_url = url_for('test_endpoint', _external=True)
@@ -19,7 +19,7 @@ def test_check_notification_error_handling(client, live_server, measure_memory_u
     assert b"Watch added" in res.data
 
     wait_for_all_checks(client)
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
 
     working_notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
     broken_notification_url = "jsons://broken-url-xxxxxxxx123/test"
@@ -73,9 +73,9 @@ def test_check_notification_error_handling(client, live_server, measure_memory_u
     assert found_name_resolution_error
 
     # And the working one, which is after the 'broken' one should still have fired
-    with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
+    with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
-    os.unlink(os.path.join(datastore_path, "notification.txt"))
+    os.unlink("test-datastore/notification.txt")
     assert 'xxxxx' in notification_submission
 
     client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_obfuscations.py

@@ -3,10 +3,9 @@
 import time
 from flask import url_for
 from .util import live_server_setup
-import os
 
 
-def set_original_ignore_response(datastore_path):
+def set_original_ignore_response():
     test_return_data = """<html>
        <body>
      <span>The price is</span><span>$<!-- -->90<!-- -->.<!-- -->74</span>
@@ -15,12 +14,12 @@ def set_original_ignore_response(datastore_path):
 
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-def test_obfuscations(client, live_server, measure_memory_usage, datastore_path):
-    set_original_ignore_response(datastore_path)
+def test_obfuscations(client, live_server, measure_memory_usage):
+    set_original_ignore_response()
    #  live_server_setup(live_server) # Setup on conftest per function
     time.sleep(1)
     # Add our URL to the import page

changedetectionio/tests/test_pdf.py

@@ -3,16 +3,15 @@
 import time
 from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
-import os
 
 
 # `subtractive_selectors` should still work in `source:` type requests
-def test_fetch_pdf(client, live_server, measure_memory_usage, datastore_path):
+def test_fetch_pdf(client, live_server, measure_memory_usage):
     import shutil
     import os
 
-    shutil.copy("tests/test.pdf", os.path.join(datastore_path, "endpoint-test.pdf"))
-    first_version_size = os.path.getsize(os.path.join(datastore_path, "endpoint-test.pdf"))
+    shutil.copy("tests/test.pdf", "test-datastore/endpoint-test.pdf")
+    first_version_size = os.path.getsize("test-datastore/endpoint-test.pdf")
 
     test_url = url_for('test_pdf_endpoint', _external=True)
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
@@ -22,7 +21,7 @@ def test_fetch_pdf(client, live_server, measure_memory_usage, datastore_path):
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
+    snapshot_contents = watch.get_history_snapshot(dates[0])
 
     # PDF header should not be there (it was converted to text)
     assert 'PDF' not in snapshot_contents
@@ -36,14 +35,14 @@ def test_fetch_pdf(client, live_server, measure_memory_usage, datastore_path):
 
     # So we know if the file changes in other ways
     import hashlib
-    original_md5 = hashlib.md5(open(os.path.join(datastore_path, "endpoint-test.pdf"), 'rb').read()).hexdigest().upper()
+    original_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
     # We should have one
     assert len(original_md5) >0
     # And it's going to be in the document
     assert f'Document checksum - {original_md5}' in snapshot_contents
 
-    shutil.copy("tests/test2.pdf", os.path.join(datastore_path, "endpoint-test.pdf"))
-    changed_md5 = hashlib.md5(open(os.path.join(datastore_path, "endpoint-test.pdf"), 'rb').read()).hexdigest().upper()
+    shutil.copy("tests/test2.pdf", "test-datastore/endpoint-test.pdf")
+    changed_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     assert b'Queued 1 watch for rechecking.' in res.data
 
@@ -75,11 +74,11 @@ def test_fetch_pdf(client, live_server, measure_memory_usage, datastore_path):
 
     dates = list(watch.history.keys())
     # new snapshot was also OK, no HTML
-    snapshot_contents = watch.get_history_snapshot(timestamp=dates[1])
+    snapshot_contents = watch.get_history_snapshot(dates[1])
     assert 'html' not in snapshot_contents.lower()
-    assert f'Original file size - {os.path.getsize(os.path.join(datastore_path, "endpoint-test.pdf"))}' in snapshot_contents
+    assert f'Original file size - {os.path.getsize("test-datastore/endpoint-test.pdf")}' in snapshot_contents
     assert f'here is a change' in snapshot_contents
-    assert os.path.getsize(os.path.join(datastore_path, "endpoint-test.pdf")) != first_version_size # And the disk change worked
+    assert os.path.getsize("test-datastore/endpoint-test.pdf") != first_version_size # And the disk change worked
 
 
     

changedetectionio/tests/test_preview_endpoints.py

@@ -3,13 +3,12 @@
 import time
 from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
-import os
 
 
 # `subtractive_selectors` should still work in `source:` type requests
-def test_fetch_pdf(client, live_server, measure_memory_usage, datastore_path):
+def test_fetch_pdf(client, live_server, measure_memory_usage):
     import shutil
-    shutil.copy("tests/test.pdf", os.path.join(datastore_path, "endpoint-test.pdf"))
+    shutil.copy("tests/test.pdf", "test-datastore/endpoint-test.pdf")
 
    #  live_server_setup(live_server) # Setup on conftest per function
     test_url = url_for('test_pdf_endpoint', _external=True)
@@ -30,14 +29,14 @@ def test_fetch_pdf(client, live_server, measure_memory_usage, datastore_path):
 
     # So we know if the file changes in other ways
     import hashlib
-    original_md5 = hashlib.md5(open(os.path.join(datastore_path, "endpoint-test.pdf"), 'rb').read()).hexdigest().upper()
+    original_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
     # We should have one
     assert len(original_md5) > 0
     # And it's going to be in the document
     assert b'Document checksum - ' + bytes(str(original_md5).encode('utf-8')) in res.data
 
-    shutil.copy("tests/test2.pdf", os.path.join(datastore_path, "endpoint-test.pdf"))
-    changed_md5 = hashlib.md5(open(os.path.join(datastore_path, "endpoint-test.pdf"), 'rb').read()).hexdigest().upper()
+    shutil.copy("tests/test2.pdf", "test-datastore/endpoint-test.pdf")
+    changed_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     assert b'Queued 1 watch for rechecking.' in res.data
 

changedetectionio/tests/test_request.py

@@ -8,7 +8,7 @@ from . util import set_original_response, set_modified_response, live_server_set
 
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
-def test_headers_in_request(client, live_server, measure_memory_usage, datastore_path):
+def test_headers_in_request(client, live_server, measure_memory_usage):
     #ve_server_setup(live_server)
     # Add our URL to the import page
     test_url = url_for('test_headers', _external=True)
@@ -76,8 +76,7 @@ def test_headers_in_request(client, live_server, measure_memory_usage, datastore
 
     delete_all_watches(client)
 
-def test_body_in_request(client, live_server, measure_memory_usage, datastore_path):
-    import os
+def test_body_in_request(client, live_server, measure_memory_usage):
 
     # Add our URL to the import page
     test_url = url_for('test_body', _external=True)
@@ -142,7 +141,7 @@ def test_body_in_request(client, live_server, measure_memory_usage, datastore_pa
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     watches_with_body = 0
-    with open(os.path.join(datastore_path, 'url-watches.json')) as f:
+    with open('test-datastore/url-watches.json') as f:
         app_struct = json.load(f)
         for uuid in app_struct['watching']:
             if app_struct['watching'][uuid]['body']==body_value:
@@ -166,8 +165,7 @@ def test_body_in_request(client, live_server, measure_memory_usage, datastore_pa
     assert b"Body must be empty when Request Method is set to GET" in res.data
     delete_all_watches(client)
 
-def test_method_in_request(client, live_server, measure_memory_usage, datastore_path):
-    import os
+def test_method_in_request(client, live_server, measure_memory_usage):
     # Add our URL to the import page
     test_url = url_for('test_method', _external=True)
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):
@@ -225,7 +223,7 @@ def test_method_in_request(client, live_server, measure_memory_usage, datastore_
     wait_for_all_checks(client)
 
     watches_with_method = 0
-    with open(os.path.join(datastore_path, 'url-watches.json')) as f:
+    with open('test-datastore/url-watches.json') as f:
         app_struct = json.load(f)
         for uuid in app_struct['watching']:
             if app_struct['watching'][uuid]['method'] == 'PATCH':
@@ -237,7 +235,7 @@ def test_method_in_request(client, live_server, measure_memory_usage, datastore_
     delete_all_watches(client)
 
 # Re #2408 - user-agent override test, also should handle case-insensitive header deduplication
-def test_ua_global_override(client, live_server, measure_memory_usage, datastore_path):
+def test_ua_global_override(client, live_server, measure_memory_usage):
     ##  live_server_setup(live_server) # Setup on conftest per function
     test_url = url_for('test_headers', _external=True)
 
@@ -288,9 +286,8 @@ def test_ua_global_override(client, live_server, measure_memory_usage, datastore
     assert b"html-requests-user-agent" not in res.data
     delete_all_watches(client)
 
-def test_headers_textfile_in_request(client, live_server, measure_memory_usage, datastore_path):
-    import os
-
+def test_headers_textfile_in_request(client, live_server, measure_memory_usage):
+    
     # Add our URL to the import page
 
     webdriver_ua = "Hello fancy webdriver UA 1.0"
@@ -346,14 +343,14 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage,
     assert b"Updated watch." in res.data
     wait_for_all_checks(client)
 
-    with open(os.path.join(datastore_path, 'headers-testtag.txt'), 'w') as f:
+    with open('test-datastore/headers-testtag.txt', 'w') as f:
         f.write("tag-header: test\r\nurl-header: http://example.com")
 
-    with open(os.path.join(datastore_path, 'headers.txt'), 'w') as f:
+    with open('test-datastore/headers.txt', 'w') as f:
         f.write("global-header: nice\r\nnext-global-header: nice\r\nurl-header-global: http://example.com/global")
 
     uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
-    with open(os.path.join(datastore_path, uuid, 'headers.txt'), 'w') as f:
+    with open(f'test-datastore/{uuid}/headers.txt', 'w') as f:
         f.write("watch-header: nice\r\nurl-header-watch: http://example.com/watch")
 
     wait_for_all_checks(client)
@@ -371,8 +368,8 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage,
     assert b"Extra headers file found and will be added to this watch" in res.data
 
     # Not needed anymore
-    os.unlink(os.path.join(datastore_path, 'headers.txt'))
-    os.unlink(os.path.join(datastore_path, 'headers-testtag.txt'))
+    os.unlink('test-datastore/headers.txt')
+    os.unlink('test-datastore/headers-testtag.txt')
 
     # The service should echo back the request verb
     res = client.get(
@@ -398,7 +395,7 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage,
     # unlink headers.txt on start/stop
     delete_all_watches(client)
 
-def test_headers_validation(client, live_server, measure_memory_usage, datastore_path):
+def test_headers_validation(client, live_server, measure_memory_usage):
     
 
     test_url = url_for('test_headers', _external=True)

changedetectionio/tests/test_restock_itemprop.py

@@ -21,7 +21,8 @@ out_of_stock_props = [
     '<script type="application/ld+json">{"@context":"http://schema.org","@type":"WebSite","url":"https://www.medimops.de/","potentialAction":{"@type":"SearchAction","target":"https://www.medimops.de/produkte-C0/?fcIsSearch=1&searchparam={searchparam}","query-input":"required name=searchparam"}}</script><script type="application/ld+json">{"@context":"http://schema.org","@type":"Product","name":"Horsetrader: Robert Sangster and the Rise and Fall of the Sport of Kings","image":"https://images2.medimops.eu/product/43a982/M00002551322-large.jpg","productID":"isbn:9780002551328","gtin13":"9780002551328","category":"Livres en langue étrangère","offers":{"@type":"Offer","priceCurrency":"EUR","price":$$PRICE$$,"itemCondition":"UsedCondition","availability":"OutOfStock"},"brand":{"@type":"Thing","name":"Patrick Robinson","url":"https://www.momox-shop.fr/,patrick-robinson/"}}</script>'
 ]
 
-def set_original_response(datastore_path, props_markup='', price="121.95"):
+def set_original_response(props_markup='', price="121.95"):
+
     props_markup=props_markup.replace('$$PRICE$$', price)
     test_return_data = f"""<html>
        <body>
@@ -35,19 +36,28 @@ def set_original_response(datastore_path, props_markup='', price="121.95"):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     time.sleep(1)
     return None
 
-def test_restock_itemprop_basic(client, live_server, measure_memory_usage, datastore_path):
+
+
+
+# def test_setup(client, live_server, measure_memory_usage):
+
+   #  live_server_setup(live_server) # Setup on conftest per function
+
+def test_restock_itemprop_basic(client, live_server, measure_memory_usage):
+
+    
 
     test_url = url_for('test_endpoint', _external=True)
 
     # By default it should enable ('in_stock_processing') == 'all_changes'
 
     for p in instock_props:
-        set_original_response(props_markup=p, datastore_path=datastore_path)
+        set_original_response(props_markup=p)
         client.post(
             url_for("ui.ui_views.form_quick_watch_add"),
             data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
@@ -63,7 +73,7 @@ def test_restock_itemprop_basic(client, live_server, measure_memory_usage, datas
 
 
     for p in out_of_stock_props:
-        set_original_response(props_markup=p, datastore_path=datastore_path)
+        set_original_response(props_markup=p)
         client.post(
             url_for("ui.ui_views.form_quick_watch_add"),
             data={"url": test_url, "tags": '', 'processor': 'restock_diff'},
@@ -76,13 +86,13 @@ def test_restock_itemprop_basic(client, live_server, measure_memory_usage, datas
 
         delete_all_watches(client)
 
-def test_itemprop_price_change(client, live_server, measure_memory_usage, datastore_path):
+def test_itemprop_price_change(client, live_server, measure_memory_usage):
     
 
     # Out of the box 'Follow price changes' should be ON
     test_url = url_for('test_endpoint', _external=True)
 
-    set_original_response(props_markup=instock_props[0], price="190.95", datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price="190.95")
     client.post(
         url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
@@ -95,7 +105,7 @@ def test_itemprop_price_change(client, live_server, measure_memory_usage, datast
     assert b'190.95' in res.data
 
     # basic price change, look for notification
-    set_original_response(props_markup=instock_props[0], price='180.45', datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price='180.45')
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -106,7 +116,7 @@ def test_itemprop_price_change(client, live_server, measure_memory_usage, datast
 
 
     # turning off price change trigger, but it should show the new price, with no change notification
-    set_original_response(props_markup=instock_props[0], price='120.45', datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price='120.45')
     res = client.post(
         url_for("ui.ui_edit.edit_page", uuid="first"),
         data={"restock_settings-follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests", "time_between_check_use_default": "y"},
@@ -122,13 +132,13 @@ def test_itemprop_price_change(client, live_server, measure_memory_usage, datast
 
     delete_all_watches(client)
 
-def _run_test_minmax_limit(client, extra_watch_edit_form, datastore_path):
+def _run_test_minmax_limit(client, extra_watch_edit_form):
 
     delete_all_watches(client)
 
     test_url = url_for('test_endpoint', _external=True)
 
-    set_original_response(props_markup=instock_props[0], price="950.95", datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price="950.95")
     client.post(
         url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
@@ -156,7 +166,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form, datastore_path):
     client.get(url_for("ui.mark_all_viewed"))
 
     # price changed to something greater than min (900), BUT less than max (1100).. should be no change
-    set_original_response(props_markup=instock_props[0], price='1000.45', datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price='1000.45')
     client.get(url_for("ui.form_watch_checknow"))
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -167,7 +177,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form, datastore_path):
     assert b'has-unread-changes' not in res.data
 
     # price changed to something LESS than min (900), SHOULD be a change
-    set_original_response(props_markup=instock_props[0], price='890.45', datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price='890.45')
 
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     assert b'Queued 1 watch for rechecking.' in res.data
@@ -180,7 +190,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form, datastore_path):
 
 
     # 2715 - Price detection (once it crosses the "lower" threshold) again with a lower price - should trigger again!
-    set_original_response(props_markup=instock_props[0], price='820.45', datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price='820.45')
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     assert b'Queued 1 watch for rechecking.' in res.data
     wait_for_all_checks(client)
@@ -190,7 +200,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form, datastore_path):
     client.get(url_for("ui.mark_all_viewed"))
 
     # price changed to something MORE than max (1100.10), SHOULD be a change
-    set_original_response(props_markup=instock_props[0], price='1890.45', datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price='1890.45')
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -201,16 +211,16 @@ def _run_test_minmax_limit(client, extra_watch_edit_form, datastore_path):
     delete_all_watches(client)
 
 
-def test_restock_itemprop_minmax(client, live_server, measure_memory_usage, datastore_path):
+def test_restock_itemprop_minmax(client, live_server, measure_memory_usage):
     
     extras = {
         "restock_settings-follow_price_changes": "y",
         "restock_settings-price_change_min": 900.0,
         "restock_settings-price_change_max": 1100.10
     }
-    _run_test_minmax_limit(client, extra_watch_edit_form=extras, datastore_path=datastore_path)
+    _run_test_minmax_limit(client, extra_watch_edit_form=extras)
 
-def test_restock_itemprop_with_tag(client, live_server, measure_memory_usage, datastore_path):
+def test_restock_itemprop_with_tag(client, live_server, measure_memory_usage):
     
 
     res = client.post(
@@ -235,18 +245,18 @@ def test_restock_itemprop_with_tag(client, live_server, measure_memory_usage, da
         "tags": "test-tag"
     }
 
-    _run_test_minmax_limit(client, extra_watch_edit_form=extras,datastore_path=datastore_path)
+    _run_test_minmax_limit(client, extra_watch_edit_form=extras)
 
 
 
-def test_itemprop_percent_threshold(client, live_server, measure_memory_usage, datastore_path):
+def test_itemprop_percent_threshold(client, live_server, measure_memory_usage):
     
 
     delete_all_watches(client)
 
     test_url = url_for('test_endpoint', _external=True)
 
-    set_original_response(props_markup=instock_props[0], price="950.95", datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price="950.95")
     client.post(
         url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
@@ -273,7 +283,7 @@ def test_itemprop_percent_threshold(client, live_server, measure_memory_usage, d
 
 
     # Basic change should not trigger
-    set_original_response(props_markup=instock_props[0], price='960.45', datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price='960.45')
     client.get(url_for("ui.form_watch_checknow"))
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -281,7 +291,7 @@ def test_itemprop_percent_threshold(client, live_server, measure_memory_usage, d
     assert b'has-unread-changes' not in res.data
 
     # Bigger INCREASE change than the threshold should trigger
-    set_original_response(props_markup=instock_props[0], price='1960.45', datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price='1960.45')
     client.get(url_for("ui.form_watch_checknow"))
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -291,7 +301,7 @@ def test_itemprop_percent_threshold(client, live_server, measure_memory_usage, d
 
     # Small decrease should NOT trigger
     client.get(url_for("ui.mark_all_viewed"))
-    set_original_response(props_markup=instock_props[0], price='1950.45', datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price='1950.45')
     client.get(url_for("ui.form_watch_checknow"))
     wait_for_all_checks(client)
     res = client.get(url_for("watchlist.index"))
@@ -305,14 +315,14 @@ def test_itemprop_percent_threshold(client, live_server, measure_memory_usage, d
 
 
 
-def test_change_with_notification_values(client, live_server, measure_memory_usage, datastore_path):
+def test_change_with_notification_values(client, live_server, measure_memory_usage):
     
 
-    if os.path.isfile(os.path.join(datastore_path, "notification.txt")):
-        os.unlink(os.path.join(datastore_path, "notification.txt"))
+    if os.path.isfile("test-datastore/notification.txt"):
+        os.unlink("test-datastore/notification.txt")
 
     test_url = url_for('test_endpoint', _external=True)
-    set_original_response(props_markup=instock_props[0], price='960.45', datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price='960.45')
 
     notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
 
@@ -353,34 +363,34 @@ def test_change_with_notification_values(client, live_server, measure_memory_usa
     assert b"Settings updated." in res.data
 
 
-    set_original_response(props_markup=instock_props[0], price='960.45', datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price='960.45')
     # A change in price, should trigger a change by default
-    set_original_response(props_markup=instock_props[0], price='1950.45', datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price='1950.45')
     client.get(url_for("ui.form_watch_checknow"))
     wait_for_all_checks(client)
-    wait_for_notification_endpoint_output(datastore_path=datastore_path)
-    assert os.path.isfile(os.path.join(datastore_path, "notification.txt")), "Notification received"
-    with open(os.path.join(datastore_path, "notification.txt"), 'r') as f:
+    wait_for_notification_endpoint_output()
+    assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
+    with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
         assert "new price 1950.45" in notification
         assert "title new price 1950.45" in notification
 
     ## Now test the "SEND TEST NOTIFICATION" is working
-    os.unlink(os.path.join(datastore_path, "notification.txt"))
+    os.unlink("test-datastore/notification.txt")
     uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
     res = client.post(url_for("ui.ui_notification.ajax_callback_send_notification_test", watch_uuid=uuid), data={}, follow_redirects=True)
     time.sleep(5)
-    assert os.path.isfile(os.path.join(datastore_path, "notification.txt")), "Notification received"
+    assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
 
 
-def test_data_sanity(client, live_server, measure_memory_usage, datastore_path):
+def test_data_sanity(client, live_server, measure_memory_usage):
     
 
     delete_all_watches(client)
 
     test_url = url_for('test_endpoint', _external=True)
     test_url2 = url_for('test_endpoint2', _external=True)
-    set_original_response(props_markup=instock_props[0], price="950.95", datastore_path=datastore_path)
+    set_original_response(props_markup=instock_props[0], price="950.95")
     client.post(
         url_for("ui.ui_views.form_quick_watch_add"),
         data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
@@ -419,7 +429,7 @@ def test_data_sanity(client, live_server, measure_memory_usage, datastore_path):
     delete_all_watches(client)
 
 # All examples should give a prive of 666.66
-def test_special_prop_examples(client, live_server, measure_memory_usage, datastore_path):
+def test_special_prop_examples(client, live_server, measure_memory_usage):
     import glob
     
 
@@ -429,7 +439,7 @@ def test_special_prop_examples(client, live_server, measure_memory_usage, datast
     assert files
     for test_example_filename in files:
         with open(test_example_filename, 'r') as example_f:
-            with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as test_f:
+            with open("test-datastore/endpoint-content.txt", "w") as test_f:
                 test_f.write(f"<html><body>{example_f.read()}</body></html>")
 
             # Now fetch it and check the price worked

changedetectionio/tests/test_rss.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python3
-import os
+
 import time
 from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
     extract_UUID_from_client, delete_all_watches
 
 
-def set_original_cdata_xml(datastore_path):
+def set_original_cdata_xml():
     test_return_data = """<rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:media="http://search.yahoo.com/mrss/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
     <channel>
     <title>Gizi</title>
@@ -45,12 +45,12 @@ def set_original_cdata_xml(datastore_path):
     </rss>
             """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
 
-def set_html_content(datastore_path, content):
+def set_html_content(content):
     test_return_data = f"""<html>
        <body>
      Some initial text<br>
@@ -62,16 +62,16 @@ def set_html_content(datastore_path, content):
     """
 
     # Write as UTF-8 encoded bytes
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "wb") as f:
+    with open("test-datastore/endpoint-content.txt", "wb") as f:
         f.write(test_return_data.encode('utf-8'))
 
-# def test_setup(client, live_server, measure_memory_usage, datastore_path):
+# def test_setup(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def test_rss_and_token(client, live_server, measure_memory_usage, datastore_path):
+def test_rss_and_token(client, live_server, measure_memory_usage):
     #   #  live_server_setup(live_server) # Setup on conftest per function
 
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
     rss_token = extract_rss_token_from_UI(client)
 
     # Add our URL to the import page
@@ -84,7 +84,7 @@ def test_rss_and_token(client, live_server, measure_memory_usage, datastore_path
     assert b"1 Imported" in res.data
 
     wait_for_all_checks(client)
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
     time.sleep(1)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
@@ -106,10 +106,10 @@ def test_rss_and_token(client, live_server, measure_memory_usage, datastore_path
 
     client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
 
-def test_basic_cdata_rss_markup(client, live_server, measure_memory_usage, datastore_path):
+def test_basic_cdata_rss_markup(client, live_server, measure_memory_usage):
     
 
-    set_original_cdata_xml(datastore_path)
+    set_original_cdata_xml()
     # Rarely do endpoints give the right header, usually just text/xml, so we check also for <rss
     # This also triggers the automatic CDATA text parser so the RSS goes back a nice content list
     test_url = url_for('test_endpoint', content_type="text/xml; charset=UTF-8", _external=True)
@@ -130,10 +130,10 @@ def test_basic_cdata_rss_markup(client, live_server, measure_memory_usage, datas
     assert b'The days of Terminator' in res.data
     delete_all_watches(client)
 
-def test_rss_xpath_filtering(client, live_server, measure_memory_usage, datastore_path):
+def test_rss_xpath_filtering(client, live_server, measure_memory_usage):
     
 
-    set_original_cdata_xml(datastore_path)
+    set_original_cdata_xml()
 
     test_url = url_for('test_endpoint', content_type="application/atom+xml; charset=UTF-8", _external=True)
 
@@ -179,7 +179,7 @@ def test_rss_xpath_filtering(client, live_server, measure_memory_usage, datastor
     delete_all_watches(client)
 
 
-def test_rss_bad_chars_breaking(client, live_server, measure_memory_usage, datastore_path):
+def test_rss_bad_chars_breaking(client, live_server, measure_memory_usage):
     """This should absolutely trigger the RSS builder to go into worst state mode
 
     - source: prefix means no html conversion (which kinda filters out the bad stuff)
@@ -190,7 +190,7 @@ def test_rss_bad_chars_breaking(client, live_server, measure_memory_usage, datas
     """
     
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         ten_kb_string = "A" * 10_000
         f.write(ten_kb_string)
 
@@ -204,7 +204,7 @@ def test_rss_bad_chars_breaking(client, live_server, measure_memory_usage, datas
     wait_for_all_checks(client)
 
     # Set the bad content
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         jpeg_bytes = "\xff\xd8\xff\xe0\x00\x10XXXXXXXX\x00\x01\x02\x00\x00\x01\x00\x01\x00\x00"  # JPEG header
         jpeg_bytes += "A" * 10_000
 

changedetectionio/tests/test_rss_reader_mode.py

@@ -1,14 +1,12 @@
 #!/usr/bin/env python3
 
 import time
-import os
-
 from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
     extract_UUID_from_client, delete_all_watches
 
 
-def set_original_cdata_xml(datastore_path):
+def set_original_cdata_xml():
     test_return_data = """<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
 <channel>
 <title>Security Bulletins on wetscale</title>
@@ -42,13 +40,13 @@ def set_original_cdata_xml(datastore_path):
     </rss>
             """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
 
-def test_rss_reader_mode(client, live_server, measure_memory_usage, datastore_path):
-    set_original_cdata_xml(datastore_path=datastore_path)
+def test_rss_reader_mode(client, live_server, measure_memory_usage):
+    set_original_cdata_xml()
 
     # Rarely do endpoints give the right header, usually just text/xml, so we check also for <rss
     # This also triggers the automatic CDATA text parser so the RSS goes back a nice content list
@@ -65,7 +63,7 @@ def test_rss_reader_mode(client, live_server, measure_memory_usage, datastore_pa
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
+    snapshot_contents = watch.get_history_snapshot(dates[0])
     assert 'Wet noodles escape' in snapshot_contents
     assert '<br>' not in snapshot_contents
     assert '&lt;' not in snapshot_contents
@@ -73,8 +71,8 @@ def test_rss_reader_mode(client, live_server, measure_memory_usage, datastore_pa
     assert 'PubDate: Thu, 07 Aug 2025 00:00:00 GMT' in snapshot_contents
     delete_all_watches(client)
 
-def test_rss_reader_mode_with_css_filters(client, live_server, measure_memory_usage, datastore_path):
-    set_original_cdata_xml(datastore_path=datastore_path)
+def test_rss_reader_mode_with_css_filters(client, live_server, measure_memory_usage):
+    set_original_cdata_xml()
 
     # Rarely do endpoints give the right header, usually just text/xml, so we check also for <rss
     # This also triggers the automatic CDATA text parser so the RSS goes back a nice content list
@@ -91,7 +89,7 @@ def test_rss_reader_mode_with_css_filters(client, live_server, measure_memory_us
 
     watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
     dates = list(watch.history.keys())
-    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
+    snapshot_contents = watch.get_history_snapshot(dates[0])
     assert 'Wet noodles escape' not in snapshot_contents
     assert '<br>' not in snapshot_contents
     assert '&lt;' not in snapshot_contents

changedetectionio/tests/test_scheduler.py

@@ -9,10 +9,10 @@ from .util import  live_server_setup, wait_for_all_checks, extract_UUID_from_cli
 from ..forms import REQUIRE_ATLEAST_ONE_TIME_PART_MESSAGE_DEFAULT, REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT
 
 
-# def test_setup(client, live_server, measure_memory_usage, datastore_path):
+# def test_setup(client, live_server, measure_memory_usage):
    #  live_server_setup(live_server) # Setup on conftest per function
 
-def test_check_basic_scheduler_functionality(client, live_server, measure_memory_usage, datastore_path):
+def test_check_basic_scheduler_functionality(client, live_server, measure_memory_usage):
     
     days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
     test_url = url_for('test_random_content_endpoint', _external=True)
@@ -90,7 +90,7 @@ def test_check_basic_scheduler_functionality(client, live_server, measure_memory
     delete_all_watches(client)
 
 
-def test_check_basic_global_scheduler_functionality(client, live_server, measure_memory_usage, datastore_path):
+def test_check_basic_global_scheduler_functionality(client, live_server, measure_memory_usage):
     
     days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
     test_url = url_for('test_random_content_endpoint', _external=True)
@@ -172,7 +172,7 @@ def test_check_basic_global_scheduler_functionality(client, live_server, measure
     delete_all_watches(client)
 
 
-def test_validation_time_interval_field(client, live_server, measure_memory_usage, datastore_path):
+def test_validation_time_interval_field(client, live_server, measure_memory_usage):
     test_url = url_for('test_endpoint', _external=True)
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_search.py

@@ -4,7 +4,7 @@ import time
 
 
 
-def test_basic_search(client, live_server, measure_memory_usage, datastore_path):
+def test_basic_search(client, live_server, measure_memory_usage):
     
 
     urls = ['https://localhost:12300?first-result=1',
@@ -37,7 +37,7 @@ def test_basic_search(client, live_server, measure_memory_usage, datastore_path)
     assert urls[1].encode('utf-8') not in res.data
 
 
-def test_search_in_tag_limit(client, live_server, measure_memory_usage, datastore_path):
+def test_search_in_tag_limit(client, live_server, measure_memory_usage):
     
 
     urls = ['https://localhost:12300?first-result=1 tag-one',

changedetectionio/tests/test_security.py

@@ -7,7 +7,7 @@ from .util import live_server_setup, wait_for_all_checks, delete_all_watches
 from .. import strtobool
 
 
-def set_original_response(datastore_path):
+def set_original_response():
     test_return_data = """<html>
     <head><title>head title</title></head>
     <body>
@@ -20,12 +20,12 @@ def set_original_response(datastore_path):
      </html>
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
-def test_bad_access(client, live_server, measure_memory_usage, datastore_path):
-
+def test_bad_access(client, live_server, measure_memory_usage):
+    
     res = client.post(
         url_for("imports.import_page"),
         data={"urls": 'https://localhost'},
@@ -48,7 +48,7 @@ def test_bad_access(client, live_server, measure_memory_usage, datastore_path):
         follow_redirects=True
     )
 
-    assert b'Watch protocol is not permitted or invalid URL format' in res.data
+    assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
 
     res = client.post(
         url_for("ui.ui_views.form_quick_watch_add"),
@@ -56,7 +56,7 @@ def test_bad_access(client, live_server, measure_memory_usage, datastore_path):
         follow_redirects=True
     )
 
-    assert b'Watch protocol is not permitted or invalid URL format' in res.data
+    assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
 
     res = client.post(
         url_for("ui.ui_views.form_quick_watch_add"),
@@ -64,7 +64,7 @@ def test_bad_access(client, live_server, measure_memory_usage, datastore_path):
         follow_redirects=True
     )
 
-    assert b'Watch protocol is not permitted or invalid URL format' in res.data
+    assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
 
 
     res = client.post(
@@ -73,15 +73,8 @@ def test_bad_access(client, live_server, measure_memory_usage, datastore_path):
         follow_redirects=True
     )
 
-    assert b'Watch protocol is not permitted or invalid URL format' in res.data
+    assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
 
-    res = client.post(
-        url_for("ui.ui_views.form_quick_watch_add"),
-        data={"url": 'https://i-wanna-xss-you.com?hereis=<script>alert(1)</script>', "tags": ''},
-        follow_redirects=True
-    )
-
-    assert b'Watch protocol is not permitted or invalid URL format' in res.data
 
 def _runner_test_various_file_slash(client, file_uri):
 
@@ -111,17 +104,17 @@ def _runner_test_various_file_slash(client, file_uri):
 
     delete_all_watches(client)
 
-def test_file_slash_access(client, live_server, measure_memory_usage, datastore_path):
+def test_file_slash_access(client, live_server, measure_memory_usage):
     
 
     # file: is NOT permitted by default, so it will be caught by ALLOW_FILE_URI check
 
     test_file_path = os.path.abspath(__file__)
     _runner_test_various_file_slash(client, file_uri=f"file://{test_file_path}")
-#    _runner_test_various_file_slash(client, file_uri=f"file:/{test_file_path}")
-#    _runner_test_various_file_slash(client, file_uri=f"file:{test_file_path}") # CVE-2024-56509
+    _runner_test_various_file_slash(client, file_uri=f"file:/{test_file_path}")
+    _runner_test_various_file_slash(client, file_uri=f"file:{test_file_path}") # CVE-2024-56509
 
-def test_xss(client, live_server, measure_memory_usage, datastore_path):
+def test_xss(client, live_server, measure_memory_usage):
     
     from changedetectionio.notification import (
         default_notification_format
@@ -142,12 +135,12 @@ def test_xss(client, live_server, measure_memory_usage, datastore_path):
     assert b"&lt;img" in res.data
 
     # Check that even forcing an update directly still doesnt get to the frontend
-    set_original_response(datastore_path=datastore_path)
+    set_original_response()
     XSS_HACK = 'javascript:alert(document.domain)'
     uuid = client.application.config.get('DATASTORE').add_watch(url=url_for('test_endpoint', _external=True))
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
 
@@ -162,8 +155,8 @@ def test_xss(client, live_server, measure_memory_usage, datastore_path):
     assert XSS_HACK.encode('utf-8') not in res.data and res.status_code == 200
 
 
-def test_xss_watch_last_error(client, live_server, measure_memory_usage, datastore_path):
-    set_original_response(datastore_path=datastore_path)
+def test_xss_watch_last_error(client, live_server, measure_memory_usage):
+    set_original_response()
     # Add our URL to the import page
     res = client.post(
         url_for("imports.import_page"),

changedetectionio/tests/test_share_watch.py

@@ -9,9 +9,8 @@ import re
 sleep_time_for_fetch_thread = 3
 
 
-def test_share_watch(client, live_server, measure_memory_usage, datastore_path):
-    set_original_response(datastore_path=datastore_path)
-
+def test_share_watch(client, live_server, measure_memory_usage):
+    set_original_response()
    #  live_server_setup(live_server) # Setup on conftest per function
 
     test_url = url_for('test_endpoint', _external=True)

changedetectionio/tests/test_source.py

@@ -9,9 +9,8 @@ sleep_time_for_fetch_thread = 3
 
 
 
-def test_check_basic_change_detection_functionality_source(client, live_server, measure_memory_usage, datastore_path):
-    set_original_response(datastore_path=datastore_path)
-
+def test_check_basic_change_detection_functionality_source(client, live_server, measure_memory_usage):
+    set_original_response()
     test_url = 'source:'+url_for('test_endpoint', _external=True)
     # Add our URL to the import page
     uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
@@ -31,7 +30,7 @@ def test_check_basic_change_detection_functionality_source(client, live_server,
     assert b'foobar-detection' in res.data
 
     # Make a change
-    set_modified_response(datastore_path=datastore_path)
+    set_modified_response()
 
     # Force recheck
     res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -53,9 +52,8 @@ def test_check_basic_change_detection_functionality_source(client, live_server,
 
 
 # `subtractive_selectors` should still work in `source:` type requests
-def test_check_ignore_elements(client, live_server, measure_memory_usage, datastore_path):
-    set_original_response(datastore_path=datastore_path)
-
+def test_check_ignore_elements(client, live_server, measure_memory_usage):
+    set_original_response()
     time.sleep(1)
     test_url = 'source:'+url_for('test_endpoint', _external=True)
     # Add our URL to the import page

changedetectionio/tests/test_trigger.py

@@ -3,10 +3,9 @@
 import time
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks
-import os
 
 
-def set_original_ignore_response(datastore_path):
+def set_original_ignore_response():
     test_return_data = """<html>
        <body>
      Some initial text<br>
@@ -18,11 +17,11 @@ def set_original_ignore_response(datastore_path):
 
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-def set_modified_original_ignore_response(datastore_path):
+def set_modified_original_ignore_response():
     test_return_data = """<html>
        <body>
      Some NEW nice initial text<br>
@@ -34,11 +33,11 @@ def set_modified_original_ignore_response(datastore_path):
 
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-def set_modified_with_trigger_text_response(datastore_path):
+def set_modified_with_trigger_text_response():
     test_return_data = """<html>
        <body>
      Some NEW nice initial text<br>
@@ -52,16 +51,16 @@ def set_modified_with_trigger_text_response(datastore_path):
 
     """
 
-    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-def test_trigger_functionality(client, live_server, measure_memory_usage, datastore_path):
+def test_trigger_functionality(client, live_server, measure_memory_usage):
 
    #  live_server_setup(live_server) # Setup on conftest per function
 
     trigger_text = "Add to cart"
-    set_original_ignore_response(datastore_path=datastore_path)
+    set_original_ignore_response()
 
 
     # Add our URL to the import page
@@ -107,7 +106,7 @@ def test_trigger_functionality(client, live_server, measure_memory_usage, datast
     assert b'/test-endpoint' in res.data
 
     #  Make a change
-    set_modified_original_ignore_response(datastore_path=datastore_path)
+    set_modified_original_ignore_response()
 
     # Trigger a check
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -118,7 +117,7 @@ def test_trigger_functionality(client, live_server, measure_memory_usage, datast
     assert b'has-unread-changes' not in res.data
 
     # Now set the content which contains the trigger text
-    set_modified_with_trigger_text_response(datastore_path=datastore_path)
+    set_modified_with_trigger_text_response()
 
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)