re #2554 - Colors should be the same as UI preview and {{diff_full}} should get the colour output too

.github/workflows/build-deb-package.yml

@@ -1,53 +0,0 @@
-name: Build Debian Package
-# Check status: systemctl status changedetection.io.service
-# Get logs: journalctl -u changedetection.io.service
-on: [push, pull_request]
-
-jobs:
-  build-deb:
-    runs-on: ubuntu-latest
-    name: Build and Package changedetection.io
-    env:
-      PACKAGE_VERSION: 0.48.5  # or load from somewhere else
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        with:
-          path: changedetection.io-${{ env.PACKAGE_VERSION }}
-
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-
-      - name: Install Build Dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y \
-            build-essential \
-            devscripts \
-            dh-virtualenv \
-            dh-python \
-            python3-all \
-            python3-all-dev \
-            python3.10 \
-            python3.10-venv \
-            python3.10-dev \
-            debhelper-compat
-
-      - name: Build the Debian Package
-        # Build it the same as the pypi way, then use the same package tar
-        run: |
-          mkdir /tmp/changedetection.io
-          python3 -m build
-          mv dist/*gz .
-          debuild -us -uc
-
-      - name: Upload Debian Package Artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: changedetection.io-deb-package
-          path: ../*.deb
-
-#@todo install and test that some basic content appears

.github/workflows/build-deb.yml

@@ -1,165 +0,0 @@
-name: Build Debian Package
-
-on:
-  push:
-    branches: [ master ]
-    tags:
-      - '*'
-  pull_request:
-    branches: [ master ]
-  workflow_dispatch:
-
-jobs:
-  build-deb:
-    runs-on: ubuntu-latest
-    container:
-      image: debian:bookworm
-
-    steps:
-    - name: Install git and dependencies
-      run: |
-        apt-get update
-        apt-get install -y git
-
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-
-    - name: Install build dependencies
-      run: |
-        apt-get install -y \
-          dpkg-dev \
-          debhelper \
-          dh-python \
-          python3-all \
-          python3-setuptools \
-          python3-pip \
-          python3-venv \
-          build-essential \
-          fakeroot
-
-    - name: Build Debian package
-      run: |
-        # Build the package
-        dpkg-buildpackage -us -uc -b
-
-        # Move built package to workspace for artifact upload
-        mkdir -p deb-packages
-        mv ../*.deb deb-packages/ || true
-        mv ../*.buildinfo deb-packages/ || true
-        mv ../*.changes deb-packages/ || true
-
-        # List what was built
-        ls -lh deb-packages/
-
-    - name: Upload .deb package
-      uses: actions/upload-artifact@v4
-      with:
-        name: changedetection-deb-package
-        path: deb-packages/*.deb
-        retention-days: 30
-
-    - name: Upload build artifacts
-      uses: actions/upload-artifact@v4
-      with:
-        name: changedetection-build-info
-        path: |
-          deb-packages/*.buildinfo
-          deb-packages/*.changes
-        retention-days: 30
-
-  test-install:
-    needs: build-deb
-    runs-on: ubuntu-latest
-    container:
-      image: debian:bookworm
-
-    steps:
-    - name: Download .deb package
-      uses: actions/download-artifact@v4
-      with:
-        name: changedetection-deb-package
-        path: ./deb-packages
-
-    - name: Install systemd and dependencies
-      run: |
-        apt-get update
-        apt-get install -y systemd systemctl
-
-    - name: Install package
-      run: |
-        ls -lh deb-packages/
-        apt-get install -y ./deb-packages/*.deb
-
-    - name: Verify installation
-      run: |
-        echo "Checking if _changedetection user exists..."
-        id _changedetection
-
-        echo "Checking if data directory exists..."
-        test -d /var/lib/changedetection
-        ls -ld /var/lib/changedetection
-
-        echo "Checking if service file exists..."
-        test -f /lib/systemd/system/changedetection.io.service
-
-        echo "Checking if binary wrapper exists..."
-        test -x /usr/bin/changedetection.io
-
-        echo "Checking if venv exists..."
-        test -d /opt/changedetection
-        test -x /opt/changedetection/bin/python3
-
-        echo "Testing binary execution..."
-        /usr/bin/changedetection.io --help || echo "Help command returned: $?"
-
-        echo ""
-        echo "✓ Installation verification passed!"
-
-    - name: Test service start and HTTP endpoint
-      run: |
-        apt-get install -y curl procps
-
-        echo "Starting changedetection.io service manually..."
-        # Start service in background as _changedetection user
-        su - _changedetection -s /bin/sh -c 'cd /var/lib/changedetection && /opt/changedetection/bin/changedetection.io -h 127.0.0.1 -p 5000 -d /var/lib/changedetection' &
-
-        echo "Waiting 5 seconds for service to start..."
-        sleep 5
-
-        echo "Checking if process is running..."
-        ps aux | grep changedetection || true
-
-        echo "Testing HTTP endpoint on localhost:5000..."
-        if curl -f -s http://127.0.0.1:5000/ | grep -i "changedetection" > /dev/null; then
-          echo "✓ Service is responding on port 5000!"
-          echo "✓ HTML content received successfully!"
-        else
-          echo "✗ Service did not respond correctly"
-          exit 1
-        fi
-
-  # Optional: Publish to GitHub Releases on tag push
-  publish-release:
-    if: startsWith(github.ref, 'refs/tags/')
-    needs: [build-deb, test-install]
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-
-    steps:
-    - name: Download .deb package
-      uses: actions/download-artifact@v4
-      with:
-        name: changedetection-deb-package
-        path: ./deb-packages
-
-    - name: Create Release
-      uses: softprops/action-gh-release@v1
-      with:
-        files: deb-packages/*.deb
-        draft: false
-        prerelease: false
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

MANIFEST.in

@@ -10,8 +10,6 @@ recursive-include changedetectionio/tests *
 prune changedetectionio/static/package-lock.json
 prune changedetectionio/static/styles/node_modules
 prune changedetectionio/static/styles/package-lock.json
-prune changedetectionio/tests/test-datastore
-
 include changedetection.py
 include requirements.txt
 include README-pip.md
@@ -20,6 +18,5 @@ global-exclude node_modules
 global-exclude venv
 
 global-exclude test-datastore
-global-exclude changedetectionio/tests/test-datastore
 global-exclude changedetection.io*dist-info
 global-exclude changedetectionio/tests/proxy_socks5/test-datastore

build-deb.sh

@@ -1,40 +0,0 @@
-#!/bin/bash
-set -e
-
-echo "========================================"
-echo "Building changedetection.io Debian package"
-echo "========================================"
-
-# Check if running on Debian-based system
-if ! command -v dpkg-buildpackage &> /dev/null; then
-    echo "Error: dpkg-buildpackage not found. Install with:"
-    echo "  sudo apt-get install dpkg-dev debhelper dh-python python3-all python3-setuptools"
-    exit 1
-fi
-
-# Clean previous builds
-echo "Cleaning previous builds..."
-rm -rf debian/.debhelper debian/changedetection.io debian/files debian/*.debhelper* debian/*.substvars
-rm -f ../changedetection.io_*.deb ../changedetection.io_*.buildinfo ../changedetection.io_*.changes
-
-# Build the package
-echo "Building package..."
-dpkg-buildpackage -us -uc -b
-
-echo ""
-echo "========================================"
-echo "Build complete!"
-echo "========================================"
-echo ""
-echo "Package created at:"
-ls -lh ../changedetection.io_*.deb
-echo ""
-echo "To install locally:"
-echo "  sudo dpkg -i ../changedetection.io_*.deb"
-echo "  sudo apt-get install -f  # If there are dependency issues"
-echo ""
-echo "To test in a clean environment:"
-echo "  docker run --rm -it -v \$(pwd)/..:/build debian:bookworm bash"
-echo "  # Inside container:"
-echo "  apt-get update && apt-get install -y /build/changedetection.io_*.deb"
-echo "  systemctl status changedetection.io"

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.48.05'
+__version__ = '0.48.01'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/__main__.py

@@ -1,4 +0,0 @@
-from . import main
-
-if __name__ == '__main__':
-    main()

changedetectionio/notification.py

@@ -23,7 +23,7 @@ valid_tokens = {
 }
 
 default_notification_format_for_watch = 'System default'
-default_notification_format = 'HTML Color'
+default_notification_format = 'Text'
 default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
 default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
 

changedetectionio/processors/__init__.py

@@ -33,8 +33,8 @@ class difference_detection_processor():
 
         url = self.watch.link
 
-        # Protect against file:, file:/, file:// access, check the real "link" without any meta "source:" etc prepended.
-        if re.search(r'^file:', url.strip(), re.IGNORECASE):
+        # Protect against file://, file:/ access, check the real "link" without any meta "source:" etc prepended.
+        if re.search(r'^file:/', url.strip(), re.IGNORECASE):
             if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
                 raise Exception(
                     "file:// type access is denied for security reasons."

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -113,8 +113,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     res = client.post(
         url_for("settings_page"),
         data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
-              # triggered_text will contain multiple lines
-              "application-notification_body": 'triggered text was -{{triggered_text}}- ### 网站监测 内容更新了 ####',
+              "application-notification_body": 'triggered text was -{{triggered_text}}- 网站监测 内容更新了',
               # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
               "application-notification_urls": test_notification_url,
               "application-minutes_between_check": 180,
@@ -172,7 +171,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file"
     with open("test-datastore/notification.txt", 'rb') as f:
         response = f.read()
-        assert b'-Oh yes please' in response
+        assert b'-Oh yes please-' in response
         assert '网站监测 内容更新了'.encode('utf-8') in response
 
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_notification.py

@@ -502,7 +502,7 @@ def _test_color_notifications(client, notification_body_token):
 
 def test_html_color_notifications(client, live_server, measure_memory_usage):
 
-    #live_server_setup(live_server)
+    live_server_setup(live_server)
     _test_color_notifications(client, '{{diff}}')
     _test_color_notifications(client, '{{diff_full}}')
     

changedetectionio/tests/test_security.py

@@ -1,7 +1,9 @@
 import os
 
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+import time
+
 from .. import strtobool
 
 
@@ -59,44 +61,54 @@ def test_bad_access(client, live_server, measure_memory_usage):
     assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
 
 
-def _runner_test_various_file_slash(client, file_uri):
+def test_file_slashslash_access(client, live_server, measure_memory_usage):
+    #live_server_setup(live_server)
 
+    test_file_path = os.path.abspath(__file__)
+
+    # file:// is permitted by default, but it will be caught by ALLOW_FILE_URI
     client.post(
         url_for("form_quick_watch_add"),
-        data={"url": file_uri, "tags": ''},
+        data={"url": f"file://{test_file_path}", "tags": ''},
         follow_redirects=True
     )
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
 
-    substrings = [b"URLs with hostname components are not permitted", b"No connection adapters were found for"]
-
-
     # If it is enabled at test time
     if strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
-        if file_uri.startswith('file:///'):
-            # This one should be the full qualified path to the file and should get the contents of this file
-            res = client.get(
-                url_for("preview_page", uuid="first"),
-                follow_redirects=True
-            )
-            assert b'_runner_test_various_file_slash' in res.data
-        else:
-            # This will give some error from requests or if it went to chrome, will give some other error :-)
-            assert any(s in res.data for s in substrings)
+        res = client.get(
+            url_for("preview_page", uuid="first"),
+            follow_redirects=True
+        )
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
+        assert b"test_file_slashslash_access" in res.data
+    else:
+        # Default should be here
+        assert b'file:// type access is denied for security reasons.' in res.data
 
 def test_file_slash_access(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)
 
-    # file: is NOT permitted by default, so it will be caught by ALLOW_FILE_URI check
-
     test_file_path = os.path.abspath(__file__)
-    _runner_test_various_file_slash(client, file_uri=f"file://{test_file_path}")
-    _runner_test_various_file_slash(client, file_uri=f"file:/{test_file_path}")
-    _runner_test_various_file_slash(client, file_uri=f"file:{test_file_path}") # CVE-2024-56509
+
+    # file:// is permitted by default, but it will be caught by ALLOW_FILE_URI
+    client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": f"file:/{test_file_path}", "tags": ''},
+        follow_redirects=True
+    )
+    wait_for_all_checks(client)
+    res = client.get(url_for("index"))
+
+    # If it is enabled at test time
+    if strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
+        # So it should permit it, but it should fall back to the 'requests' library giving an error
+        # (but means it gets passed to playwright etc)
+        assert b"URLs with hostname components are not permitted" in res.data
+    else:
+        # Default should be here
+        assert b'file:// type access is denied for security reasons.' in res.data
 
 def test_xss(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)

changedetectionio/update_worker.py

@@ -28,8 +28,6 @@ class update_worker(threading.Thread):
 
     def queue_notification_for_watch(self, notification_q, n_object, watch):
         from changedetectionio import diff
-        from changedetectionio.notification import default_notification_format_for_watch
-
         dates = []
         trigger_text = ''
 
@@ -46,10 +44,6 @@ class update_worker(threading.Thread):
         else:
             snapshot_contents = "No snapshot/history available, the watch should fetch atleast once."
 
-        # If we ended up here with "System default"
-        if n_object.get('notification_format') == default_notification_format_for_watch:
-            n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
-
         html_colour_enable = False
         # HTML needs linebreak, but MarkDown and Text can use a linefeed
         if n_object.get('notification_format') == 'HTML':

debian/changedetection-wrapper

@@ -1,3 +0,0 @@
-#!/bin/sh
-# Wrapper script to run changedetection.io from venv
-exec /opt/changedetection/bin/python3 /opt/changedetection/bin/changedetection.io "$@"

debian/changedetection.io.service

@@ -1,31 +0,0 @@
-[Unit]
-Description=changedetection.io - Website change detection and monitoring
-After=network.target
-Documentation=https://changedetection.io
-
-[Service]
-Type=simple
-User=_changedetection
-Group=_changedetection
-WorkingDirectory=/var/lib/changedetection
-Environment="DATASTORE_PATH=/var/lib/changedetection"
-Environment="HOST=127.0.0.1"
-Environment="PORT=5000"
-ExecStart=/opt/changedetection/bin/changedetection.io -h 127.0.0.1 -p 5000 -d /var/lib/changedetection
-
-# Security hardening
-NoNewPrivileges=true
-PrivateTmp=true
-ProtectSystem=strict
-ProtectHome=true
-ReadWritePaths=/var/lib/changedetection
-ProtectKernelTunables=true
-ProtectKernelModules=true
-ProtectControlGroups=true
-
-# Restart policy
-Restart=on-failure
-RestartSec=10s
-
-[Install]
-WantedBy=multi-user.target

debian/changedetection.io/lib/systemd/system/changedetection.io.service

@@ -1,13 +0,0 @@
-[Unit]
-Description=changedetection.io Service
-After=network.target
-
-[Service]
-User=changedetio
-Group=changedetio
-WorkingDirectory=/opt/changedetection.io
-ExecStart=/opt/changedetection.io/bin/python -m changedetectionio
-Restart=on-failure
-
-[Install]
-WantedBy=multi-user.target

debian/changelog

@@ -1,7 +0,0 @@
-changedetection.io (0.50.38) stable; urgency=medium
-
-  * Initial Debian package release
-  * Service runs as _changedetection user on localhost:5000
-  * Systemd integration with security hardening
-
- -- dgtlmoon <dgtlmoon@changedetection.io>  Sat, 02 Nov 2024 00:00:00 +0000

debian/compat

@@ -1 +0,0 @@
-13

debian/control

@@ -1,38 +0,0 @@
-Source: changedetection.io
-Section: web
-Priority: optional
-Maintainer: dgtlmoon <dgtlmoon@changedetection.io>
-Build-Depends: debhelper-compat (= 13),
-               dh-python,
-               python3-all (>= 3.10),
-               python3-setuptools,
-               python3-pip,
-               python3-venv
-Standards-Version: 4.6.2
-Homepage: https://changedetection.io
-Vcs-Browser: https://github.com/dgtlmoon/changedetection.io
-Vcs-Git: https://github.com/dgtlmoon/changedetection.io.git
-Rules-Requires-Root: no
-
-Package: changedetection.io
-Architecture: all
-Depends: ${misc:Depends},
-         ${python3:Depends},
-         python3 (>= 3.10),
-         python3-pip,
-         python3-venv,
-         adduser
-Description: Website change detection and monitoring service
- changedetection.io is a self-hosted open source application for monitoring
- websites for changes. It detects changes to web pages and sends alerts or
- notifications when changes are detected.
- .
- Features include:
-  * Monitor websites for visual and text changes
-  * XPath, CSS, and JSON filtering
-  * Notifications via multiple services (Apprise)
-  * API for automation
-  * Headless browser support
- .
- This package installs changedetection.io as a systemd service running on
- localhost (127.0.0.1:5000) as the unprivileged _changedetection user.

debian/debhelper-build-stamp

@@ -1 +0,0 @@
-python-changedetection.io

debian/install

@@ -1,2 +0,0 @@
-# Install wrapper script
-debian/changedetection-wrapper usr/bin/changedetection.io

debian/postinst

@@ -1,51 +0,0 @@
-#!/bin/sh
-set -e
-
-case "$1" in
-    configure)
-        # Create system user if it doesn't exist
-        if ! getent passwd _changedetection >/dev/null; then
-            adduser --system --group --home /var/lib/changedetection \
-                    --gecos "changedetection.io service" \
-                    --shell /usr/sbin/nologin \
-                    _changedetection
-        fi
-
-        # Create data directory with proper permissions
-        if [ ! -d /var/lib/changedetection ]; then
-            mkdir -p /var/lib/changedetection
-        fi
-        chown _changedetection:_changedetection /var/lib/changedetection
-        chmod 755 /var/lib/changedetection
-
-        # Reload systemd and enable service
-        if [ -d /run/systemd/system ]; then
-            systemctl daemon-reload >/dev/null || true
-            deb-systemd-helper enable changedetection.io.service >/dev/null || true
-
-            # Start service if this is a fresh install
-            if [ -z "$2" ]; then
-                echo "Starting changedetection.io service..."
-                systemctl start changedetection.io.service || true
-                echo ""
-                echo "changedetection.io has been installed and started."
-                echo "Access it at: http://127.0.0.1:5000"
-                echo ""
-                echo "To check status: systemctl status changedetection.io"
-                echo "To view logs: journalctl -u changedetection.io -f"
-            fi
-        fi
-        ;;
-
-    abort-upgrade|abort-remove|abort-deconfigure)
-        ;;
-
-    *)
-        echo "postinst called with unknown argument \`$1'" >&2
-        exit 1
-        ;;
-esac
-
-#DEBHELPER#
-
-exit 0

debian/postrm

@@ -1,29 +0,0 @@
-#!/bin/sh
-set -e
-
-case "$1" in
-    purge)
-        # Remove system user on purge
-        if getent passwd _changedetection >/dev/null; then
-            deluser --system _changedetection >/dev/null || true
-        fi
-
-        # Remove data directory on purge (with confirmation in package description)
-        if [ -d /var/lib/changedetection ]; then
-            echo "Note: Data directory /var/lib/changedetection has been preserved."
-            echo "Remove manually with: rm -rf /var/lib/changedetection"
-        fi
-        ;;
-
-    remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
-        ;;
-
-    *)
-        echo "postrm called with unknown argument \`$1'" >&2
-        exit 1
-        ;;
-esac
-
-#DEBHELPER#
-
-exit 0

debian/prerm

@@ -1,23 +0,0 @@
-#!/bin/sh
-set -e
-
-case "$1" in
-    remove|deconfigure)
-        # Stop service before removal
-        if [ -d /run/systemd/system ]; then
-            deb-systemd-invoke stop changedetection.io.service >/dev/null || true
-        fi
-        ;;
-
-    upgrade|failed-upgrade)
-        ;;
-
-    *)
-        echo "prerm called with unknown argument \`$1'" >&2
-        exit 1
-        ;;
-esac
-
-#DEBHELPER#
-
-exit 0

debian/rules

@@ -1,37 +0,0 @@
-#!/usr/bin/make -f
-
-# Debian rules file for changedetection.io
-# This uses a Python virtual environment approach for isolated dependencies
-
-export PYBUILD_NAME=changedetection.io
-export DH_VERBOSE=1
-
-# Use venv installation method
-VENV_DIR = debian/changedetection.io/opt/changedetection
-PYTHON = python3
-
-%:
-	dh $@ --with python3 --buildsystem=pybuild
-
-override_dh_auto_install:
-	# Create virtual environment
-	$(PYTHON) -m venv $(VENV_DIR)
-
-	# Install the package and dependencies into venv
-	$(VENV_DIR)/bin/pip install --upgrade pip setuptools wheel
-	$(VENV_DIR)/bin/pip install .
-
-	# Remove pip and setuptools to reduce size (optional)
-	# $(VENV_DIR)/bin/pip uninstall -y pip setuptools wheel
-
-	# Create wrapper script directory
-	install -d debian/changedetection.io/usr/bin
-
-override_dh_auto_test:
-	# Skip tests during package build
-	# Tests can be run separately with pytest
-	@echo "Skipping tests during package build"
-
-override_dh_auto_clean:
-	dh_auto_clean
-	rm -rf build dist *.egg-info

debian/source/format

@@ -1 +0,0 @@
-3.0 (native)

requirements.txt

@@ -95,5 +95,3 @@ babel
 # Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096
 greenlet >= 3.0.3
 
-# Scheduler - Windows seemed to miss a lot of default timezone info (even "UTC" !)
-tzdata