From cb30df31208eccc75b47635c189da29aedba4cba Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 22 Dec 2025 11:41:18 +0100 Subject: [PATCH] WIP --- changedetectionio/forms.py | 11 ++- changedetectionio/model/Watch.py | 29 ++++---- .../processors/image_ssim_diff/__init__.py | 10 ++- .../processors/image_ssim_diff/difference.py | 7 +- .../processors/image_ssim_diff/forms.py | 11 ++- .../image_handler/isolated_opencv.py | 7 ++ .../processors/image_ssim_diff/processor.py | 47 ++++++------- .../templates/image_ssim_diff/diff.html | 69 +++++++++---------- .../templates/image_ssim_diff/preview.html | 54 +++++++-------- 9 files changed, 123 insertions(+), 122 deletions(-) diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 6c0552672..51bcb670c 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -6,6 +6,8 @@ from wtforms.widgets.core import TimeInput from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES, RSS_TEMPLATE_TYPE_OPTIONS, RSS_TEMPLATE_HTML_DEFAULT from changedetectionio.conditions.form import ConditionFormRow from changedetectionio.notification_service import NotificationContextData +from changedetectionio.processors.image_ssim_diff import SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS, \ + SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT from changedetectionio.strtobool import strtobool from wtforms import ( @@ -996,13 +998,8 @@ class globalSettingsApplicationForm(commonSettingsForm): ignore_whitespace = BooleanField('Ignore whitespace') ssim_threshold = SelectField( 'Default Screenshot Comparison Sensitivity', - choices=[ - ('0.75', 'Low sensitivity (only major changes)'), - ('0.85', 'Medium sensitivity (moderate changes)'), - ('0.96', 'High sensitivity (small changes)'), - ('0.999', 'Very high sensitivity (any change)') - ], - default='0.96' + choices=SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS, + default=SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT ) password = SaltyPasswordField() pager_size = IntegerField('Pager size', diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 5b6e5486e..9605d3463 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -459,7 +459,6 @@ class model(watch_base): # Save some text file to the appropriate path and bump the history # result_obj from fetch_site_status.run() def save_history_text(self, contents, timestamp, snapshot_id): - import brotli import tempfile logger.trace(f"{self.get('uuid')} - Updating history.txt with timestamp {timestamp}") @@ -471,19 +470,20 @@ class model(watch_base): # Auto-detect binary vs text data if isinstance(contents, bytes): # Binary data (screenshots, images, PDFs, etc.) - # Detect image format from magic bytes for proper extension - if contents[:8] == b'\x89PNG\r\n\x1a\n': - ext = 'png' - elif contents[:3] == b'\xff\xd8\xff': - ext = 'jpg' - elif contents[:6] in (b'GIF87a', b'GIF89a'): - ext = 'gif' - elif contents[:4] == b'RIFF' and contents[8:12] == b'WEBP': - ext = 'webp' - elif contents[:4] == b'%PDF': - ext = 'pdf' - else: - ext = 'bin' # Generic binary + # Use puremagic for file type detection (already a project dependency) + try: + import puremagic + # Check first 2KB for magic bytes - enough for all common formats + detections = puremagic.magic_string(contents[:2048]) + if detections: + # Get highest confidence match + ext = detections[0].extension + logger.trace(f"Detected file type: {detections[0].mime_type} -> extension: {ext}") + else: + ext = 'bin' # Fallback for unknown binary types + except Exception as e: + logger.warning(f"puremagic detection failed: {e}, using 'bin' extension") + ext = 'bin' snapshot_fname = f"{snapshot_id}.{ext}" encoded_data = contents # Already bytes, no encoding needed @@ -503,6 +503,7 @@ class model(watch_base): # Write snapshot file atomically if it doesn't exist if not os.path.exists(dest): if encoded_data is None: + import brotli # Brotli compression in subprocess try: actual_dest = _brotli_subprocess_save(contents, dest, mode=brotli.MODE_TEXT, fallback_uncompressed=True) diff --git a/changedetectionio/processors/image_ssim_diff/__init__.py b/changedetectionio/processors/image_ssim_diff/__init__.py index 9da9ae585..96dcbf53c 100644 --- a/changedetectionio/processors/image_ssim_diff/__init__.py +++ b/changedetectionio/processors/image_ssim_diff/__init__.py @@ -11,8 +11,6 @@ processor_description = "Visual/Screenshot change detection (Fast)" processor_name = "image_ssim_diff" processor_weight = 2 # Lower weight = appears at top, heavier weight = appears lower (bottom) -# Default comparison settings -DEFAULT_COMPARISON_THRESHOLD_OPENCV = float(os.getenv('COMPARISON_THRESHOLD_OPENCV', '30')) # Subprocess timeout settings # Maximum time to wait for subprocess operations (seconds) @@ -20,3 +18,11 @@ POLL_TIMEOUT_ABSOLUTE = int(os.getenv('OPENCV_SUBPROCESS_TIMEOUT', '20')) # Template tracking filename CROPPED_IMAGE_TEMPLATE_FILENAME = 'cropped_image_template.png' + +SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS = [ + ('0.75', 'Low sensitivity (only major changes)'), + ('0.85', 'Medium sensitivity (moderate changes)'), + ('0.96', 'High sensitivity (small changes)'), + ('0.999', 'Very high sensitivity (any change)') +] +SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT=0.999 diff --git a/changedetectionio/processors/image_ssim_diff/difference.py b/changedetectionio/processors/image_ssim_diff/difference.py index f25f7fe29..3838b8edb 100644 --- a/changedetectionio/processors/image_ssim_diff/difference.py +++ b/changedetectionio/processors/image_ssim_diff/difference.py @@ -9,7 +9,8 @@ import os import json import time from loguru import logger -from . import DEFAULT_COMPARISON_THRESHOLD_OPENCV + +from changedetectionio.processors.image_ssim_diff import SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT # All image operations now use OpenCV via isolated_opencv subprocess handler # No direct handler imports needed - subprocess isolation handles everything @@ -81,7 +82,7 @@ def get_asset(asset_name, watch, datastore, request): # Get threshold threshold = watch.get('comparison_threshold') if not threshold or threshold == '': - threshold = datastore.data['settings']['application'].get('comparison_threshold', DEFAULT_COMPARISON_THRESHOLD_OPENCV) + threshold = datastore.data['settings']['application'].get('comparison_threshold', SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT) try: threshold = float(threshold) @@ -304,7 +305,7 @@ def render(watch, datastore, request, url_for, render_template, flash, redirect) # Get threshold (per-watch > global > env default) threshold = watch.get('comparison_threshold') if not threshold or threshold == '': - threshold = datastore.data['settings']['application'].get('comparison_threshold', DEFAULT_COMPARISON_THRESHOLD_OPENCV) + threshold = datastore.data['settings']['application'].get('comparison_threshold', SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT) # Convert threshold to appropriate type try: diff --git a/changedetectionio/processors/image_ssim_diff/forms.py b/changedetectionio/processors/image_ssim_diff/forms.py index 18874d14a..3a1426414 100644 --- a/changedetectionio/processors/image_ssim_diff/forms.py +++ b/changedetectionio/processors/image_ssim_diff/forms.py @@ -6,6 +6,8 @@ from wtforms import SelectField, StringField, validators, ValidationError from changedetectionio.forms import processor_text_json_diff_form import re +from changedetectionio.processors.image_ssim_diff import SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS + def validate_bounding_box(form, field): """Validate bounding box format: x,y,width,height with integers.""" @@ -43,13 +45,8 @@ class processor_settings_form(processor_text_json_diff_form): comparison_threshold = SelectField( 'Screenshot Comparison Sensitivity', choices=[ - ('', 'Use global default'), - ('10', 'Very low sensitivity (only major changes)'), - ('20', 'Low sensitivity (significant changes)'), - ('30', 'Medium sensitivity (moderate changes)'), - ('50', 'High sensitivity (small changes)'), - ('75', 'Very high sensitivity (any visible change)') - ], + ('', 'Use global default') + ] + SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS, validators=[validators.Optional()], default='' ) diff --git a/changedetectionio/processors/image_ssim_diff/image_handler/isolated_opencv.py b/changedetectionio/processors/image_ssim_diff/image_handler/isolated_opencv.py index 3d70904a5..54afcbce0 100644 --- a/changedetectionio/processors/image_ssim_diff/image_handler/isolated_opencv.py +++ b/changedetectionio/processors/image_ssim_diff/image_handler/isolated_opencv.py @@ -42,6 +42,13 @@ def _worker_compare(conn, img_bytes_from, img_bytes_to, threshold, blur_sigma, m print(f"[{time.time():.3f}] [Worker] Loading images (from={len(img_bytes_from)} bytes, to={len(img_bytes_to)} bytes)", flush=True) img_from = cv2.imdecode(np.frombuffer(img_bytes_from, np.uint8), cv2.IMREAD_COLOR) img_to = cv2.imdecode(np.frombuffer(img_bytes_to, np.uint8), cv2.IMREAD_COLOR) + + # Check if decoding succeeded + if img_from is None: + raise ValueError("Failed to decode 'from' image - may be corrupt or unsupported format") + if img_to is None: + raise ValueError("Failed to decode 'to' image - may be corrupt or unsupported format") + print(f"[{time.time():.3f}] [Worker] Images loaded: from={img_from.shape}, to={img_to.shape}", flush=True) # Crop if region specified diff --git a/changedetectionio/processors/image_ssim_diff/processor.py b/changedetectionio/processors/image_ssim_diff/processor.py index ee9078afa..36829af5a 100644 --- a/changedetectionio/processors/image_ssim_diff/processor.py +++ b/changedetectionio/processors/image_ssim_diff/processor.py @@ -10,10 +10,9 @@ import hashlib import os import time from loguru import logger -from changedetectionio import strtobool -from changedetectionio.processors import difference_detection_processor, SCREENSHOT_FORMAT_PNG +from changedetectionio.processors import difference_detection_processor from changedetectionio.processors.exceptions import ProcessorException -from . import DEFAULT_COMPARISON_THRESHOLD_OPENCV, CROPPED_IMAGE_TEMPLATE_FILENAME +from . import SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT # All image operations now use OpenCV via isolated_opencv subprocess handler # Template matching temporarily disabled pending OpenCV implementation @@ -51,20 +50,22 @@ class perform_site_check(difference_detection_processor): current_md5 = hashlib.md5(self.screenshot).hexdigest() previous_md5 = watch.get('previous_md5') if previous_md5 and current_md5 == previous_md5: - logger.debug(f"Screenshot MD5 unchanged ({current_md5}), skipping comparison") + logger.debug(f"UUID: {watch.get('uuid')} - Screenshot MD5 unchanged ({current_md5}), skipping comparison") raise checksumFromPreviousCheckWasTheSame() + else: + logger.debug(f"UUID: {watch.get('uuid')} - Screenshot MD5 changed") - # Get threshold (per-watch > global > env default) + # Get threshold (per-watch > global) threshold = watch.get('comparison_threshold') - if not threshold or threshold == '': - threshold = self.datastore.data['settings']['application'].get('comparison_threshold', DEFAULT_COMPARISON_THRESHOLD_OPENCV) + if not threshold: + threshold = self.datastore.data['settings']['application'].get('comparison_threshold', SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT) # Convert string to appropriate type try: threshold = float(threshold) except (ValueError, TypeError): logger.warning(f"Invalid threshold value '{threshold}', using default") - threshold = 30.0 + threshold = SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT # Check if bounding box is set (for drawn area mode) # Read from processor-specific config JSON file (named after processor) @@ -76,7 +77,7 @@ class perform_site_check(difference_detection_processor): bounding_box = processor_config.get('bounding_box') if processor_config else None # Template matching for tracking content movement - template_matching_enabled = processor_config.get('auto_track_region', False) + template_matching_enabled = processor_config.get('auto_track_region', False) #@@todo disabled for now if bounding_box: try: @@ -86,11 +87,11 @@ class perform_site_check(difference_detection_processor): x, y, width, height = parts # Crop uses (left, top, right, bottom) crop_region = (max(0, x), max(0, y), x + width, y + height) - logger.info(f"Bounding box enabled: cropping to region {crop_region} (x={x}, y={y}, w={width}, h={height})") + logger.info(f"UUID: {watch.get('uuid')} - Bounding box enabled: cropping to region {crop_region} (x={x}, y={y}, w={width}, h={height})") else: - logger.warning(f"Invalid bounding box format: {bounding_box} (expected 4 values)") + logger.warning(f"UUID: {watch.get('uuid')} - Invalid bounding box format: {bounding_box} (expected 4 values)") except Exception as e: - logger.warning(f"Failed to parse bounding box '{bounding_box}': {e}") + logger.warning(f"UUID: {watch.get('uuid')} - Failed to parse bounding box '{bounding_box}': {e}") # If no bounding box, check if visual selector (include_filters) is set for region-based comparison if not crop_region: @@ -119,11 +120,11 @@ class perform_site_check(difference_detection_processor): # Crop uses (left, top, right, bottom) crop_region = (max(0, left), max(0, top), left + width, top + height) - logger.info(f"Visual selector enabled: cropping to region {crop_region} for filter: {first_filter}") + logger.info(f"UUID: {watch.get('uuid')} - Visual selector enabled: cropping to region {crop_region} for filter: {first_filter}") break except Exception as e: - logger.warning(f"Failed to parse xpath_data for visual selector: {e}") + logger.warning(f"UUID: {watch.get('uuid')} - Failed to parse xpath_data for visual selector: {e}") # Store original crop region for template matching original_crop_region = crop_region @@ -132,7 +133,7 @@ class perform_site_check(difference_detection_processor): history_keys = list(watch.history.keys()) if len(history_keys) == 0: # First check - save baseline, no comparison - logger.info(f"First check for watch {watch.get('uuid')} - saving baseline screenshot") + logger.info(f"UUID: {watch.get('uuid')} - First check for watch {watch.get('uuid')} - saving baseline screenshot") # LibVIPS uses automatic reference counting - no explicit cleanup needed update_obj = { @@ -162,7 +163,7 @@ class perform_site_check(difference_detection_processor): blur_sigma = float(os.getenv("OPENCV_BLUR_SIGMA", "0.8")) min_change_percentage = float(os.getenv("OPENCV_MIN_CHANGE_PERCENT", "0.1")) - logger.debug(f"Starting isolated subprocess comparison (crop_region={crop_region})") + logger.debug(f"UUID: {watch.get('uuid')} - Starting isolated subprocess comparison (crop_region={crop_region})") # Compare using isolated subprocess with OpenCV (async-safe to avoid blocking event loop) # Pass raw bytes and crop region - subprocess handles all image operations @@ -202,15 +203,15 @@ class perform_site_check(difference_detection_processor): changed_detected, change_score = result_container[0] - logger.debug(f"Isolated subprocess comparison completed: changed={changed_detected}, score={change_score:.2f}") + logger.debug(f"UUID: {watch.get('uuid')} - Isolated subprocess comparison completed: changed={changed_detected}, score={change_score:.2f}") logger.info(f"{process_screenshot_handler.IMPLEMENTATION_NAME}: {change_score:.2f}% pixels changed, threshold: {threshold:.0f}") except Exception as e: - logger.error(f"Failed to compare screenshots: {e}") - logger.trace(f"Processed in {time.time() - now:.3f}s") + logger.error(f"UUID: {watch.get('uuid')} - Failed to compare screenshots: {e}") + logger.trace(f"UUID: {watch.get('uuid')} - Processed in {time.time() - now:.3f}s") raise ProcessorException( - message=f"Screenshot comparison failed: {e}", + message=f"UUID: {watch.get('uuid')} - Screenshot comparison failed: {e}", url=watch.get('url') ) @@ -221,10 +222,10 @@ class perform_site_check(difference_detection_processor): } if changed_detected: - logger.info(f"Change detected using OpenCV! Score: {change_score:.2f}") + logger.info(f"UUID: {watch.get('uuid')} - Change detected using OpenCV! Score: {change_score:.2f}") else: - logger.debug(f"No significant change using OpenCV. Score: {change_score:.2f}") - logger.trace(f"Processed in {time.time() - now:.3f}s") + logger.debug(f"UUID: {watch.get('uuid')} - No significant change using OpenCV. Score: {change_score:.2f}") + logger.trace(f"UUID: {watch.get('uuid')} - Processed in {time.time() - now:.3f}s") return changed_detected, update_obj, self.screenshot diff --git a/changedetectionio/processors/image_ssim_diff/templates/image_ssim_diff/diff.html b/changedetectionio/processors/image_ssim_diff/templates/image_ssim_diff/diff.html index 42d5196d4..98b4ddb03 100644 --- a/changedetectionio/processors/image_ssim_diff/templates/image_ssim_diff/diff.html +++ b/changedetectionio/processors/image_ssim_diff/templates/image_ssim_diff/diff.html @@ -3,52 +3,51 @@ {% block content %} +
- - Edit - Edit Watch - - -
+
-

Screenshot Comparison (Fast)

- - {% if versions|length >= 2 %} -
- - -
- -
- - -
+ {% if versions|length >= 1 %} + + + + + + + + {% endif %} - - -
+
+
+ Change Detection: {{ "%.2f"|format(change_percentage) }}% of pixels changed -
{% if change_percentage > 0.1 %} ⚠ Change Detected {% else %} ✓ No Significant Change {% endif %} -
+ + {%- if versions|length >= 2 -%} +
+ Keyboard: + ← Previous +   → Next +
+ {%- endif -%} diff --git a/changedetectionio/processors/image_ssim_diff/templates/image_ssim_diff/preview.html b/changedetectionio/processors/image_ssim_diff/templates/image_ssim_diff/preview.html index e832d24c7..e0ff30706 100644 --- a/changedetectionio/processors/image_ssim_diff/templates/image_ssim_diff/preview.html +++ b/changedetectionio/processors/image_ssim_diff/templates/image_ssim_diff/preview.html @@ -1,37 +1,30 @@ {% extends 'base.html' %} {% block content %} -
-
-
-

Screenshot Preview - {{ watch.label or watch.url }}

- Recheck - View Diff History - Edit -
-
- -
-
-
- Preview Options -
- - + {% for version in versions|reverse %} + - {% endfor %} - -
-
- - {{ current_diff_url }} -
-
-
-
+ {% endfor %} + + + + + +
+ Keyboard: + ← Previous   + → Next +
+ {% endif %}

Screenshot from {{ timestamp|format_timestamp_timeago }}

@@ -54,5 +47,4 @@ to see what changed between versions.

- {% endblock %}