WIP

changedetectionio/forms.py

@@ -6,6 +6,8 @@ from wtforms.widgets.core import TimeInput
 from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES, RSS_TEMPLATE_TYPE_OPTIONS, RSS_TEMPLATE_HTML_DEFAULT
 from changedetectionio.conditions.form import ConditionFormRow
 from changedetectionio.notification_service import NotificationContextData
+from changedetectionio.processors.image_ssim_diff import SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS, \
+    SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT
 from changedetectionio.strtobool import strtobool
 
 from wtforms import (
@@ -996,13 +998,8 @@ class globalSettingsApplicationForm(commonSettingsForm):
     ignore_whitespace = BooleanField('Ignore whitespace')
     ssim_threshold = SelectField(
         'Default Screenshot Comparison Sensitivity',
-        choices=[
-            ('0.75', 'Low sensitivity (only major changes)'),
-            ('0.85', 'Medium sensitivity (moderate changes)'),
-            ('0.96', 'High sensitivity (small changes)'),
-            ('0.999', 'Very high sensitivity (any change)')
-        ],
-        default='0.96'
+        choices=SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS,
+        default=SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT
     )
     password = SaltyPasswordField()
     pager_size = IntegerField('Pager size',

changedetectionio/model/Watch.py

@@ -459,7 +459,6 @@ class model(watch_base):
    # Save some text file to the appropriate path and bump the history
     # result_obj from fetch_site_status.run()
     def save_history_text(self, contents, timestamp, snapshot_id):
-        import brotli
         import tempfile
         logger.trace(f"{self.get('uuid')} - Updating history.txt with timestamp {timestamp}")
 
@@ -471,19 +470,20 @@ class model(watch_base):
         # Auto-detect binary vs text data
         if isinstance(contents, bytes):
             # Binary data (screenshots, images, PDFs, etc.)
-            # Detect image format from magic bytes for proper extension
-            if contents[:8] == b'\x89PNG\r\n\x1a\n':
-                ext = 'png'
-            elif contents[:3] == b'\xff\xd8\xff':
-                ext = 'jpg'
-            elif contents[:6] in (b'GIF87a', b'GIF89a'):
-                ext = 'gif'
-            elif contents[:4] == b'RIFF' and contents[8:12] == b'WEBP':
-                ext = 'webp'
-            elif contents[:4] == b'%PDF':
-                ext = 'pdf'
-            else:
-                ext = 'bin'  # Generic binary
+            # Use puremagic for file type detection (already a project dependency)
+            try:
+                import puremagic
+                # Check first 2KB for magic bytes - enough for all common formats
+                detections = puremagic.magic_string(contents[:2048])
+                if detections:
+                    # Get highest confidence match
+                    ext = detections[0].extension
+                    logger.trace(f"Detected file type: {detections[0].mime_type} -> extension: {ext}")
+                else:
+                    ext = 'bin'  # Fallback for unknown binary types
+            except Exception as e:
+                logger.warning(f"puremagic detection failed: {e}, using 'bin' extension")
+                ext = 'bin'
 
             snapshot_fname = f"{snapshot_id}.{ext}"
             encoded_data = contents  # Already bytes, no encoding needed
@@ -503,6 +503,7 @@ class model(watch_base):
         # Write snapshot file atomically if it doesn't exist
         if not os.path.exists(dest):
             if encoded_data is None:
+                import brotli
                 # Brotli compression in subprocess
                 try:
                     actual_dest = _brotli_subprocess_save(contents, dest, mode=brotli.MODE_TEXT, fallback_uncompressed=True)

changedetectionio/processors/image_ssim_diff/__init__.py

@@ -11,8 +11,6 @@ processor_description = "Visual/Screenshot change detection (Fast)"
 processor_name = "image_ssim_diff"
 processor_weight = 2  # Lower weight = appears at top, heavier weight = appears lower (bottom)
 
-# Default comparison settings
-DEFAULT_COMPARISON_THRESHOLD_OPENCV = float(os.getenv('COMPARISON_THRESHOLD_OPENCV', '30'))
 
 # Subprocess timeout settings
 # Maximum time to wait for subprocess operations (seconds)
@@ -20,3 +18,11 @@ POLL_TIMEOUT_ABSOLUTE = int(os.getenv('OPENCV_SUBPROCESS_TIMEOUT', '20'))
 
 # Template tracking filename
 CROPPED_IMAGE_TEMPLATE_FILENAME = 'cropped_image_template.png'
+
+SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS = [
+    ('0.75', 'Low sensitivity (only major changes)'),
+    ('0.85', 'Medium sensitivity (moderate changes)'),
+    ('0.96', 'High sensitivity (small changes)'),
+    ('0.999', 'Very high sensitivity (any change)')
+]
+SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT=0.999

changedetectionio/processors/image_ssim_diff/difference.py

@@ -9,7 +9,8 @@ import os
 import json
 import time
 from loguru import logger
-from . import DEFAULT_COMPARISON_THRESHOLD_OPENCV
+
+from changedetectionio.processors.image_ssim_diff import SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT
 
 # All image operations now use OpenCV via isolated_opencv subprocess handler
 # No direct handler imports needed - subprocess isolation handles everything
@@ -81,7 +82,7 @@ def get_asset(asset_name, watch, datastore, request):
             # Get threshold
             threshold = watch.get('comparison_threshold')
             if not threshold or threshold == '':
-                threshold = datastore.data['settings']['application'].get('comparison_threshold', DEFAULT_COMPARISON_THRESHOLD_OPENCV)
+                threshold = datastore.data['settings']['application'].get('comparison_threshold', SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT)
 
             try:
                 threshold = float(threshold)
@@ -304,7 +305,7 @@ def render(watch, datastore, request, url_for, render_template, flash, redirect)
     # Get threshold (per-watch > global > env default)
     threshold = watch.get('comparison_threshold')
     if not threshold or threshold == '':
-        threshold = datastore.data['settings']['application'].get('comparison_threshold', DEFAULT_COMPARISON_THRESHOLD_OPENCV)
+        threshold = datastore.data['settings']['application'].get('comparison_threshold', SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT)
 
     # Convert threshold to appropriate type
     try:

changedetectionio/processors/image_ssim_diff/forms.py

@@ -6,6 +6,8 @@ from wtforms import SelectField, StringField, validators, ValidationError
 from changedetectionio.forms import processor_text_json_diff_form
 import re
 
+from changedetectionio.processors.image_ssim_diff import SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS
+
 
 def validate_bounding_box(form, field):
     """Validate bounding box format: x,y,width,height with integers."""
@@ -43,13 +45,8 @@ class processor_settings_form(processor_text_json_diff_form):
     comparison_threshold = SelectField(
         'Screenshot Comparison Sensitivity',
         choices=[
-            ('', 'Use global default'),
-            ('10', 'Very low sensitivity (only major changes)'),
-            ('20', 'Low sensitivity (significant changes)'),
-            ('30', 'Medium sensitivity (moderate changes)'),
-            ('50', 'High sensitivity (small changes)'),
-            ('75', 'Very high sensitivity (any visible change)')
-        ],
+                    ('', 'Use global default')
+                ] + SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS,
         validators=[validators.Optional()],
         default=''
     )

changedetectionio/processors/image_ssim_diff/image_handler/isolated_opencv.py

@@ -42,6 +42,13 @@ def _worker_compare(conn, img_bytes_from, img_bytes_to, threshold, blur_sigma, m
         print(f"[{time.time():.3f}] [Worker] Loading images (from={len(img_bytes_from)} bytes, to={len(img_bytes_to)} bytes)", flush=True)
         img_from = cv2.imdecode(np.frombuffer(img_bytes_from, np.uint8), cv2.IMREAD_COLOR)
         img_to = cv2.imdecode(np.frombuffer(img_bytes_to, np.uint8), cv2.IMREAD_COLOR)
+
+        # Check if decoding succeeded
+        if img_from is None:
+            raise ValueError("Failed to decode 'from' image - may be corrupt or unsupported format")
+        if img_to is None:
+            raise ValueError("Failed to decode 'to' image - may be corrupt or unsupported format")
+
         print(f"[{time.time():.3f}] [Worker] Images loaded: from={img_from.shape}, to={img_to.shape}", flush=True)
 
         # Crop if region specified

changedetectionio/processors/image_ssim_diff/processor.py

@@ -10,10 +10,9 @@ import hashlib
 import os
 import time
 from loguru import logger
-from changedetectionio import strtobool
-from changedetectionio.processors import difference_detection_processor, SCREENSHOT_FORMAT_PNG
+from changedetectionio.processors import difference_detection_processor
 from changedetectionio.processors.exceptions import ProcessorException
-from . import DEFAULT_COMPARISON_THRESHOLD_OPENCV, CROPPED_IMAGE_TEMPLATE_FILENAME
+from . import SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT
 
 # All image operations now use OpenCV via isolated_opencv subprocess handler
 # Template matching temporarily disabled pending OpenCV implementation
@@ -51,20 +50,22 @@ class perform_site_check(difference_detection_processor):
         current_md5 = hashlib.md5(self.screenshot).hexdigest()
         previous_md5 = watch.get('previous_md5')
         if previous_md5 and current_md5 == previous_md5:
-            logger.debug(f"Screenshot MD5 unchanged ({current_md5}), skipping comparison")
+            logger.debug(f"UUID: {watch.get('uuid')} - Screenshot MD5 unchanged ({current_md5}), skipping comparison")
             raise checksumFromPreviousCheckWasTheSame()
+        else:
+            logger.debug(f"UUID: {watch.get('uuid')} - Screenshot MD5 changed")
 
-        # Get threshold (per-watch > global > env default)
+        # Get threshold (per-watch > global)
         threshold = watch.get('comparison_threshold')
-        if not threshold or threshold == '':
-            threshold = self.datastore.data['settings']['application'].get('comparison_threshold', DEFAULT_COMPARISON_THRESHOLD_OPENCV)
+        if not threshold:
+            threshold = self.datastore.data['settings']['application'].get('comparison_threshold', SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT)
 
         # Convert string to appropriate type
         try:
             threshold = float(threshold)
         except (ValueError, TypeError):
             logger.warning(f"Invalid threshold value '{threshold}', using default")
-            threshold = 30.0
+            threshold = SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT
 
         # Check if bounding box is set (for drawn area mode)
         # Read from processor-specific config JSON file (named after processor)
@@ -76,7 +77,7 @@ class perform_site_check(difference_detection_processor):
         bounding_box = processor_config.get('bounding_box') if processor_config else None
 
         # Template matching for tracking content movement
-        template_matching_enabled = processor_config.get('auto_track_region', False)
+        template_matching_enabled = processor_config.get('auto_track_region', False) #@@todo disabled for now
 
         if bounding_box:
             try:
@@ -86,11 +87,11 @@ class perform_site_check(difference_detection_processor):
                     x, y, width, height = parts
                     # Crop uses (left, top, right, bottom)
                     crop_region = (max(0, x), max(0, y), x + width, y + height)
-                    logger.info(f"Bounding box enabled: cropping to region {crop_region} (x={x}, y={y}, w={width}, h={height})")
+                    logger.info(f"UUID: {watch.get('uuid')} - Bounding box enabled: cropping to region {crop_region} (x={x}, y={y}, w={width}, h={height})")
                 else:
-                    logger.warning(f"Invalid bounding box format: {bounding_box} (expected 4 values)")
+                    logger.warning(f"UUID: {watch.get('uuid')} - Invalid bounding box format: {bounding_box} (expected 4 values)")
             except Exception as e:
-                logger.warning(f"Failed to parse bounding box '{bounding_box}': {e}")
+                logger.warning(f"UUID: {watch.get('uuid')} - Failed to parse bounding box '{bounding_box}': {e}")
 
         # If no bounding box, check if visual selector (include_filters) is set for region-based comparison
         if not crop_region:
@@ -119,11 +120,11 @@ class perform_site_check(difference_detection_processor):
                                 # Crop uses (left, top, right, bottom)
                                 crop_region = (max(0, left), max(0, top), left + width, top + height)
 
-                                logger.info(f"Visual selector enabled: cropping to region {crop_region} for filter: {first_filter}")
+                                logger.info(f"UUID: {watch.get('uuid')} - Visual selector enabled: cropping to region {crop_region} for filter: {first_filter}")
                                 break
 
                     except Exception as e:
-                        logger.warning(f"Failed to parse xpath_data for visual selector: {e}")
+                        logger.warning(f"UUID: {watch.get('uuid')} - Failed to parse xpath_data for visual selector: {e}")
 
         # Store original crop region for template matching
         original_crop_region = crop_region
@@ -132,7 +133,7 @@ class perform_site_check(difference_detection_processor):
         history_keys = list(watch.history.keys())
         if len(history_keys) == 0:
             # First check - save baseline, no comparison
-            logger.info(f"First check for watch {watch.get('uuid')} - saving baseline screenshot")
+            logger.info(f"UUID: {watch.get('uuid')} - First check for watch {watch.get('uuid')} - saving baseline screenshot")
 
             # LibVIPS uses automatic reference counting - no explicit cleanup needed
             update_obj = {
@@ -162,7 +163,7 @@ class perform_site_check(difference_detection_processor):
             blur_sigma = float(os.getenv("OPENCV_BLUR_SIGMA", "0.8"))
             min_change_percentage = float(os.getenv("OPENCV_MIN_CHANGE_PERCENT", "0.1"))
 
-            logger.debug(f"Starting isolated subprocess comparison (crop_region={crop_region})")
+            logger.debug(f"UUID: {watch.get('uuid')} - Starting isolated subprocess comparison (crop_region={crop_region})")
 
             # Compare using isolated subprocess with OpenCV (async-safe to avoid blocking event loop)
             # Pass raw bytes and crop region - subprocess handles all image operations
@@ -202,15 +203,15 @@ class perform_site_check(difference_detection_processor):
 
             changed_detected, change_score = result_container[0]
 
-            logger.debug(f"Isolated subprocess comparison completed: changed={changed_detected}, score={change_score:.2f}")
+            logger.debug(f"UUID: {watch.get('uuid')} - Isolated subprocess comparison completed: changed={changed_detected}, score={change_score:.2f}")
             logger.info(f"{process_screenshot_handler.IMPLEMENTATION_NAME}: {change_score:.2f}% pixels changed, threshold: {threshold:.0f}")
 
         except Exception as e:
-            logger.error(f"Failed to compare screenshots: {e}")
-            logger.trace(f"Processed in {time.time() - now:.3f}s")
+            logger.error(f"UUID: {watch.get('uuid')} - Failed to compare screenshots: {e}")
+            logger.trace(f"UUID: {watch.get('uuid')} - Processed in {time.time() - now:.3f}s")
 
             raise ProcessorException(
-                message=f"Screenshot comparison failed: {e}",
+                message=f"UUID: {watch.get('uuid')} - Screenshot comparison failed: {e}",
                 url=watch.get('url')
             )
 
@@ -221,10 +222,10 @@ class perform_site_check(difference_detection_processor):
         }
 
         if changed_detected:
-            logger.info(f"Change detected using OpenCV! Score: {change_score:.2f}")
+            logger.info(f"UUID: {watch.get('uuid')} - Change detected using OpenCV! Score: {change_score:.2f}")
         else:
-            logger.debug(f"No significant change using OpenCV. Score: {change_score:.2f}")
-        logger.trace(f"Processed in {time.time() - now:.3f}s")
+            logger.debug(f"UUID: {watch.get('uuid')} - No significant change using OpenCV. Score: {change_score:.2f}")
+        logger.trace(f"UUID: {watch.get('uuid')} - Processed in {time.time() - now:.3f}s")
 
         return changed_detected, update_obj, self.screenshot
 

changedetectionio/processors/image_ssim_diff/templates/image_ssim_diff/diff.html

@@ -3,52 +3,51 @@
 
 {% block content %}
 <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='diff-image.css')}}?v={{ get_css_version() }}">
+<script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
 <div id="settings">
-    <a href="{{ url_for('ui.ui_edit.edit_page', uuid=uuid, next='diff') }}" class="pure-button button-small edit-link">
-        <img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread_the_word.svg')}}" alt="Edit" />
-        Edit Watch
-    </a>
-
-    <form class="pure-form" action="{{ url_for('ui.ui_diff.diff_history_page', uuid=uuid) }}" method="GET" id="diff-form">
+    <form class="pure-form " action="{{ url_for('ui.ui_diff.diff_history_page', uuid=uuid) }}" method="GET" id="diff-form">
         <fieldset class="diff-fieldset">
-            <h2>Screenshot Comparison (Fast)</h2>
-
-            {% if versions|length >= 2 %}
-                <div class="version-selector">
-                    <label for="diff-from-version">From:</label>
-                    <select id="diff-from-version" name="from_version" class="needs-localtime" onchange="this.form.submit()">
-                        {%- for version in versions|reverse -%}
-                            <option value="{{ version }}" {% if version== from_version %} selected="" {% endif %}>
-                                {{ version }}
-                            </option>
-                        {%- endfor -%}
-                    </select>
-                </div>
-
-                <div class="version-selector">
-                    <label for="diff-to-version">To:</label>
-                    <select id="diff-to-version" name="to_version" class="needs-localtime" onchange="this.form.submit()">
-                        {%- for version in versions|reverse -%}
-                            <option value="{{ version }}" {% if version== to_version %} selected="" {% endif %}>
-                                {{ version }}
-                            </option>
-                        {%- endfor -%}
-                    </select>
-                </div>
+            {% if versions|length >= 1 %}
+                <span style="white-space: nowrap;">
+                <label id="change-from" for="diff-from-version" class="from-to-label">From</label>
+                <select id="diff-from-version" name="from_version" class="needs-localtime">
+                    {%- for version in versions|reverse -%}
+                        <option value="{{ version }}" {% if version== from_version %} selected="" {% endif %}>
+                            {{ version }}
+                        </option>
+                    {%- endfor -%}
+                </select>
+                </span>
+                <span style="white-space: nowrap;">
+                <label id="change-to" for="diff-to-version" class="from-to-label">To</label>
+                <select id="diff-to-version" name="to_version" class="needs-localtime">
+                    {%- for version in versions|reverse -%}
+                        <option value="{{ version }}" {% if version== to_version %} selected="" {% endif %}>
+                            {{ version }}
+                        </option>
+                    {%- endfor -%}
+                </select>
+                </span>
             {% endif %}
-
-            <!-- Comparison score display -->
-            <div class="comparison-score">
+        </fieldset>
+        <fieldset id="diff-style">
+            <span>
                 <strong>Change Detection:</strong> {{ "%.2f"|format(change_percentage) }}% of pixels changed
-                <br>
                 {% if change_percentage > 0.1 %}
                     <span class="change-detected">⚠ Change Detected</span>
                 {% else %}
                     <span class="no-change">✓ No Significant Change</span>
                 {% endif %}
-            </div>
+            </span>
         </fieldset>
+        {%- if versions|length >= 2 -%}
+            <div id="keyboard-nav">
+                <strong>Keyboard: </strong>
+                <a href="" class="pure-button pure-button-primary" id="btn-previous"> &larr; Previous</a>
+                &nbsp; <a class="pure-button pure-button-primary" id="btn-next" href=""> &rarr; Next</a>
+            </div>
+        {%- endif -%}
     </form>
 </div>
 

changedetectionio/processors/image_ssim_diff/templates/image_ssim_diff/preview.html

@@ -1,37 +1,30 @@
 {% extends 'base.html' %}
 
 {% block content %}
-<div class="box">
-    <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 1em;">
-        <div style="flex: 1;">
-            <h2>Screenshot Preview - {{ watch.label or watch.url }}</h2>
-            <a href="{{ url_for('ui.form_watch_checknow', uuid=uuid) }}" class="pure-button button-small button-primary">Recheck</a>
-            <a href="{{ url_for('ui.ui_diff.diff_history_page', uuid=uuid) }}" class="pure-button button-small">View Diff History</a>
-            <a href="{{ url_for('ui.ui_edit.edit_page', uuid=uuid) }}" class="pure-button button-small">Edit</a>
-        </div>
-    </div>
-
-    <div id="settings" style="margin-bottom: 1.5em; padding: 1em; background: #f5f5f5; border-radius: 4px;">
-        <form action="{{ url_for('ui.ui_preview.preview_page', uuid=uuid) }}">
-            <fieldset>
-                <legend>Preview Options</legend>
-                <div class="pure-control-group">
-                    <label for="version">Version:</label>
-                    <select name="version" id="version" onchange="this.form.submit()" class="pure-input-1-3">
-                        {% for version_timestamp in versions|reverse %}
-                        <option value="{{ version_timestamp }}" {% if version_timestamp == timestamp %}selected{% endif %}>
-                            {{ version_timestamp|format_timestamp_timeago }}
+<script src="{{ url_for('static_content', group='js', filename='preview.js') }}" defer></script>
+    {% if versions|length >= 2 %}
+        <div id="diff-form" style="text-align: center;">
+            <form class="pure-form " action="" method="GET">
+                <fieldset>
+                    <label for="preview-version">Select timestamp</label> <select id="preview-version"
+                                                                                 name="version"
+                                                                                 class="needs-localtime">
+                    {% for version in versions|reverse %}
+                        <option value="{{ version }}" {% if version == timestamp %} selected="" {% endif %}>
+                            {{ version }}
                         </option>
-                        {% endfor %}
-                    </select>
-                </div>
-                <div class="pure-control-group">
-                    <label for="url">Watching:</label>
-                    <a href="{{ current_diff_url }}" target="_blank" rel="noopener" style="word-break: break-all;">{{ current_diff_url }}</a>
-                </div>
-            </fieldset>
-        </form>
-    </div>
+                    {% endfor %}
+                </select>
+                    <button type="submit" class="pure-button pure-button-primary">Go</button>
+
+                </fieldset>
+            </form>
+                <br>
+                <strong>Keyboard: </strong><a href="" class="pure-button pure-button-primary" id="btn-previous">
+                &larr; Previous</a> &nbsp; <a class="pure-button pure-button-primary" id="btn-next" href="">
+                &rarr; Next</a>
+        </div>
+    {% endif %}
 
     <div id="screenshot-container" style="text-align: center; border: 1px solid #ddd; padding: 2em; background: #fafafa; border-radius: 4px;">
         <h3 style="margin-top: 0;">Screenshot from {{ timestamp|format_timestamp_timeago }}</h3>
@@ -54,5 +47,4 @@
             to see what changed between versions.
         </p>
     </div>
-</div>
 {% endblock %}