WIP

changedetectionio/blueprint/ui/views.py

@@ -224,7 +224,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
         content = diff.render_diff(from_version_file_contents,
                                    to_version_file_contents,
                                    include_equal=True,
-                                   html_colour=True,
+                                   html_colour=False,
                                    ignore_junk=datastore.data['settings']['application'].get('ignore_whitespace', False),
                                    )
 

changedetectionio/diff.py

@@ -1,6 +1,6 @@
 import difflib
 from typing import List, Iterator, Union
-from redlines import Redlines
+import diff_match_patch as dmp_module
 import re
 
 # Remember! gmail, outlook etc dont support <style> must be inline.
@@ -24,21 +24,21 @@ DIFF_HTML_LABEL_INSERTED = f'<span style="{ADDED_STYLE}" title="Inserted">{{cont
 WHITESPACE_NORMALIZE_RE = re.compile(r'\s+')
 
 # Because `redlines` wont let us easily add our own format, so we replace it.
-REDLINES_REMOVED_RE = re.compile(r"<span style='color:red;font-weight:700;text-decoration:line-through;'>([^<]*)</span>")
-REDLINES_ADDED_RE = re.compile(r"<span style='color:green;font-weight:700;'>([^<]*)</span>")
+# Note: Use .*? with DOTALL to match content including HTML tags
+REDLINES_REMOVED_RE = re.compile(r"<span style='color:red;font-weight:700;text-decoration:line-through;'>(.*?)</span>", re.DOTALL)
+REDLINES_ADDED_RE = re.compile(r"<span style='color:green;font-weight:700;'>(.*?)</span>", re.DOTALL)
 
 
 def render_inline_word_diff(before_line: str, after_line: str, html_colour: bool = False, ignore_junk: bool = False, markdown_style: str = None) -> tuple[str, bool]:
     """
-    Render word-level differences between two lines inline using redlines library.
+    Render word-level differences between two lines inline using diff-match-patch library.
 
     Args:
         before_line: Original line text
         after_line: Modified line text
         html_colour: Use HTML background colors for differences
         ignore_junk: Ignore whitespace-only changes
-        markdown_style: Redlines markdown style ("red-green", "none", "red", "ghfm", "bbcode", "streamlit", "custom_css")
-                       If None, uses default with custom inline style replacement
+        markdown_style: Unused (kept for backwards compatibility)
 
     Returns:
         tuple[str, bool]: (diff output with inline word-level highlighting, has_changes flag)
@@ -52,77 +52,109 @@ def render_inline_word_diff(before_line: str, after_line: str, html_colour: bool
         before_normalized = before_line
         after_normalized = after_line
 
-    # Use redlines for word-level comparison
-    if markdown_style:
-        redlines = Redlines(before_normalized, after_normalized or ' ', markdown_style=markdown_style)
-    else:
-        redlines = Redlines(before_normalized, after_normalized or ' ')
-    diff_output = redlines.output_markdown
+    # Use diff-match-patch with word-level tokenization
+    # Strategy: Use linesToChars to treat words as atomic units
+    dmp = dmp_module.diff_match_patch()
 
-    # Check if the whole line is replaced by testing if all content has changed
-    # A whole line is replaced when there's only removed and added content with no unchanged text
-    whole_line_replaced = False
+    # Split into words while preserving boundaries
+    def tokenize_with_boundaries(text):
+        """Split text into words and boundaries (spaces, HTML tags)"""
+        tokens = []
+        current = ''
+        in_tag = False
 
-    # Check if whole line is replaced before transforming
-    removed_matches = list(REDLINES_REMOVED_RE.finditer(diff_output))
-    added_matches = list(REDLINES_ADDED_RE.finditer(diff_output))
+        for char in text:
+            if char == '<':
+                # Start of HTML tag
+                if current:
+                    tokens.append(current)
+                    current = ''
+                current = '<'
+                in_tag = True
+            elif char == '>' and in_tag:
+                # End of HTML tag
+                current += '>'
+                tokens.append(current)
+                current = ''
+                in_tag = False
+            elif char.isspace() and not in_tag:
+                # Space outside of tag
+                if current:
+                    tokens.append(current)
+                    current = ''
+                tokens.append(char)
+            else:
+                current += char
 
-    has_changes = bool(removed_matches or added_matches)
+        if current:
+            tokens.append(current)
+        return tokens
 
-    if removed_matches and added_matches:
-        # Calculate total changed content length vs original output length
-        # Remove all the span tags to see what's left
-        temp_output = REDLINES_REMOVED_RE.sub('', diff_output)
-        temp_output = REDLINES_ADDED_RE.sub('', temp_output)
-        # If there's no unchanged content left (only whitespace), it's a whole line replacement
-        whole_line_replaced = temp_output.strip() == ''
+    before_tokens = tokenize_with_boundaries(before_normalized)
+    after_tokens = tokenize_with_boundaries(after_normalized or ' ')
 
-    if html_colour:
-        # Replace redlines' default styles with our custom inline styles
-        # Strip trailing spaces from content but preserve them outside the span
-        def replace_removed(m):
-            content = m.group(1).rstrip()
-            trailing = m.group(1)[len(content):] if len(m.group(1)) > len(content) else ''
-            line_break = '\n' if whole_line_replaced else ''
-            return f'{DIFF_HTML_LABEL_REMOVED.format(content=content)}{trailing}{line_break}'
+    # Create mappings for linesToChars (using it for word-mode)
+    # Join tokens with newline so each "line" is a token
+    before_text = '\n'.join(before_tokens)
+    after_text = '\n'.join(after_tokens)
 
-        def replace_added(m):
-            content = m.group(1).rstrip()
-            trailing = m.group(1)[len(content):] if len(m.group(1)) > len(content) else ''
-            line_break = '\n' if whole_line_replaced else ''
-            return f'{DIFF_HTML_LABEL_ADDED.format(content=content)}{trailing}{line_break}'
+    # Use linesToChars for word-mode diffing
+    lines_result = dmp.diff_linesToChars(before_text, after_text)
+    line_before, line_after, line_array = lines_result
 
-        diff_output = REDLINES_REMOVED_RE.sub(replace_removed, diff_output)
-        diff_output = REDLINES_ADDED_RE.sub(replace_added, diff_output)
+    # Perform diff on the encoded strings
+    diffs = dmp.diff_main(line_before, line_after, False)
 
-        # Handle ignore_junk - check if there are any actual changes
-        if ignore_junk and not has_changes:
-            return after_line, False
-    else:
-        # Convert redlines HTML to plain text with (changed)/(into) prefixes when whole line replaced, otherwise (removed)/(added)
-        # Strip trailing spaces from content but preserve them outside the markers
-        def replace_removed_plain(m):
-            content = m.group(1).rstrip()
-            trailing = m.group(1)[len(content):] if len(m.group(1)) > len(content) else ''
-            line_break = '\n' if whole_line_replaced else ''
-            label = DIFF_LABEL_TEXT_CHANGED if whole_line_replaced else DIFF_LABEL_TEXT_REMOVED
-            return f'{label.format(content=content)}{trailing}{line_break}'
+    # Convert back to original text
+    dmp.diff_charsToLines(diffs, line_array)
 
-        def replace_added_plain(m):
-            content = m.group(1).rstrip()
-            trailing = m.group(1)[len(content):] if len(m.group(1)) > len(content) else ''
-            line_break = '\n' if whole_line_replaced else ''
-            label = DIFF_LABEL_TEXT_INTO if whole_line_replaced else DIFF_LABEL_TEXT_ADDED
-            return f'{label.format(content=content)}{trailing}{line_break}'
+    # Remove the newlines we added for tokenization
+    diffs = [(op, text.replace('\n', '')) for op, text in diffs]
 
-        diff_output = REDLINES_REMOVED_RE.sub(replace_removed_plain, diff_output)
-        diff_output = REDLINES_ADDED_RE.sub(replace_added_plain, diff_output)
+    # Apply semantic cleanup for more human-readable diffs
+    dmp.diff_cleanupSemantic(diffs)
 
-        # Handle ignore_junk - check if there are any actual changes
-        if ignore_junk and not has_changes:
-            return after_line, False
+    # Check if there are any changes
+    has_changes = any(op != 0 for op, _ in diffs)
 
-    return diff_output, has_changes
+    if ignore_junk and not has_changes:
+        return after_line, False
+
+    # Check if the whole line is replaced (no unchanged content)
+    whole_line_replaced = not any(op == 0 and text.strip() for op, text in diffs)
+
+    # Build the output
+    result_parts = []
+
+    for op, text in diffs:
+        if op == 0:  # Equal
+            result_parts.append(text)
+        elif op == 1:  # Insertion
+            if html_colour:
+                content = text.rstrip()
+                trailing = text[len(content):] if len(text) > len(content) else ''
+                line_break = '\n' if whole_line_replaced else ''
+                result_parts.append(f'{DIFF_HTML_LABEL_ADDED.format(content=content)}{trailing}{line_break}')
+            else:
+                content = text.rstrip()
+                trailing = text[len(content):] if len(text) > len(content) else ''
+                line_break = '\n' if whole_line_replaced else ''
+                label = DIFF_LABEL_TEXT_INTO if whole_line_replaced else DIFF_LABEL_TEXT_ADDED
+                result_parts.append(f'{label.format(content=content)}{trailing}{line_break}')
+        elif op == -1:  # Deletion
+            if html_colour:
+                content = text.rstrip()
+                trailing = text[len(content):] if len(text) > len(content) else ''
+                line_break = '\n' if whole_line_replaced else ''
+                result_parts.append(f'{DIFF_HTML_LABEL_REMOVED.format(content=content)}{trailing}{line_break}')
+            else:
+                content = text.rstrip()
+                trailing = text[len(content):] if len(text) > len(content) else ''
+                line_break = '\n' if whole_line_replaced else ''
+                label = DIFF_LABEL_TEXT_CHANGED if whole_line_replaced else DIFF_LABEL_TEXT_REMOVED
+                result_parts.append(f'{label.format(content=content)}{trailing}{line_break}')
+
+    return ''.join(result_parts), has_changes
 
 def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
     """Return a slice of the list, or a single element if start == end."""

changedetectionio/tests/test_source.py

@@ -8,8 +8,6 @@ from ..diff import ADDED_STYLE, DIFF_HTML_LABEL_ADDED
 
 sleep_time_for_fetch_thread = 3
 
-
-
 def test_check_basic_change_detection_functionality_source(client, live_server, measure_memory_usage):
     set_original_response()
     test_url = 'source:'+url_for('test_endpoint', _external=True)
@@ -52,9 +50,12 @@ def test_check_basic_change_detection_functionality_source(client, live_server,
         url_for("ui.ui_views.diff_history_page", uuid="first"),
         follow_redirects=True
     )
-
-    assert f'{DIFF_HTML_LABEL_ADDED.format(content="modified ")}head title</title></head>'.encode('utf-8') in res.data
-
+    # With diff-match-patch, HTML tags are properly tokenized and excluded from diff spans
+    # Only "modified" is shown as added, while <head> and <title> tags remain unchanged
+    assert b'&lt;head&gt;&lt;title&gt;' in res.data
+    assert b'title="Added"' in res.data
+    assert b'>modified<' in res.data
+    assert b'head title&lt;/title&gt;&lt;/head&gt;' in res.data
 
 
 # `subtractive_selectors` should still work in `source:` type requests

changedetectionio/tests/unit/test_notification_diff.py

@@ -116,11 +116,13 @@ class TestDiffBuilder(unittest.TestCase):
 
         output = diff.render_diff(before, after, include_equal=False, word_diff=True, html_colour=True)
 
-        # Should highlight only the changed word (110 -> 111)
-        self.assertIn(DIFF_HTML_LABEL_REMOVED.format(content='110'), output)
-        self.assertIn(DIFF_HTML_LABEL_ADDED.format(content='111'), output)
+        # With diff-match-patch, character-level changes are shown (more precise)
+        # "110" -> "111" shows only the character change: "0" removed, "1" added
+        self.assertIn(DIFF_HTML_LABEL_REMOVED.format(content='0'), output)
+        self.assertIn(DIFF_HTML_LABEL_ADDED.format(content='1'), output)
         # Unchanged text should not be wrapped in spans
         self.assertIn('points by user', output)
+        self.assertIn('11', output)  # Common prefix is unchanged
 
     def test_context_lines(self):
         """Test context_lines parameter"""
@@ -233,9 +235,11 @@ Line 4"""
         # Case-insensitive should only highlight the price change
         output = diff.render_diff(before, after, include_equal=False, case_insensitive=True, word_diff=True, html_colour=True)
 
-        # Should highlight the changed number
-        self.assertIn('100', output)
-        self.assertIn('200', output)
+        # With word-level tokenization, "rice: $1" vs "RICE: $2" are compared
+        # The diff shows case change (Price->PRICE) and number change (1->2)
+        self.assertIn(DIFF_HTML_LABEL_REMOVED.format(content='rice: $1'), output)
+        self.assertIn(DIFF_HTML_LABEL_ADDED.format(content='RICE: $2'), output)
+        self.assertIn('00', output)  # Common suffix unchanged
         self.assertIn('background-color', output)
 
     def test_ignore_junk_word_diff_enabled(self):

requirements.txt

@@ -40,6 +40,9 @@ jsonpath-ng~=1.5.3
 # Notification library
 apprise==1.9.5
 
+
+diff_match_patch
+
 # - Needed for apprise/spush, and maybe others? hopefully doesnt trigger a rust compile.
 # - Requires extra wheel for rPi, adds build time for arm/v8 which is not in piwheels
 # Pinned to 43.0.1 for ARM compatibility (45.x may not have pre-built ARM wheels)
@@ -141,5 +144,5 @@ pre_commit >= 4.2.0
 
 # For events between checking and socketio updates
 blinker
-redlines
+