lee/changedetection.io
1
0
Fork 0
mirror of https://github.com/dgtlmoon/changedetection.io.git synced 2026-04-29 22:37:09 +00:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

Option to ignore junk/whitespace etc

Browse Source
This commit is contained in:
dgtlmoon
2025-10-03 10:55:56 +02:00
parent 98745bbe00
commit a57d046b0c
4 changed files with 227 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
changedetectionio/blueprint/ui/views.py
+1 -1
View File
@@ -224,7 +224,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
to_version_file_contents,
include_equal=True,
html_colour=True,
case_insensitive=datastore.data['settings']['application'].get('ignore_whitespace', False)
case_insensitive=datastore.data['settings']['application'].get('ignore_whitespace', False),
)
return render_template("diff.html",
changedetectionio/diff.py
+101 -13
View File
@@ -6,7 +6,7 @@ from typing import List, Iterator, Union
REMOVED_STYLE = "background-color: #fadad7; color: #b30000;"
ADDED_STYLE = "background-color: #eaf2c2; color: #406619;"
def render_inline_word_diff(before_line: str, after_line: str, html_colour: bool = False) -> str:
def render_inline_word_diff(before_line: str, after_line: str, html_colour: bool = False, ignore_junk: bool = False) -> str:
"""
Render word-level differences between two lines inline.
@@ -14,6 +14,7 @@ def render_inline_word_diff(before_line: str, after_line: str, html_colour: bool
before_line: Original line text
after_line: Modified line text
html_colour: Use HTML background colors for differences
ignore_junk: Ignore whitespace-only changes
Returns:
str: Single line with inline word-level highlighting
@@ -30,7 +31,9 @@ def render_inline_word_diff(before_line: str, after_line: str, html_colour: bool
after_tokens = tokenize(after_line)
# Use SequenceMatcher to find word-level differences
matcher = difflib.SequenceMatcher(None, before_tokens, after_tokens)
# If ignore_junk is True, treat whitespace tokens as junk
isjunk = (lambda x: x.strip() == '') if ignore_junk else None
matcher = difflib.SequenceMatcher(isjunk, before_tokens, after_tokens)
if html_colour:
result = []
@@ -39,15 +42,45 @@ def render_inline_word_diff(before_line: str, after_line: str, html_colour: bool
result.append(''.join(before_tokens[i1:i2]))
elif tag == 'delete':
deleted = ''.join(before_tokens[i1:i2])
# If only whitespace and ignore_junk is enabled, preserve whitespace without marking
if ignore_junk and deleted.strip() == '':
result.append(deleted)
continue
result.append(f'<span style="{REMOVED_STYLE}" title="Removed">{deleted}</span>')
elif tag == 'insert':
inserted = ''.join(after_tokens[j1:j2])
# If only whitespace and ignore_junk is enabled, preserve whitespace without marking
if ignore_junk and inserted.strip() == '':
result.append(inserted)
continue
result.append(f'<span style="{ADDED_STYLE}" title="Added">{inserted}</span>')
elif tag == 'replace':
deleted = ''.join(before_tokens[i1:i2])
inserted = ''.join(after_tokens[j1:j2])
result.append(f'<span style="{REMOVED_STYLE}" title="Removed">{deleted}</span>')
result.append(f'<span style="{ADDED_STYLE}" title="Added">{inserted}</span>')
# If both are only whitespace and ignore_junk is enabled, use the after version
if ignore_junk and deleted.strip() == '' and inserted.strip() == '':
result.append(inserted)
continue
# When ignore_junk is enabled, filter out whitespace-only tokens from replace operations
if ignore_junk:
deleted_parts = []
inserted_parts = []
for token in before_tokens[i1:i2]:
if token.strip() != '':
deleted_parts.append(token)
for token in after_tokens[j1:j2]:
if token.strip() != '':
inserted_parts.append(token)
# Add a single space between words (normalized whitespace)
if deleted_parts or inserted_parts:
result.append(' ')
if deleted_parts:
result.append(f'<span style="{REMOVED_STYLE}" title="Removed">{"".join(deleted_parts)}</span>')
if inserted_parts:
result.append(f'<span style="{ADDED_STYLE}" title="Added">{"".join(inserted_parts)}</span>')
else:
result.append(f'<span style="{REMOVED_STYLE}" title="Removed">{deleted}</span>')
result.append(f'<span style="{ADDED_STYLE}" title="Added">{inserted}</span>')
return ''.join(result)
else:
# Plain text format with markers
@@ -57,14 +90,45 @@ def render_inline_word_diff(before_line: str, after_line: str, html_colour: bool
result.append(''.join(before_tokens[i1:i2]))
elif tag == 'delete':
deleted = ''.join(before_tokens[i1:i2])
# If only whitespace and ignore_junk is enabled, preserve whitespace without marking
if ignore_junk and deleted.strip() == '':
result.append(deleted)
continue
result.append(f'[-{deleted}-]')
elif tag == 'insert':
inserted = ''.join(after_tokens[j1:j2])
# If only whitespace and ignore_junk is enabled, preserve whitespace without marking
if ignore_junk and inserted.strip() == '':
result.append(inserted)
continue
result.append(f'[+{inserted}+]')
elif tag == 'replace':
deleted = ''.join(before_tokens[i1:i2])
inserted = ''.join(after_tokens[j1:j2])
result.append(f'[-{deleted}-][+{inserted}+]')
# If both are only whitespace and ignore_junk is enabled, use the after version
if ignore_junk and deleted.strip() == '' and inserted.strip() == '':
result.append(inserted)
continue
# When ignore_junk is enabled, filter out whitespace-only tokens from replace operations
if ignore_junk:
deleted_parts = []
inserted_parts = []
for token in before_tokens[i1:i2]:
if token.strip() != '':
deleted_parts.append(token)
for token in after_tokens[j1:j2]:
if token.strip() != '':
inserted_parts.append(token)
# Add a single space between words (normalized whitespace)
if deleted_parts or inserted_parts:
result.append(' ')
if deleted_parts:
result.append(f'[-{"".join(deleted_parts)}-]')
if inserted_parts:
result.append(f'[+{"".join(inserted_parts)}+]')
else:
result.append(f'[-{deleted}-]')
result.append(f'[+{inserted}+]')
return ''.join(result)
def same_slicer(lst: List[str], start: int, end: int) -> List[str]:
@@ -82,7 +146,8 @@ def customSequenceMatcher(
html_colour: bool = False,
word_diff: bool = False,
context_lines: int = 0,
case_insensitive: bool = False
case_insensitive: bool = False,
ignore_junk: bool = False
) -> Iterator[List[str]]:
"""
Compare two sequences and yield differences based on specified parameters.
@@ -99,13 +164,23 @@ def customSequenceMatcher(
word_diff (bool): Use word-level diffing for replaced lines
context_lines (int): Number of unchanged lines to show around changes (like grep -C)
case_insensitive (bool): Perform case-insensitive comparison
ignore_junk (bool): Ignore whitespace-only changes
Yields:
List[str]: Differences between sequences
"""
# Prepare sequences for comparison (lowercase if case-insensitive)
compare_before = [line.lower() for line in before] if case_insensitive else before
compare_after = [line.lower() for line in after] if case_insensitive else after
# Prepare sequences for comparison (lowercase if case-insensitive, normalize whitespace if ignore_junk)
import re
def prepare_line(line):
if case_insensitive:
line = line.lower()
if ignore_junk:
# Normalize whitespace: replace multiple spaces/tabs with single space
line = re.sub(r'\s+', ' ', line)
return line
compare_before = [prepare_line(line) for line in before]
compare_after = [prepare_line(line) for line in after]
cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \t", a=compare_before, b=compare_after)
@@ -157,7 +232,17 @@ def customSequenceMatcher(
# Use word-level diff for single line replacements when enabled
if word_diff and len(before_lines) == 1 and len(after_lines) == 1:
inline_diff = render_inline_word_diff(before_lines[0], after_lines[0], html_colour)
inline_diff = render_inline_word_diff(before_lines[0], after_lines[0], html_colour, ignore_junk)
# Check if there are any actual changes (not just whitespace when ignore_junk is enabled)
if ignore_junk:
# Check if the output contains any change markers
if html_colour:
has_changes = '<span style=' in inline_diff
else:
has_changes = '[-' in inline_diff or '[+' in inline_diff
if not has_changes:
# No real changes, skip this line
continue
yield [inline_diff]
else:
# Fall back to line-level diff for multi-line changes or when word_diff disabled
@@ -186,7 +271,8 @@ def render_diff(
html_colour: bool = False,
word_diff: bool = True,
context_lines: int = 0,
case_insensitive: bool = False
case_insensitive: bool = False,
ignore_junk: bool = False
) -> str:
"""
Render the difference between two file contents.
@@ -204,7 +290,8 @@ def render_diff(
html_colour (bool): Use HTML background colors for differences
word_diff (bool): Use word-level diffing for replaced lines
context_lines (int): Number of unchanged lines to show around changes (like grep -C)
case_insensitive (bool): Perform case-insensitive comparison
case_insensitive (bool): Perform case-insensitive comparison, By default the test_json_diff/process.py is case sensitive, so this follows same logic
ignore_junk (bool): Ignore whitespace-only changes
Returns:
str: Rendered difference
@@ -227,7 +314,8 @@ def render_diff(
html_colour=html_colour,
word_diff=word_diff,
context_lines=context_lines,
case_insensitive=case_insensitive
case_insensitive=case_insensitive,
ignore_junk=ignore_junk
)
def flatten(lst: List[Union[str, List[str]]]) -> str:
changedetectionio/notification_service.py
+6 -6
View File
@@ -76,15 +76,15 @@ class NotificationService:
prev_snapshot = watch.get_history_snapshot(dates[-2])
current_snapshot = watch.get_history_snapshot(dates[-1])
case_insensitive=self. datastore.data['settings']['application'].get('ignore_whitespace', False)
ignore_junk = self.datastore.data['settings']['application'].get('ignore_whitespace', False)
n_object.update({
'current_snapshot': snapshot_contents,
'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable, case_insensitive=case_insensitive),
'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep, case_insensitive=case_insensitive),
'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep, html_colour=html_colour_enable, case_insensitive=case_insensitive),
'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True, case_insensitive=case_insensitive),
'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep, case_insensitive=case_insensitive),
'diff': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, html_colour=html_colour_enable, ignore_junk=ignore_junk),
'diff_added': diff.render_diff(prev_snapshot, current_snapshot, include_removed=False, line_feed_sep=line_feed_sep, ignore_junk=ignore_junk),
'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep, html_colour=html_colour_enable, ignore_junk=ignore_junk),
'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True, ignore_junk=ignore_junk),
'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep, ignore_junk=ignore_junk),
'notification_timestamp': now,
'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
'triggered_text': triggered_text,
changedetectionio/tests/unit/test_notification_diff.py
+119
View File
@@ -228,5 +228,124 @@ Line 4"""
self.assertIn('200', output)
self.assertIn('background-color', output)
def test_ignore_junk_word_diff_enabled(self):
"""Test ignore_junk with word_diff=True"""
before = "The quick brown fox"
after = "The quick brown fox"
# Without ignore_junk, should detect whitespace changes
output = diff.render_diff(before, after, include_equal=False, word_diff=True, ignore_junk=False)
# Should show some difference (whitespace changes)
self.assertTrue(len(output.strip()) > 0, "Should detect whitespace changes when ignore_junk=False")
# With ignore_junk, should ignore whitespace-only changes
output = diff.render_diff(before, after, include_equal=False, word_diff=True, ignore_junk=True)
lines = [l for l in output.split("\n") if l.strip()]
self.assertEqual(len(lines), 0, "Should ignore whitespace-only changes when ignore_junk=True")
def test_ignore_junk_word_diff_disabled(self):
"""Test ignore_junk with word_diff=False"""
before = "Hello World"
after = "Hello World"
# Without ignore_junk, should detect line change
output = diff.render_diff(before, after, include_equal=False, word_diff=False, ignore_junk=False)
self.assertIn('(changed)', output)
self.assertIn('(into)', output)
# With ignore_junk enabled and word_diff disabled
# When ignore_junk is enabled, whitespace is normalized at line level so lines match
output = diff.render_diff(before, after, include_equal=False, word_diff=False, ignore_junk=True)
# Lines should be treated as equal
lines = [l for l in output.split("\n") if l.strip()]
self.assertEqual(len(lines), 0, "Should ignore whitespace differences at line level")
def test_ignore_junk_with_real_changes(self):
"""Test ignore_junk doesn't ignore actual word changes"""
before = "The quick brown fox"
after = "The quick brown cat"
output = diff.render_diff(before, after, include_equal=False, word_diff=True, ignore_junk=True)
# Should still detect the word change (fox -> cat)
self.assertIn('[-fox-]', output)
self.assertIn('[+cat+]', output)
# But shouldn't highlight whitespace differences
def test_ignore_junk_tabs_vs_spaces(self):
"""Test ignore_junk treats tabs and spaces as equivalent"""
before = "Column1\tColumn2\tColumn3"
after = "Column1 Column2 Column3"
# Without ignore_junk, should detect difference
output = diff.render_diff(before, after, include_equal=False, word_diff=True, ignore_junk=False)
self.assertTrue(len(output.strip()) > 0, "Should detect tab vs space differences")
# With ignore_junk, should ignore tab/space differences
output = diff.render_diff(before, after, include_equal=False, word_diff=True, ignore_junk=True)
lines = [l for l in output.split("\n") if l.strip()]
self.assertEqual(len(lines), 0, "Should ignore tab vs space differences when ignore_junk=True")
def test_ignore_junk_html_output(self):
"""Test ignore_junk with HTML coloring"""
before = "Value: 100 points"
after = "Value: 200 points"
output = diff.render_diff(before, after, include_equal=False, word_diff=True, html_colour=True, ignore_junk=True)
# Should only highlight the actual value change
self.assertIn('100', output)
self.assertIn('200', output)
self.assertIn('background-color', output)
# Should not create separate spans for whitespace changes
def test_ignore_junk_case_insensitive_combination(self):
"""Test ignore_junk combined with case_insensitive"""
before = "The QUICK Brown Fox"
after = "The quick brown FOX"
# Both enabled: should ignore case and whitespace
output = diff.render_diff(before, after, include_equal=False, word_diff=True,
case_insensitive=True, ignore_junk=True)
lines = [l for l in output.split("\n") if l.strip()]
self.assertEqual(len(lines), 0, "Should ignore both case and whitespace differences")
# Only case_insensitive: should detect whitespace changes
output = diff.render_diff(before, after, include_equal=False, word_diff=True,
case_insensitive=True, ignore_junk=False)
self.assertTrue(len(output.strip()) > 0, "Should detect whitespace changes")
# Only ignore_junk: should detect case changes
output = diff.render_diff(before, after, include_equal=False, word_diff=True,
case_insensitive=False, ignore_junk=True)
# Should detect case differences
self.assertIn('QUICK', output)
self.assertIn('quick', output)
self.assertIn('Brown', output)
self.assertIn('brown', output)
# Should show changes (though may be grouped together)
self.assertTrue('[-' in output and '-]' in output, "Should show removed text")
self.assertTrue('[+' in output and '+]' in output, "Should show added text")
def test_ignore_junk_multiline(self):
"""Test ignore_junk with multiple lines"""
before = """Line 1 with spaces
Line 2 unchanged
Line 3 with tabs and spaces"""
after = """Line 1 with spaces
Line 2 unchanged
Line 3 with tabs and spaces"""
# With ignore_junk, should only show unchanged line when include_equal=True
output = diff.render_diff(before, after, include_equal=False, word_diff=True, ignore_junk=True)
lines = [l for l in output.split("\n") if l.strip()]
# Should be empty since only whitespace changed
self.assertEqual(len(lines), 0, "Should ignore whitespace changes across multiple lines")
# Verify Line 2 is not shown as changed
self.assertNotIn('[-Line 2-]', output)
self.assertNotIn('[+Line 2+]', output)
if __name__ == '__main__':
unittest.main()