From b25e242b8ddc964ca844b63b865b2dce7ac07c92 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Fri, 20 Feb 2026 19:16:17 +0100 Subject: [PATCH] Ignore white-space should strip following white-space from Difference / Preview / API endpoints --- changedetectionio/api/Watch.py | 7 +++ changedetectionio/blueprint/ui/preview.py | 3 ++ changedetectionio/html_tools.py | 11 ++++ .../processors/text_json_diff/difference.py | 5 ++ .../tests/unit/test_html_to_text.py | 50 ++++++++++++++++++- 5 files changed, 75 insertions(+), 1 deletion(-) diff --git a/changedetectionio/api/Watch.py b/changedetectionio/api/Watch.py index b0e56f140..a6fee7b05 100644 --- a/changedetectionio/api/Watch.py +++ b/changedetectionio/api/Watch.py @@ -260,7 +260,10 @@ class WatchSingleHistory(Resource): response = make_response("No content found", 404) response.mimetype = "text/plain" else: + from changedetectionio import html_tools content = watch.get_history_snapshot(timestamp=timestamp) + if self.datastore.data['settings']['application'].get('ignore_whitespace', False): + content = html_tools.rstrip_snapshot_content(content) response = make_response(content, 200) response.mimetype = "text/plain" @@ -328,8 +331,12 @@ class WatchHistoryDiff(Resource): no_markup = strtobool(request.args.get('no_markup', 'false')) # Retrieve snapshot contents + from changedetectionio import html_tools from_version_file_contents = watch.get_history_snapshot(from_timestamp) to_version_file_contents = watch.get_history_snapshot(to_timestamp) + if self.datastore.data['settings']['application'].get('ignore_whitespace', False): + from_version_file_contents = html_tools.rstrip_snapshot_content(from_version_file_contents) + to_version_file_contents = html_tools.rstrip_snapshot_content(to_version_file_contents) # Get diff preferences from query parameters (matching UI preferences in DIFF_PREFERENCES_CONFIG) # Support both 'type' (UI parameter) and 'word_diff' (API parameter) for backward compatibility diff --git a/changedetectionio/blueprint/ui/preview.py b/changedetectionio/blueprint/ui/preview.py index 1eac4a3d7..737281bc2 100644 --- a/changedetectionio/blueprint/ui/preview.py +++ b/changedetectionio/blueprint/ui/preview.py @@ -84,6 +84,9 @@ def construct_blueprint(datastore: ChangeDetectionStore): versions = list(watch.history.keys()) content = watch.get_history_snapshot(timestamp=timestamp) + if datastore.data['settings']['application'].get('ignore_whitespace', False): + content = html_tools.rstrip_snapshot_content(content) + triggered_line_numbers = html_tools.strip_ignore_text(content=content, wordlist=watch.get('trigger_text'), mode='line numbers' diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index 789aed4b3..a0175f2ed 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -631,6 +631,17 @@ def workarounds_for_obfuscations(content): return content +def rstrip_snapshot_content(content: str) -> str: + """Strip trailing whitespace from each line of a snapshot. + + Table-layout pages (common on older sites) cause inscriptis to pad every cell + to the maximum column width found in any row, producing lines of thousands of + trailing spaces from simple spacer/nav rows that sit alongside wide content rows. + This should be applied at display/output time, not at storage time. + """ + return '\n'.join(line.rstrip() for line in content.splitlines()) + + def get_triggered_text(content, trigger_text): triggered_text = [] result = strip_ignore_text(content=content, diff --git a/changedetectionio/processors/text_json_diff/difference.py b/changedetectionio/processors/text_json_diff/difference.py index e89b5d9b4..88533faa4 100644 --- a/changedetectionio/processors/text_json_diff/difference.py +++ b/changedetectionio/processors/text_json_diff/difference.py @@ -152,6 +152,11 @@ def render(watch, datastore, request, url_for, render_template, flash, redirect, logger.error(f"Unable to read watch history from-version for version {from_version}: {str(e)}") from_version_file_contents = f"Unable to read to-version {from_version}.\n" + if datastore.data['settings']['application'].get('ignore_whitespace', False): + from changedetectionio import html_tools + to_version_file_contents = html_tools.rstrip_snapshot_content(to_version_file_contents) + from_version_file_contents = html_tools.rstrip_snapshot_content(from_version_file_contents) + screenshot_url = watch.get_screenshot() system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver' diff --git a/changedetectionio/tests/unit/test_html_to_text.py b/changedetectionio/tests/unit/test_html_to_text.py index a25e9d493..2d1d3267b 100644 --- a/changedetectionio/tests/unit/test_html_to_text.py +++ b/changedetectionio/tests/unit/test_html_to_text.py @@ -8,7 +8,7 @@ import threading import unittest from queue import Queue -from changedetectionio.html_tools import html_to_text +from changedetectionio.html_tools import html_to_text, rstrip_snapshot_content class TestHtmlToText(unittest.TestCase): @@ -622,6 +622,54 @@ var also = 1; assert 'Real content after the data attribute' in text +class TestRstripSnapshotContent(unittest.TestCase): + """Tests for rstrip_snapshot_content — the display-layer whitespace trimmer.""" + + def test_strips_trailing_spaces(self): + """Lines with trailing spaces are rstripped.""" + content = "hello \nworld\nfoo " + result = rstrip_snapshot_content(content) + assert result == "hello\nworld\nfoo" + + def test_preserves_content_without_trailing_whitespace(self): + """Lines without trailing whitespace are left unchanged.""" + content = "line one\nline two\nline three" + assert rstrip_snapshot_content(content) == content + + def test_table_padding_artifact(self): + """Simulates inscriptis table-layout padding: a symbol followed by thousands of spaces.""" + padded_line = "▲" + " " * 27000 + content = padded_line + "\n" + " " * 27000 + "\nsome real content" + result = rstrip_snapshot_content(content) + lines = result.splitlines() + assert lines[0] == "▲" + assert lines[1] == "" + assert lines[2] == "some real content" + + def test_preserves_leading_whitespace(self): + """Leading whitespace (indentation) is never touched.""" + content = " indented line \n another " + result = rstrip_snapshot_content(content) + assert result == " indented line\n another" + + def test_empty_string(self): + assert rstrip_snapshot_content("") == "" + + def test_html_to_text_does_not_rstrip(self): + """html_to_text itself must NOT apply rstrip — that's the output layer's job. + A table with a narrow cell next to a wide cell produces trailing padding.""" + html = """ + + + +
narrow{}
xy
+ """.format("w" * 500) + text = html_to_text(html) + # At least one line should have trailing spaces (inscriptis table padding) + has_trailing = any(line != line.rstrip() for line in text.splitlines()) + assert has_trailing, "Expected inscriptis to produce trailing-space padding on table rows" + + if __name__ == '__main__': # Can run this file directly for quick testing unittest.main()