diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index 824c0e6aa..6b93402e2 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -565,6 +565,12 @@ def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=Fals if is_rss: html_content = re.sub(r'])', r'', r'', html_content) + else: + # Strip bloat in one pass, SPA's often dump 10Mb+ into the for styles, which is not needed + # Causing inscriptis to silently exit when more than ~10MB is found. + # All we are doing here is converting the HTML to text, no CSS layout etc + html_content = re.sub(r'<(?:style|script|svg|noscript)[^>]*>.*?|<(?:link|meta)[^>]*/?>|', + '', html_content, flags=re.DOTALL | re.IGNORECASE) text_content = get_text(html_content, config=parser_config) return text_content diff --git a/changedetectionio/tests/unit/test_html_to_text.py b/changedetectionio/tests/unit/test_html_to_text.py index f4787a47c..9815c27a1 100644 --- a/changedetectionio/tests/unit/test_html_to_text.py +++ b/changedetectionio/tests/unit/test_html_to_text.py @@ -199,6 +199,91 @@ class TestHtmlToText(unittest.TestCase): print(f"✓ Basic thread-safety test passed: {len(results)} threads, no errors") + def test_large_html_with_bloated_head(self): + """ + Test that html_to_text can handle large HTML documents with massive bloat. + + SPAs often dump 10MB+ of styles, scripts, and other bloat into the section. + This can cause inscriptis to silently exit when processing very large documents. + The fix strips \n' + + # Generate massive script block (~5MB) + large_script = '\n' + + # Generate lots of SVG bloat (~3MB) + svg_bloat = '\n' * 50000 + + # Generate meta/link tags (~2MB) + meta_bloat = '\n' * 50000 + link_bloat = '\n' * 50000 + + # Generate HTML comments (~1MB) + comment_bloat = '\n' * 50000 + + # Generate noscript bloat + noscript_bloat = '\n' * 10000 + + # Build the large HTML document + html = f''' + + + Test Page + {large_style} + {large_script} + {svg_bloat} + {meta_bloat} + {link_bloat} + {comment_bloat} + {noscript_bloat} + + +

Important Heading

+

This is the actual content that should be extracted.

+
+

First paragraph with meaningful text.

+

Second paragraph with more content.

+
+
Footer text
+ + +''' + + # Verify the HTML is actually large (should be ~20MB+) + html_size_mb = len(html) / (1024 * 1024) + assert html_size_mb > 15, f"HTML should be >15MB, got {html_size_mb:.2f}MB" + + print(f" Testing {html_size_mb:.2f}MB HTML document with bloated head...") + + # This should not crash or silently exit + text = html_to_text(html) + + # Verify we got actual text output (not empty/None) + assert text is not None, "html_to_text returned None" + assert len(text) > 0, "html_to_text returned empty string" + + # Verify the actual body content was extracted + assert 'Important Heading' in text, "Failed to extract heading" + assert 'actual content that should be extracted' in text, "Failed to extract paragraph" + assert 'First paragraph with meaningful text' in text, "Failed to extract first paragraph" + assert 'Second paragraph with more content' in text, "Failed to extract second paragraph" + assert 'Footer text' in text, "Failed to extract footer" + + # Verify bloat was stripped (output should be tiny compared to input) + text_size_kb = len(text) / 1024 + assert text_size_kb < 1, f"Output too large ({text_size_kb:.2f}KB), bloat not stripped" + + # Verify no CSS, script content, or SVG leaked through + assert 'color:red' not in text, "Style content leaked into text output" + assert 'console.log' not in text, "Script content leaked into text output" + assert ' {text_size_kb:.2f}KB text") + if __name__ == '__main__': # Can run this file directly for quick testing