diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index 6b93402e..7286e4c9 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -572,6 +572,20 @@ def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=Fals html_content = re.sub(r'<(?:style|script|svg|noscript)[^>]*>.*?|<(?:link|meta)[^>]*/?>|', '', html_content, flags=re.DOTALL | re.IGNORECASE) + # SPAs often use to hide content until JS loads + # inscriptis respects CSS display rules, so we need to remove these hiding styles + # to extract the actual page content + body_style_pattern = r'(]*)\s+style\s*=\s*["\']([^"\']*\b(?:display\s*:\s*none|visibility\s*:\s*hidden)\b[^"\']*)["\']' + + # Check if body has hiding styles that need to be fixed + body_match = re.search(body_style_pattern, html_content, flags=re.IGNORECASE) + if body_match: + from loguru import logger + logger.debug(f"html_to_text: Removing hiding styles from body tag (found: '{body_match.group(2)}')") + + html_content = re.sub(body_style_pattern, r'\1', html_content, flags=re.IGNORECASE) + + text_content = get_text(html_content, config=parser_config) return text_content diff --git a/changedetectionio/processors/text_json_diff/processor.py b/changedetectionio/processors/text_json_diff/processor.py index b039723e..77ef2ba8 100644 --- a/changedetectionio/processors/text_json_diff/processor.py +++ b/changedetectionio/processors/text_json_diff/processor.py @@ -347,6 +347,7 @@ class ContentProcessor: def extract_text_from_html(self, html_content, stream_content_type): """Convert HTML to plain text.""" do_anchor = self.datastore.data["settings"]["application"].get("render_anchor_tag_content", False) + return html_tools.html_to_text( html_content=html_content, render_anchor_tag_content=do_anchor, diff --git a/changedetectionio/tests/unit/test_html_to_text.py b/changedetectionio/tests/unit/test_html_to_text.py index 9815c27a..04b16584 100644 --- a/changedetectionio/tests/unit/test_html_to_text.py +++ b/changedetectionio/tests/unit/test_html_to_text.py @@ -284,6 +284,85 @@ class TestHtmlToText(unittest.TestCase): print(f" ✓ Successfully processed {html_size_mb:.2f}MB HTML -> {text_size_kb:.2f}KB text") + def test_body_display_none_spa_pattern(self): + """ + Test that html_to_text can extract content from pages with display:none body. + + SPAs (Single Page Applications) often use to hide content + until JavaScript loads and renders the page. inscriptis respects CSS display rules, + so without preprocessing, it would skip all content and return only newlines. + + The fix strips display:none and visibility:hidden styles from the body tag before + processing, allowing text extraction from client-side rendered applications. + """ + # Test case 1: Basic display:none + html1 = ''' + +What's New – Fluxguard + +

Important Heading

+

This is actual content that should be extracted.

+
+

First paragraph with meaningful text.

+

Second paragraph with more content.

+
+ +''' + + text1 = html_to_text(html1) + + # Before fix: would return ~33 newlines, len(text) ~= 33 + # After fix: should extract actual content, len(text) > 100 + assert len(text1) > 100, f"Expected substantial text output, got {len(text1)} chars" + assert 'Important Heading' in text1, "Failed to extract heading from display:none body" + assert 'actual content' in text1, "Failed to extract paragraph from display:none body" + assert 'First paragraph' in text1, "Failed to extract nested content" + + # Should not be mostly newlines + newline_ratio = text1.count('\n') / len(text1) + assert newline_ratio < 0.5, f"Output is mostly newlines ({newline_ratio:.2%}), content not extracted" + + # Test case 2: visibility:hidden (another hiding pattern) + html2 = '

Hidden Content

Test paragraph.

' + text2 = html_to_text(html2) + + assert 'Hidden Content' in text2, "Failed to extract content from visibility:hidden body" + assert 'Test paragraph' in text2, "Failed to extract paragraph from visibility:hidden body" + + # Test case 3: Mixed styles (display:none with other CSS) + html3 = '

Mixed style content

' + text3 = html_to_text(html3) + + assert 'Mixed style content' in text3, "Failed to extract content from body with mixed styles" + + # Test case 4: Case insensitivity (DISPLAY:NONE uppercase) + html4 = '

Uppercase style

' + text4 = html_to_text(html4) + + assert 'Uppercase style' in text4, "Failed to handle uppercase DISPLAY:NONE" + + # Test case 5: Space variations (display: none vs display:none) + html5 = '

With spaces

' + text5 = html_to_text(html5) + + assert 'With spaces' in text5, "Failed to handle 'display: none' with space" + + # Test case 6: Body with other attributes (class, id) + html6 = '

With attributes

' + text6 = html_to_text(html6) + + assert 'With attributes' in text6, "Failed to extract from body with multiple attributes" + + # Test case 7: Should NOT affect opacity:0 (which doesn't hide from inscriptis) + html7 = '

Transparent content

' + text7 = html_to_text(html7) + + # Opacity doesn't affect inscriptis text extraction, content should be there + assert 'Transparent content' in text7, "Incorrectly stripped opacity:0 style" + + print(" ✓ All display:none body tag tests passed") + + if __name__ == '__main__': # Can run this file directly for quick testing