diff --git a/.github/workflows/test-stack-reusable-workflow.yml b/.github/workflows/test-stack-reusable-workflow.yml index 938d2da6e..42da7df8d 100644 --- a/.github/workflows/test-stack-reusable-workflow.yml +++ b/.github/workflows/test-stack-reusable-workflow.yml @@ -84,6 +84,7 @@ jobs: docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model' docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security' docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver' + docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_html_to_text' # Basic pytest tests with ancillary services basic-tests: diff --git a/changedetectionio/async_update_worker.py b/changedetectionio/async_update_worker.py index b12574616..88037c45d 100644 --- a/changedetectionio/async_update_worker.py +++ b/changedetectionio/async_update_worker.py @@ -75,7 +75,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec continue uuid = queued_item_data.item.get('uuid') - # RACE CONDITION FIX: Check if this UUID is already being processed by another worker from changedetectionio import worker_handler from changedetectionio.queuedWatchMetaData import PrioritizedItem diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index b0ae4374c..3718e3fdf 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -539,8 +539,32 @@ def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False, timeout=10) -> str: - from inscriptis import get_text + """ + Convert HTML content to plain text using inscriptis. + + CRITICAL FIX: Uses thread-local lxml parsers to prevent race conditions. + + Under high concurrency (50+ threads), lxml's default shared global parser + (lxml.html.html_parser) caused non-deterministic parsing behavior, resulting in: + - Different text extraction from identical HTML + - Different MD5 checksums + - False change detection alerts + + This function ensures each thread gets its own HTMLParser instance via + thread-local storage, eliminating the race condition. + + See: LXML_THREADING_FIX.md for full details + """ + from inscriptis import Inscriptis from inscriptis.model.config import ParserConfig + from lxml.html import fromstring, HTMLParser + import threading + + # Thread-local storage for HTML parser instances to avoid race conditions + # under high concurrency. lxml's default parser is a shared global which + # causes non-deterministic parsing when used from multiple threads. + if not hasattr(html_to_text, '_thread_local'): + html_to_text._thread_local = threading.local() if render_anchor_tag_content: parser_config = ParserConfig( @@ -554,7 +578,15 @@ def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=Fals html_content = re.sub(r'