Important fix for possible wrong detection of changes under high-concurrency setups (many many fetch workers)

.github/workflows/test-stack-reusable-workflow.yml

@@ -84,6 +84,7 @@ jobs:
           docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
           docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'
           docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'
+          docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_html_to_text'
 
   # Basic pytest tests with ancillary services
   basic-tests:

changedetectionio/async_update_worker.py

@@ -75,7 +75,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore, exec
             continue
 
         uuid = queued_item_data.item.get('uuid')
-
         # RACE CONDITION FIX: Check if this UUID is already being processed by another worker
         from changedetectionio import worker_handler
         from changedetectionio.queuedWatchMetaData import PrioritizedItem

changedetectionio/html_tools.py

@@ -539,8 +539,32 @@ def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False
 
 
 def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False, timeout=10) -> str:
-    from inscriptis import get_text
+    """
+    Convert HTML content to plain text using inscriptis.
+
+    CRITICAL FIX: Uses thread-local lxml parsers to prevent race conditions.
+
+    Under high concurrency (50+ threads), lxml's default shared global parser
+    (lxml.html.html_parser) caused non-deterministic parsing behavior, resulting in:
+    - Different text extraction from identical HTML
+    - Different MD5 checksums
+    - False change detection alerts
+
+    This function ensures each thread gets its own HTMLParser instance via
+    thread-local storage, eliminating the race condition.
+
+    See: LXML_THREADING_FIX.md for full details
+    """
+    from inscriptis import Inscriptis
     from inscriptis.model.config import ParserConfig
+    from lxml.html import fromstring, HTMLParser
+    import threading
+
+    # Thread-local storage for HTML parser instances to avoid race conditions
+    # under high concurrency. lxml's default parser is a shared global which
+    # causes non-deterministic parsing when used from multiple threads.
+    if not hasattr(html_to_text, '_thread_local'):
+        html_to_text._thread_local = threading.local()
 
     if render_anchor_tag_content:
         parser_config = ParserConfig(
@@ -554,7 +578,15 @@ def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=Fals
         html_content = re.sub(r'<title([\s>])', r'<h1\1', html_content)
         html_content = re.sub(r'</title>', r'</h1>', html_content)
 
-    text_content = get_text(html_content, config=parser_config)
+    # Get or create thread-local parser
+    if not hasattr(html_to_text._thread_local, 'parser'):
+        html_to_text._thread_local.parser = HTMLParser()
+
+    # Parse HTML with thread-local parser to ensure thread-safety
+    html_tree = fromstring(html_content, parser=html_to_text._thread_local.parser)
+
+    # Convert to text using inscriptis
+    text_content = Inscriptis(html_tree, config=parser_config).get_text()
     return text_content
 
 # Does LD+JSON exist with a @type=='product' and a .price set anywhere?