Keep the surviving lines of strip_ignore_text() in document order (#4536)

The set difference is iterated in hash-slot order, so when only a few
lines survive the strip they come back out of document order and the
snapshot (and its checksum) depends on the hash table size rather than
on the input.

Co-authored-by: Yi-111-a <Yi-111-a@users.noreply.github.com>

changedetectionio/html_tools.py

@@ -782,10 +782,9 @@ def strip_ignore_text(content, wordlist, mode="content"):
 
     # Used for finding out what to highlight
     if mode == "line numbers":
-        return [i + 1 for i in ignored_lines]
+        return sorted(i + 1 for i in ignored_lines)
 
-    output_lines = set(range(len(lines))) - ignored_lines
-    return ''.join([lines[i] for i in output_lines])
+    return ''.join([line for i, line in enumerate(lines) if i not in ignored_lines])
 
 def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False) -> str:
     from xml.sax.saxutils import escape as xml_escape

changedetectionio/tests/test_ignore_regex_text.py

@@ -85,3 +85,16 @@ def test_strip_regex_text_func():
     content = "some text\n\nand other text\n"
     stripped_content = html_tools.strip_ignore_text(content, ignore_lines)
     assert content == stripped_content
+
+
+def test_strip_regex_text_keeps_document_order():
+    # Only a few lines surviving the strip must still come back in document order
+    content = ''.join(f"line{i}\n" for i in range(100))
+    ignore_lines = [f"/^line{i}$/" for i in range(90)]
+
+    stripped_content = html_tools.strip_ignore_text(content, ignore_lines)
+    assert stripped_content == "".join(f"line{i}\n" for i in range(90, 100))
+
+    # And the line numbers used for highlighting come back sorted too
+    stripped_content = html_tools.strip_ignore_text(content, ignore_lines, mode="line numbers")
+    assert stripped_content == list(range(1, 91))