From 7d8c127e1f44fbf18538a58670a0a8403f08dcd8 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 28 Oct 2025 17:30:17 +0100 Subject: [PATCH] WIP --- changedetectionio/model/App.py | 4 +++- .../processors/text_json_diff/__init__.py | 17 ++++++++++------- changedetectionio/run_basic_tests.sh | 3 ++- changedetectionio/store.py | 1 - changedetectionio/tests/conftest.py | 11 +++++++++-- .../tests/test_add_replace_remove_filter.py | 1 + changedetectionio/tests/test_backup.py | 5 +---- .../tests/test_block_while_text_present.py | 3 +-- .../tests/test_filter_failure_notification.py | 3 +-- changedetectionio/tests/test_live_preview.py | 2 +- .../tests/test_nonrenderable_pages.py | 2 +- 11 files changed, 30 insertions(+), 22 deletions(-) diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py index d3dc474b..084e957a 100644 --- a/changedetectionio/model/App.py +++ b/changedetectionio/model/App.py @@ -1,4 +1,5 @@ from os import getenv +from copy import deepcopy from changedetectionio.blueprint.rss import RSS_FORMAT_TYPES @@ -74,7 +75,8 @@ class model(dict): def __init__(self, *arg, **kw): super(model, self).__init__(*arg, **kw) - self.update(self.base_config) + # CRITICAL: deepcopy to avoid sharing mutable objects between instances + self.update(deepcopy(self.base_config)) def parse_headers_from_text_file(filepath): diff --git a/changedetectionio/processors/text_json_diff/__init__.py b/changedetectionio/processors/text_json_diff/__init__.py index 14053374..8dc10490 100644 --- a/changedetectionio/processors/text_json_diff/__init__.py +++ b/changedetectionio/processors/text_json_diff/__init__.py @@ -32,7 +32,7 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data): '''Used by @app.route("/edit//preview-rendered", methods=['POST'])''' from changedetectionio import forms, html_tools from changedetectionio.model.Watch import model as watch_model - from concurrent.futures import ProcessPoolExecutor + from concurrent.futures import ThreadPoolExecutor from copy import deepcopy from flask import request import brotli @@ -76,13 +76,16 @@ def prepare_filter_prevew(datastore, watch_uuid, form_data): update_handler.fetcher.headers['content-type'] = tmp_watch.get('content-type') # Process our watch with filters and the HTML from disk, and also a blank watch with no filters but also with the same HTML from disk - # Do this as a parallel process because it could take some time - with ProcessPoolExecutor(max_workers=2) as executor: - future1 = executor.submit(_task, tmp_watch, update_handler) - future2 = executor.submit(_task, blank_watch_no_filters, update_handler) + # Do this as parallel threads (not processes) to avoid pickle issues with Lock objects + try: + with ThreadPoolExecutor(max_workers=2) as executor: + future1 = executor.submit(_task, tmp_watch, update_handler) + future2 = executor.submit(_task, blank_watch_no_filters, update_handler) - text_after_filter = future1.result() - text_before_filter = future2.result() + text_after_filter = future1.result() + text_before_filter = future2.result() + except Exception as e: + x=1 try: trigger_line_numbers = html_tools.strip_ignore_text(content=text_after_filter, diff --git a/changedetectionio/run_basic_tests.sh b/changedetectionio/run_basic_tests.sh index d13d2dab..420f1f60 100755 --- a/changedetectionio/run_basic_tests.sh +++ b/changedetectionio/run_basic_tests.sh @@ -10,8 +10,9 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + # REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser -REMOVE_REQUESTS_OLD_SCREENSHOTS=false time pytest -n 16 --dist load --tb=long tests/test_*.py +REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -n 30 --dist load tests/test_*.py #time pytest -n auto --dist loadfile -vv --tb=long tests/test_*.py echo "RUNNING WITH BASE_URL SET" diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 27deda67..d581db17 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -421,7 +421,6 @@ class ChangeDetectionStore: self.sync_to_json() return else: - try: # Re #286 - First write to a temp file, then confirm it looks OK and rename it # This is a fairly basic strategy to deal with the case that the file is corrupted, diff --git a/changedetectionio/tests/conftest.py b/changedetectionio/tests/conftest.py index 6fa12c55..3ac99176 100644 --- a/changedetectionio/tests/conftest.py +++ b/changedetectionio/tests/conftest.py @@ -11,6 +11,7 @@ import os import sys from loguru import logger +from changedetectionio.flask_app import init_app_secret from changedetectionio.tests.util import live_server_setup, new_live_server_setup # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py @@ -131,6 +132,13 @@ def prepare_test_function(live_server, datastore_path): # CRITICAL: Point app to THIS test's unique datastore directory live_server.app.config['TEST_DATASTORE_PATH'] = datastore_path + # CRITICAL: Get datastore and stop it from writing stale data + datastore = live_server.app.config.get('DATASTORE') + + # Prevent background thread from writing during cleanup/reload + datastore.needs_write = False + datastore.needs_write_urgent = False + # CRITICAL: Clean up any files from previous tests # This ensures a completely clean directory cleanup(datastore_path) @@ -138,7 +146,6 @@ def prepare_test_function(live_server, datastore_path): # CRITICAL: Reload the EXISTING datastore instead of creating a new one # This keeps blueprint references valid (they capture datastore at construction) # reload_state() completely resets the datastore to a clean state - datastore = live_server.app.config.get('DATASTORE') # Reload state with clean data (no default watches) datastore.reload_state( @@ -146,7 +153,7 @@ def prepare_test_function(live_server, datastore_path): include_default_watches=False, version_tag=datastore.data.get('version_tag', '0.0.0') ) - + live_server.app.secret_key = init_app_secret(datastore_path) logger.debug(f"prepare_test_function: Reloaded datastore at {hex(id(datastore))}") logger.debug(f"prepare_test_function: Path {datastore.datastore_path}") diff --git a/changedetectionio/tests/test_add_replace_remove_filter.py b/changedetectionio/tests/test_add_replace_remove_filter.py index 4ef9bf0c..5f715e33 100644 --- a/changedetectionio/tests/test_add_replace_remove_filter.py +++ b/changedetectionio/tests/test_add_replace_remove_filter.py @@ -65,6 +65,7 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory "time_between_check_use_default": "y"}, follow_redirects=True ) + assert b"Updated watch." in res.data wait_for_all_checks(client) set_original(excluding='Something irrelevant', datastore_path=datastore_path) diff --git a/changedetectionio/tests/test_backup.py b/changedetectionio/tests/test_backup.py index df701ff5..e7bbc7a1 100644 --- a/changedetectionio/tests/test_backup.py +++ b/changedetectionio/tests/test_backup.py @@ -14,9 +14,6 @@ def test_backup(client, live_server, measure_memory_usage, datastore_path): set_original_response(datastore_path=datastore_path) - # Give the endpoint time to spin up - time.sleep(1) - # Add our URL to the import page res = client.post( url_for("imports.import_page"), @@ -32,7 +29,7 @@ def test_backup(client, live_server, measure_memory_usage, datastore_path): url_for("backups.request_backup"), follow_redirects=True ) - time.sleep(2) + time.sleep(4) res = client.get( url_for("backups.index"), diff --git a/changedetectionio/tests/test_block_while_text_present.py b/changedetectionio/tests/test_block_while_text_present.py index e205ab05..485732ed 100644 --- a/changedetectionio/tests/test_block_while_text_present.py +++ b/changedetectionio/tests/test_block_while_text_present.py @@ -136,8 +136,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("watchlist.index")) - with open('/tmp/fuck.html', 'wb') as f: - f.write(res.data) + assert b'has-unread-changes' in res.data diff --git a/changedetectionio/tests/test_filter_failure_notification.py b/changedetectionio/tests/test_filter_failure_notification.py index 0dfae269..99290b1d 100644 --- a/changedetectionio/tests/test_filter_failure_notification.py +++ b/changedetectionio/tests/test_filter_failure_notification.py @@ -44,8 +44,7 @@ def run_filter_test(client, live_server, content_filter, app_notification_format uuid = client.application.config.get('DATASTORE').add_watch(url=test_url) res = client.get(url_for("watchlist.index")) - with open('/tmp/fuck.html', 'wb') as f: - f.write(res.data) + assert b'No website watches configured' not in res.data diff --git a/changedetectionio/tests/test_live_preview.py b/changedetectionio/tests/test_live_preview.py index 1fe6fd6d..8c8f874b 100644 --- a/changedetectionio/tests/test_live_preview.py +++ b/changedetectionio/tests/test_live_preview.py @@ -21,7 +21,7 @@ something to trigger
def test_content_filter_live_preview(client, live_server, measure_memory_usage, datastore_path): # live_server_setup(live_server) # Setup on conftest per function - set_response(datastore_path) + set_response(datastore_path=datastore_path) test_url = url_for('test_endpoint', _external=True) diff --git a/changedetectionio/tests/test_nonrenderable_pages.py b/changedetectionio/tests/test_nonrenderable_pages.py index d62525ec..299f5412 100644 --- a/changedetectionio/tests/test_nonrenderable_pages.py +++ b/changedetectionio/tests/test_nonrenderable_pages.py @@ -101,7 +101,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure # A totally zero byte (#2528) response should also not trigger an error - set_zero_byte_response() + set_zero_byte_response(datastore_path=datastore_path) client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) # 2877