From da5585b53c5437706034cacd60f1919abcd18666 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 17 Mar 2025 10:15:45 +0100 Subject: [PATCH 01/13] Include conditions module --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 47d95225..eaf04a6f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ recursive-include changedetectionio/api * recursive-include changedetectionio/apprise_plugin * recursive-include changedetectionio/blueprint * recursive-include changedetectionio/content_fetchers * +recursive-include changedetectionio/conditions * recursive-include changedetectionio/model * recursive-include changedetectionio/processors * recursive-include changedetectionio/static * From ee7e43ea87b2134d4e4d35e5d156546575c57e61 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 17 Mar 2025 10:47:41 +0100 Subject: [PATCH 02/13] Logic fix --- changedetectionio/conditions/__init__.py | 12 ++++++------ changedetectionio/conditions/default_plugin.py | 5 +++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/changedetectionio/conditions/__init__.py b/changedetectionio/conditions/__init__.py index d0c02641..6c66c218 100644 --- a/changedetectionio/conditions/__init__.py +++ b/changedetectionio/conditions/__init__.py @@ -55,7 +55,7 @@ def filter_complete_rules(ruleset): ] return rules -def convert_to_jsonlogic(rule_dict: list): +def convert_to_jsonlogic(logic_operator: str, rule_dict: list): """ Convert a structured rule dict into a JSON Logic rule. @@ -64,9 +64,6 @@ def convert_to_jsonlogic(rule_dict: list): """ - # Determine the logical operator ("ALL" -> "and", "ANY" -> "or") - logic_operator = "and" if rule_dict.get("conditions_match_logic", "ALL") == "ALL" else "or" - json_logic_conditions = [] for condition in rule_dict: @@ -115,6 +112,7 @@ def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_dat ruleset_settings = application_datastruct['watching'].get(current_watch_uuid) if ruleset_settings.get("conditions"): + logic_operator = "and" if ruleset_settings.get("conditions_match_logic", "ALL") == "ALL" else "or" complete_rules = filter_complete_rules(ruleset_settings['conditions']) if complete_rules: # Give all plugins a chance to update the data dict again (that we will test the conditions against) @@ -126,8 +124,10 @@ def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_dat if new_execute_data and isinstance(new_execute_data, dict): EXECUTE_DATA.update(new_execute_data) - ruleset = convert_to_jsonlogic(rule_dict=complete_rules) - result = jsonLogic(logic=ruleset, data=EXECUTE_DATA) + ruleset = convert_to_jsonlogic(logic_operator=logic_operator, rule_dict=complete_rules) + + if not jsonLogic(logic=ruleset, data=EXECUTE_DATA): + result = False return result diff --git a/changedetectionio/conditions/default_plugin.py b/changedetectionio/conditions/default_plugin.py index eb0cb004..f248a729 100644 --- a/changedetectionio/conditions/default_plugin.py +++ b/changedetectionio/conditions/default_plugin.py @@ -40,12 +40,13 @@ def add_data(current_watch_uuid, application_datastruct, ephemeral_data): res = {} if 'text' in ephemeral_data: - res['page_text'] = ephemeral_data['text'] + res['page_filtered_text'] = ephemeral_data['text'] # Better to not wrap this in try/except so that the UI can see any errors price = Price.fromstring(ephemeral_data.get('text')) if price and price.amount != None: + # This is slightly misleading, it's extracting a PRICE not a Number.. res['extracted_number'] = float(price.amount) - logger.debug(f"Extracted price result: '{price}' - returning float({res['extracted_number']})") + logger.debug(f"Extracted number result: '{price}' - returning float({res['extracted_number']})") return res From c982395d72f39f45fb9032856df678ab42308211 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 17 Mar 2025 11:00:29 +0100 Subject: [PATCH 03/13] WIP --- changedetectionio/tests/test_conditions.py | 216 +++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 changedetectionio/tests/test_conditions.py diff --git a/changedetectionio/tests/test_conditions.py b/changedetectionio/tests/test_conditions.py new file mode 100644 index 00000000..a48fe626 --- /dev/null +++ b/changedetectionio/tests/test_conditions.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 + +import time +from flask import url_for +from .util import live_server_setup, wait_for_all_checks + +def set_original_response(number="50"): + test_return_data = f""" + +

Test Page for Conditions

+

This page contains a number that will be tested with conditions.

+
Current value: {number}
+ + + """ + + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(test_return_data) + +def set_number_in_range_response(number="75"): + test_return_data = f""" + +

Test Page for Conditions

+

This page contains a number that will be tested with conditions.

+
Current value: {number}
+ + + """ + + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(test_return_data) + +def set_number_out_of_range_response(number="150"): + test_return_data = f""" + +

Test Page for Conditions

+

This page contains a number that will be tested with conditions.

+
Current value: {number}
+ + + """ + + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(test_return_data) + +def test_setup(live_server): + live_server_setup(live_server) + +def test_number_within_range_condition(client, live_server, measure_memory_usage): + set_original_response() +# live_server_setup(live_server) + + test_url = url_for('test_endpoint', _external=True) + + # Add our URL to the import page + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + # Configure the watch with two conditions: + # 1. The extracted number should be >= 20 and <= 100 + # 2. The first digit of the number should be in the page filtered text + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "fetch_backend": "html_requests", + "include_filters": ".number-container", + "title": "Condition Test", + "conditions_match_logic": "ALL", # ALL = AND logic + "conditions": [ + {"operator": ">=", "field": "extracted_number", "value": "20"}, + {"operator": "<=", "field": "extracted_number", "value": "100"}, + {"operator": "in", "field": "page_filtered_text", "value": "5"} # First digit of 50 + ] + }, + follow_redirects=True + ) + assert b"Updated watch." in res.data + + # Trigger initial check + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + # Set current view state - no unviewed + client.get(url_for("diff_history_page", uuid="first")) + + # Change that stays within the conditions (number is in range and text contains first digit) + set_number_in_range_response("75") + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + # Should NOT be marked as having changes since it meets all conditions + res = client.get(url_for("index")) + assert b'unviewed' not in res.data + assert b'Condition Test' in res.data + + # Now change to value that's outside the range (but still contains the first digit in text) + set_number_out_of_range_response("150") + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + # SHOULD be marked as having changes since it violates one of the conditions + res = client.get(url_for("index")) + assert b'unviewed' in res.data + assert b'Condition Test' in res.data + + # Check the diff history shows the change + res = client.get(url_for("diff_history_page", uuid="first")) + assert b'Current value: 150' in res.data + + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data + +def test_conditions_with_text_and_number(client, live_server, measure_memory_usage): + """Test that both text and number conditions work together with AND logic.""" + + set_original_response("50") + #live_server_setup(live_server) + + test_url = url_for('test_endpoint', _external=True) + + # Add our URL to the import page + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + # Configure the watch with two conditions connected with AND: + # 1. The page filtered text must contain "5" (first digit of value) + # 2. The extracted number should be >= 20 and <= 100 + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "fetch_backend": "html_requests", + "include_filters": ".number-container", + "title": "Number AND Text Condition Test", + "conditions_match_logic": "ALL", # ALL = AND logic + "conditions": [ + {"operator": "in", "field": "page_filtered_text", "value": "5"}, # First digit of 50 + {"operator": ">=", "field": "extracted_number", "value": "20"}, + {"operator": "<=", "field": "extracted_number", "value": "100"} + ] + }, + follow_redirects=True + ) + assert b"Updated watch." in res.data + + # Trigger initial check + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + # Set current view state - no unviewed + client.get(url_for("diff_history_page", uuid="first")) + + # Case 1: The number is in range but the first digit changed (now 7 instead of 5) + # This should trigger a change notification since the text condition is violated + set_number_in_range_response("75") + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + # Should be marked as having changes since it violates the text condition + res = client.get(url_for("index")) + assert b'unviewed' in res.data + + # Reset view state + client.get(url_for("diff_history_page", uuid="first")) + + # Case 2: Change with both conditions satisfied + # Number in range (80) and contains first digit "8" in text + set_number_in_range_response("80") + + # Update the conditions to match the new value's first digit + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "fetch_backend": "html_requests", + "include_filters": ".number-container", + "title": "Number AND Text Condition Test", + "conditions_match_logic": "ALL", # ALL = AND logic + "conditions": [ + {"operator": "in", "field": "page_filtered_text", "value": "8"}, # First digit of 80 + {"operator": ">=", "field": "extracted_number", "value": "20"}, + {"operator": "<=", "field": "extracted_number", "value": "100"} + ] + }, + follow_redirects=True + ) + assert b"Updated watch." in res.data + + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + # Should NOT be marked as having changes since both conditions are met + res = client.get(url_for("index")) + assert b'unviewed' not in res.data + + # Case 3: Change with both conditions violated + # Number out of range (150) and first digit doesn't match condition + set_number_out_of_range_response("150") + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + # SHOULD be marked as having changes since both conditions are violated + res = client.get(url_for("index")) + assert b'unviewed' in res.data + + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data \ No newline at end of file From 2608980b1d3661fc374f580da3e1ea021b77846c Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 17 Mar 2025 11:20:23 +0100 Subject: [PATCH 04/13] test fixup --- changedetectionio/tests/test_conditions.py | 95 +++++++++------------- 1 file changed, 39 insertions(+), 56 deletions(-) diff --git a/changedetectionio/tests/test_conditions.py b/changedetectionio/tests/test_conditions.py index a48fe626..3933c6ce 100644 --- a/changedetectionio/tests/test_conditions.py +++ b/changedetectionio/tests/test_conditions.py @@ -71,11 +71,19 @@ def test_number_within_range_condition(client, live_server, measure_memory_usage "include_filters": ".number-container", "title": "Condition Test", "conditions_match_logic": "ALL", # ALL = AND logic - "conditions": [ - {"operator": ">=", "field": "extracted_number", "value": "20"}, - {"operator": "<=", "field": "extracted_number", "value": "100"}, - {"operator": "in", "field": "page_filtered_text", "value": "5"} # First digit of 50 - ] + + "conditions-0-operator": "in", + "conditions-0-field": "page_filtered_text", + "conditions-0-value": "5", + + "conditions-1-operator": ">=", + "conditions-1-field": "extracted_number", + "conditions-1-value": "20", + + "conditions-2-operator": "<=", + "conditions-2-field": "extracted_number", + "conditions-2-value": "100", + }, follow_redirects=True ) @@ -93,9 +101,9 @@ def test_number_within_range_condition(client, live_server, measure_memory_usage client.get(url_for("form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) - # Should NOT be marked as having changes since it meets all conditions + # Should be marked as having changes since all conditions are met res = client.get(url_for("index")) - assert b'unviewed' not in res.data + assert b'unviewed' in res.data assert b'Condition Test' in res.data # Now change to value that's outside the range (but still contains the first digit in text) @@ -103,14 +111,14 @@ def test_number_within_range_condition(client, live_server, measure_memory_usage client.get(url_for("form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) - # SHOULD be marked as having changes since it violates one of the conditions + # Should NOT be marked as having changes since one condition is not met res = client.get(url_for("index")) - assert b'unviewed' in res.data + assert b'unviewed' not in res.data assert b'Condition Test' in res.data - # Check the diff history shows the change + # Check the diff history shows the change from the earlier check (75) res = client.get(url_for("diff_history_page", uuid="first")) - assert b'Current value: 150' in res.data + assert b'Current value: 75' in res.data res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -142,11 +150,18 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa "include_filters": ".number-container", "title": "Number AND Text Condition Test", "conditions_match_logic": "ALL", # ALL = AND logic - "conditions": [ - {"operator": "in", "field": "page_filtered_text", "value": "5"}, # First digit of 50 - {"operator": ">=", "field": "extracted_number", "value": "20"}, - {"operator": "<=", "field": "extracted_number", "value": "100"} - ] + "conditions-0-operator": "in", + "conditions-0-field": "page_filtered_text", + "conditions-0-value": "5", + + "conditions-1-operator": ">=", + "conditions-1-field": "extracted_number", + "conditions-1-value": "20", + + "conditions-2-operator": "<=", + "conditions-2-field": "extracted_number", + "conditions-2-value": "100", + }, follow_redirects=True ) @@ -155,62 +170,30 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa # Trigger initial check client.get(url_for("form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) + client.get(url_for("mark_all_viewed"), follow_redirects=True) - # Set current view state - no unviewed - client.get(url_for("diff_history_page", uuid="first")) # Case 1: The number is in range but the first digit changed (now 7 instead of 5) - # This should trigger a change notification since the text condition is violated + # This should NOT trigger a change notification since not all conditions are met set_number_in_range_response("75") client.get(url_for("form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) - # Should be marked as having changes since it violates the text condition + # 75 > 20 and < 100 and contains "5" res = client.get(url_for("index")) assert b'unviewed' in res.data - - # Reset view state - client.get(url_for("diff_history_page", uuid="first")) - # Case 2: Change with both conditions satisfied - # Number in range (80) and contains first digit "8" in text - set_number_in_range_response("80") - - # Update the conditions to match the new value's first digit - res = client.post( - url_for("edit_page", uuid="first"), - data={ - "url": test_url, - "fetch_backend": "html_requests", - "include_filters": ".number-container", - "title": "Number AND Text Condition Test", - "conditions_match_logic": "ALL", # ALL = AND logic - "conditions": [ - {"operator": "in", "field": "page_filtered_text", "value": "8"}, # First digit of 80 - {"operator": ">=", "field": "extracted_number", "value": "20"}, - {"operator": "<=", "field": "extracted_number", "value": "100"} - ] - }, - follow_redirects=True - ) - assert b"Updated watch." in res.data - - client.get(url_for("form_watch_checknow"), follow_redirects=True) - wait_for_all_checks(client) - - # Should NOT be marked as having changes since both conditions are met - res = client.get(url_for("index")) - assert b'unviewed' not in res.data - # Case 3: Change with both conditions violated - # Number out of range (150) and first digit doesn't match condition + # Case 2: Change with one condition violated + # Number out of range (150) but contains '5' + client.get(url_for("mark_all_viewed"), follow_redirects=True) set_number_out_of_range_response("150") client.get(url_for("form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) - # SHOULD be marked as having changes since both conditions are violated + # Should NOT be marked as having changes since not all conditions are met res = client.get(url_for("index")) - assert b'unviewed' in res.data + assert b'unviewed' not in res.data res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data \ No newline at end of file From 4f489581872c9eeb6909c3338cac4fde2f3296a2 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 17 Mar 2025 11:48:35 +0100 Subject: [PATCH 05/13] WIP --- changedetectionio/conditions/__init__.py | 73 +++++++++++++++++++ changedetectionio/flask_app.py | 3 + changedetectionio/static/js/conditions.js | 70 +++++++++++++++++- changedetectionio/templates/_helpers.html | 1 + changedetectionio/templates/edit.html | 15 ++++ changedetectionio/tests/test_conditions.py | 85 +--------------------- 6 files changed, 163 insertions(+), 84 deletions(-) diff --git a/changedetectionio/conditions/__init__.py b/changedetectionio/conditions/__init__.py index 6c66c218..1ced1c93 100644 --- a/changedetectionio/conditions/__init__.py +++ b/changedetectionio/conditions/__init__.py @@ -1,3 +1,5 @@ +from flask import Blueprint, request, jsonify, Response +from flask_login import current_user from json_logic.builtins import BUILTINS from .exceptions import EmptyConditionRuleRowNotUsable @@ -5,6 +7,7 @@ from .pluggy_interface import plugin_manager # Import the pluggy plugin manager from . import default_plugin import re +import json # List of all supported JSON Logic operators operator_choices = [ @@ -131,6 +134,76 @@ def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_dat return result +# Flask Blueprint Definition +def construct_blueprint(datastore): + from changedetectionio.flask_app import login_optionally_required + + conditions_blueprint = Blueprint('conditions', __name__, template_folder="templates") + + @conditions_blueprint.route("//verify-condition-single-rule", methods=['POST']) + @login_optionally_required + def verify_condition_single_rule(watch_uuid): + """Verify a single condition rule against the current snapshot""" + + # Get the watch data + watch = datastore.data['watching'].get(watch_uuid) + if not watch: + return jsonify({'status': 'error', 'message': 'Watch not found'}), 404 + + # Get the rule data from the request + rule_data = request.json + if not rule_data: + return jsonify({'status': 'error', 'message': 'No rule data provided'}), 400 + + # Create ephemeral data with the current snapshot + ephemeral_data = {} + + # Get the current snapshot if available + if watch.history_n and watch.get_last_fetched_text_before_filters(): + ephemeral_data['text'] = watch.get_last_fetched_text_before_filters() + else: + return jsonify({ + 'status': 'error', + 'message': 'No snapshot available for verification. Please fetch content first.' + }), 400 + + # Test the rule + result = False + try: + # Create a temporary structure with just this rule + temp_watch_data = { + "conditions": [rule_data], + "conditions_match_logic": "ALL" # Single rule, so use ALL + } + + # Create a temporary application data structure + temp_app_data = { + 'watching': { + watch_uuid: temp_watch_data + } + } + + # Execute the rule against the current snapshot + result = execute_ruleset_against_all_plugins( + current_watch_uuid=watch_uuid, + application_datastruct=temp_app_data, + ephemeral_data=ephemeral_data + ) + + return jsonify({ + 'status': 'success', + 'result': result, + 'message': 'Condition passes' if result else 'Condition does not pass' + }) + + except Exception as e: + return jsonify({ + 'status': 'error', + 'message': f'Error verifying condition: {str(e)}' + }), 500 + + return conditions_blueprint + # Load plugins dynamically for plugin in plugin_manager.get_plugins(): new_ops = plugin.register_operators() diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index a88df476..7101e0c0 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -1684,6 +1684,9 @@ def changedetection_app(config=None, datastore_o=None): import changedetectionio.blueprint.backups as backups app.register_blueprint(backups.construct_blueprint(datastore), url_prefix='/backups') + import changedetectionio.conditions as conditions + app.register_blueprint(conditions.construct_blueprint(datastore), url_prefix='/conditions') + # @todo handle ctrl break ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() threading.Thread(target=notification_runner).start() diff --git a/changedetectionio/static/js/conditions.js b/changedetectionio/static/js/conditions.js index 1b983b51..2cb10b84 100644 --- a/changedetectionio/static/js/conditions.js +++ b/changedetectionio/static/js/conditions.js @@ -2,7 +2,7 @@ $(document).ready(function () { // Function to set up button event handlers function setupButtonHandlers() { // Unbind existing handlers first to prevent duplicates - $(".addRuleRow, .removeRuleRow").off("click"); + $(".addRuleRow, .removeRuleRow, .verifyRuleRow").off("click"); // Add row button handler $(".addRuleRow").on("click", function(e) { @@ -34,12 +34,78 @@ $(document).ready(function () { reindexRules(); } }); + + // Verify rule button handler + $(".verifyRuleRow").on("click", function(e) { + e.preventDefault(); + + let row = $(this).closest("tr"); + let field = row.find("select[name$='field']").val(); + let operator = row.find("select[name$='operator']").val(); + let value = row.find("input[name$='value']").val(); + + // Validate that all fields are filled + if (!field || field === "None" || !operator || operator === "None" || !value) { + alert("Please fill in all fields (Field, Operator, and Value) before verifying."); + return; + } + + // Extract the watch UUID from the URL + const url = window.location.pathname; + const uuidMatch = url.match(/\/edit\/([^\/]+)/); + if (!uuidMatch || !uuidMatch[1]) { + alert("Could not determine the watch UUID. Please save your changes first."); + return; + } + + const watchUuid = uuidMatch[1]; + + // Create a rule object + const rule = { + field: field, + operator: operator, + value: value + }; + + // Show a spinner or some indication that verification is in progress + const $button = $(this); + const originalHTML = $button.html(); + $button.html("⌛").prop("disabled", true); + + // Send the request to verify the rule + $.ajax({ + url: `/conditions/${watchUuid}/verify-condition-single-rule`, + type: "POST", + contentType: "application/json", + data: JSON.stringify(rule), + success: function(response) { + if (response.status === "success") { + if (response.result) { + alert("✅ Condition PASSES verification against current snapshot!"); + } else { + alert("❌ Condition FAILS verification against current snapshot."); + } + } else { + alert("Error: " + response.message); + } + $button.html(originalHTML).prop("disabled", false); + }, + error: function(xhr) { + let errorMsg = "Error verifying condition."; + if (xhr.responseJSON && xhr.responseJSON.message) { + errorMsg = xhr.responseJSON.message; + } + alert(errorMsg); + $button.html(originalHTML).prop("disabled", false); + } + }); + }); } // Function to reindex form elements and re-setup event handlers function reindexRules() { // Unbind all button handlers first - $(".addRuleRow, .removeRuleRow").off("click"); + $(".addRuleRow, .removeRuleRow, .verifyRuleRow").off("click"); // Reindex all form elements $("#rulesTable tbody tr").each(function(index) { diff --git a/changedetectionio/templates/_helpers.html b/changedetectionio/templates/_helpers.html index d0eec6df..461eb22e 100644 --- a/changedetectionio/templates/_helpers.html +++ b/changedetectionio/templates/_helpers.html @@ -89,6 +89,7 @@ + {% endfor %} diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 3f7027fd..65c71231 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -286,9 +286,24 @@ Math: {{ 1 + 1 }}") }} {% if watch['processor'] == 'text_json_diff' %}
+
{{ render_field(form.conditions_match_logic) }} {{ render_fieldlist_of_formfields_as_table(form.conditions) }} +
+ Use the verify (✓) button to test if a condition passes against the current snapshot. +
diff --git a/changedetectionio/tests/test_conditions.py b/changedetectionio/tests/test_conditions.py index 3933c6ce..b30f667b 100644 --- a/changedetectionio/tests/test_conditions.py +++ b/changedetectionio/tests/test_conditions.py @@ -43,91 +43,12 @@ def set_number_out_of_range_response(number="150"): with open("test-datastore/endpoint-content.txt", "w") as f: f.write(test_return_data) -def test_setup(live_server): - live_server_setup(live_server) - -def test_number_within_range_condition(client, live_server, measure_memory_usage): - set_original_response() -# live_server_setup(live_server) - - test_url = url_for('test_endpoint', _external=True) - - # Add our URL to the import page - res = client.post( - url_for("import_page"), - data={"urls": test_url}, - follow_redirects=True - ) - assert b"1 Imported" in res.data - - # Configure the watch with two conditions: - # 1. The extracted number should be >= 20 and <= 100 - # 2. The first digit of the number should be in the page filtered text - res = client.post( - url_for("edit_page", uuid="first"), - data={ - "url": test_url, - "fetch_backend": "html_requests", - "include_filters": ".number-container", - "title": "Condition Test", - "conditions_match_logic": "ALL", # ALL = AND logic - - "conditions-0-operator": "in", - "conditions-0-field": "page_filtered_text", - "conditions-0-value": "5", - - "conditions-1-operator": ">=", - "conditions-1-field": "extracted_number", - "conditions-1-value": "20", - - "conditions-2-operator": "<=", - "conditions-2-field": "extracted_number", - "conditions-2-value": "100", - - }, - follow_redirects=True - ) - assert b"Updated watch." in res.data - - # Trigger initial check - client.get(url_for("form_watch_checknow"), follow_redirects=True) - wait_for_all_checks(client) - - # Set current view state - no unviewed - client.get(url_for("diff_history_page", uuid="first")) - - # Change that stays within the conditions (number is in range and text contains first digit) - set_number_in_range_response("75") - client.get(url_for("form_watch_checknow"), follow_redirects=True) - wait_for_all_checks(client) - - # Should be marked as having changes since all conditions are met - res = client.get(url_for("index")) - assert b'unviewed' in res.data - assert b'Condition Test' in res.data - - # Now change to value that's outside the range (but still contains the first digit in text) - set_number_out_of_range_response("150") - client.get(url_for("form_watch_checknow"), follow_redirects=True) - wait_for_all_checks(client) - - # Should NOT be marked as having changes since one condition is not met - res = client.get(url_for("index")) - assert b'unviewed' not in res.data - assert b'Condition Test' in res.data - - # Check the diff history shows the change from the earlier check (75) - res = client.get(url_for("diff_history_page", uuid="first")) - assert b'Current value: 75' in res.data - - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) - assert b'Deleted' in res.data def test_conditions_with_text_and_number(client, live_server, measure_memory_usage): """Test that both text and number conditions work together with AND logic.""" set_original_response("50") - #live_server_setup(live_server) + live_server_setup(live_server) test_url = url_for('test_endpoint', _external=True) @@ -175,7 +96,7 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa # Case 1: The number is in range but the first digit changed (now 7 instead of 5) # This should NOT trigger a change notification since not all conditions are met - set_number_in_range_response("75") + set_number_in_range_response("70.5") client.get(url_for("form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) @@ -187,7 +108,7 @@ def test_conditions_with_text_and_number(client, live_server, measure_memory_usa # Case 2: Change with one condition violated # Number out of range (150) but contains '5' client.get(url_for("mark_all_viewed"), follow_redirects=True) - set_number_out_of_range_response("150") + set_number_out_of_range_response("150.5") client.get(url_for("form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) From 71ea8d80f3114cb89406e8000106966f117eceb5 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 17 Mar 2025 17:21:56 +0100 Subject: [PATCH 06/13] WIP --- changedetectionio/conditions/__init__.py | 63 ++++++++++--------- changedetectionio/flask_app.py | 4 +- .../processors/text_json_diff/__init__.py | 17 +++-- changedetectionio/static/js/conditions.js | 46 +++++++++----- changedetectionio/templates/edit.html | 3 + 5 files changed, 78 insertions(+), 55 deletions(-) diff --git a/changedetectionio/conditions/__init__.py b/changedetectionio/conditions/__init__.py index 1ced1c93..d66cd55c 100644 --- a/changedetectionio/conditions/__init__.py +++ b/changedetectionio/conditions/__init__.py @@ -144,46 +144,49 @@ def construct_blueprint(datastore): @login_optionally_required def verify_condition_single_rule(watch_uuid): """Verify a single condition rule against the current snapshot""" - + from changedetectionio.processors.text_json_diff import prepare_filter_prevew + from flask import request, jsonify + from copy import deepcopy + + ephemeral_data = {} + # Get the watch data watch = datastore.data['watching'].get(watch_uuid) if not watch: return jsonify({'status': 'error', 'message': 'Watch not found'}), 404 - - # Get the rule data from the request - rule_data = request.json - if not rule_data: - return jsonify({'status': 'error', 'message': 'No rule data provided'}), 400 - - # Create ephemeral data with the current snapshot - ephemeral_data = {} - - # Get the current snapshot if available - if watch.history_n and watch.get_last_fetched_text_before_filters(): - ephemeral_data['text'] = watch.get_last_fetched_text_before_filters() - else: - return jsonify({ - 'status': 'error', - 'message': 'No snapshot available for verification. Please fetch content first.' - }), 400 - - # Test the rule - result = False - try: - # Create a temporary structure with just this rule - temp_watch_data = { - "conditions": [rule_data], - "conditions_match_logic": "ALL" # Single rule, so use ALL - } - # Create a temporary application data structure + # First use prepare_filter_prevew to process the form data + # This will return text_after_filter which is after all current form settings are applied + # Create ephemeral data with the text from the current snapshot + + try: + # Call prepare_filter_prevew to get a processed version of the content with current form settings + # We'll ignore the returned response and just use the datastore which is modified by the function + + # this should apply all filters etc so then we can run the CONDITIONS against the final output text + result = prepare_filter_prevew(datastore=datastore, + form_data=request.form, + watch_uuid=watch_uuid) + + ephemeral_data['text'] = result.get('after_filter', '') + # Create a temporary watch data structure with this single rule + tmp_watch_data = deepcopy(datastore.data['watching'].get(watch_uuid)) + + # Override the conditions in the temporary watch + rule_json = request.args.get("rule") + rule = json.loads(rule_json) if rule_json else None + tmp_watch_data['conditions'] = [rule] + tmp_watch_data['conditions_match_logic'] = "ALL" # Single rule, so use ALL + + + # Create a temporary application data structure for the rule check temp_app_data = { 'watching': { - watch_uuid: temp_watch_data + watch_uuid: tmp_watch_data } } - # Execute the rule against the current snapshot + # Execute the rule against the current snapshot with form data result = execute_ruleset_against_all_plugins( current_watch_uuid=watch_uuid, application_datastruct=temp_app_data, diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 7101e0c0..52bc5ca6 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -1382,8 +1382,10 @@ def changedetection_app(config=None, datastore_o=None): @login_optionally_required def watch_get_preview_rendered(uuid): '''For when viewing the "preview" of the rendered text from inside of Edit''' + from flask import jsonify from .processors.text_json_diff import prepare_filter_prevew - return prepare_filter_prevew(watch_uuid=uuid, datastore=datastore) + result = prepare_filter_prevew(watch_uuid=uuid, form_data=request.form, datastore=datastore) + return jsonify(result) @app.route("/form/add/quickwatch", methods=['POST']) diff --git a/changedetectionio/processors/text_json_diff/__init__.py b/changedetectionio/processors/text_json_diff/__init__.py index a6e018fd..8e5bdfc8 100644 --- a/changedetectionio/processors/text_json_diff/__init__.py +++ b/changedetectionio/processors/text_json_diff/__init__.py @@ -28,13 +28,13 @@ def _task(watch, update_handler): return text_after_filter -def prepare_filter_prevew(datastore, watch_uuid): +def prepare_filter_prevew(datastore, watch_uuid, form_data): '''Used by @app.route("/edit//preview-rendered", methods=['POST'])''' from changedetectionio import forms, html_tools from changedetectionio.model.Watch import model as watch_model from concurrent.futures import ProcessPoolExecutor from copy import deepcopy - from flask import request, jsonify + from flask import request import brotli import importlib import os @@ -50,12 +50,12 @@ def prepare_filter_prevew(datastore, watch_uuid): if tmp_watch and tmp_watch.history and os.path.isdir(tmp_watch.watch_data_dir): # Splice in the temporary stuff from the form - form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None, - data=request.form + form = forms.processor_text_json_diff_form(formdata=form_data if request.method == 'POST' else None, + data=form_data ) # Only update vars that came in via the AJAX post - p = {k: v for k, v in form.data.items() if k in request.form.keys()} + p = {k: v for k, v in form.data.items() if k in form_data.keys()} tmp_watch.update(p) blank_watch_no_filters = watch_model() blank_watch_no_filters['url'] = tmp_watch.get('url') @@ -103,13 +103,12 @@ def prepare_filter_prevew(datastore, watch_uuid): logger.trace(f"Parsed in {time.time() - now:.3f}s") - return jsonify( - { + return ({ 'after_filter': text_after_filter, 'before_filter': text_before_filter.decode('utf-8') if isinstance(text_before_filter, bytes) else text_before_filter, 'duration': time.time() - now, 'trigger_line_numbers': trigger_line_numbers, 'ignore_line_numbers': ignore_line_numbers, - } - ) + }) + diff --git a/changedetectionio/static/js/conditions.js b/changedetectionio/static/js/conditions.js index 2cb10b84..8c627d63 100644 --- a/changedetectionio/static/js/conditions.js +++ b/changedetectionio/static/js/conditions.js @@ -49,16 +49,7 @@ $(document).ready(function () { alert("Please fill in all fields (Field, Operator, and Value) before verifying."); return; } - - // Extract the watch UUID from the URL - const url = window.location.pathname; - const uuidMatch = url.match(/\/edit\/([^\/]+)/); - if (!uuidMatch || !uuidMatch[1]) { - alert("Could not determine the watch UUID. Please save your changes first."); - return; - } - - const watchUuid = uuidMatch[1]; + // Create a rule object const rule = { @@ -72,13 +63,38 @@ $(document).ready(function () { const originalHTML = $button.html(); $button.html("⌛").prop("disabled", true); + // Collect form data - similar to request_textpreview_update() in watch-settings.js + let formData = new FormData(); + $('#edit-text-filter textarea, #edit-text-filter input').each(function() { + const $element = $(this); + const name = $element.attr('name'); + if (name) { + if ($element.is(':checkbox')) { + formData.append(name, $element.is(':checked') ? $element.val() : false); + } else { + formData.append(name, $element.val()); + } + } + }); + + // Also collect select values + $('#edit-text-filter select').each(function() { + const $element = $(this); + const name = $element.attr('name'); + if (name) { + formData.append(name, $element.val()); + } + }); + + // Send the request to verify the rule $.ajax({ - url: `/conditions/${watchUuid}/verify-condition-single-rule`, + url: verify_condition_rule_url+"?"+ new URLSearchParams({ rule: JSON.stringify(rule) }).toString(), type: "POST", - contentType: "application/json", - data: JSON.stringify(rule), - success: function(response) { + data: formData, + processData: false, // Prevent jQuery from converting FormData to a string + contentType: false, // Let the browser set the correct content type + success: function (response) { if (response.status === "success") { if (response.result) { alert("✅ Condition PASSES verification against current snapshot!"); @@ -90,7 +106,7 @@ $(document).ready(function () { } $button.html(originalHTML).prop("disabled", false); }, - error: function(xhr) { + error: function (xhr) { let errorMsg = "Error verifying condition."; if (xhr.responseJSON && xhr.responseJSON.message) { errorMsg = xhr.responseJSON.message; diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 65c71231..4b5e4698 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -286,6 +286,9 @@ Math: {{ 1 + 1 }}") }} {% if watch['processor'] == 'text_json_diff' %}
+