diff --git a/MANIFEST.in b/MANIFEST.in index 47d95225..eaf04a6f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ recursive-include changedetectionio/api * recursive-include changedetectionio/apprise_plugin * recursive-include changedetectionio/blueprint * recursive-include changedetectionio/content_fetchers * +recursive-include changedetectionio/conditions * recursive-include changedetectionio/model * recursive-include changedetectionio/processors * recursive-include changedetectionio/static * diff --git a/changedetectionio/conditions/__init__.py b/changedetectionio/conditions/__init__.py index d0c02641..44ebd002 100644 --- a/changedetectionio/conditions/__init__.py +++ b/changedetectionio/conditions/__init__.py @@ -1,11 +1,11 @@ +from flask import Blueprint + from json_logic.builtins import BUILTINS from .exceptions import EmptyConditionRuleRowNotUsable from .pluggy_interface import plugin_manager # Import the pluggy plugin manager from . import default_plugin -import re - # List of all supported JSON Logic operators operator_choices = [ (None, "Choose one"), @@ -30,22 +30,10 @@ field_choices = [ # The data we will feed the JSON Rules to see if it passes the test/conditions or not EXECUTE_DATA = {} -# ✅ Custom function for case-insensitive regex matching -def contains_regex(_, text, pattern): - """Returns True if `text` contains `pattern` (case-insensitive regex match).""" - return bool(re.search(pattern, text, re.IGNORECASE)) - -# ✅ Custom function for NOT matching case-insensitive regex -def not_contains_regex(_, text, pattern): - """Returns True if `text` does NOT contain `pattern` (case-insensitive regex match).""" - return not bool(re.search(pattern, text, re.IGNORECASE)) - # Define the extended operations dictionary CUSTOM_OPERATIONS = { **BUILTINS, # Include all standard operators - "contains_regex": contains_regex, - "!contains_regex": not_contains_regex } def filter_complete_rules(ruleset): @@ -55,7 +43,7 @@ def filter_complete_rules(ruleset): ] return rules -def convert_to_jsonlogic(rule_dict: list): +def convert_to_jsonlogic(logic_operator: str, rule_dict: list): """ Convert a structured rule dict into a JSON Logic rule. @@ -64,9 +52,6 @@ def convert_to_jsonlogic(rule_dict: list): """ - # Determine the logical operator ("ALL" -> "and", "ANY" -> "or") - logic_operator = "and" if rule_dict.get("conditions_match_logic", "ALL") == "ALL" else "or" - json_logic_conditions = [] for condition in rule_dict: @@ -115,6 +100,7 @@ def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_dat ruleset_settings = application_datastruct['watching'].get(current_watch_uuid) if ruleset_settings.get("conditions"): + logic_operator = "and" if ruleset_settings.get("conditions_match_logic", "ALL") == "ALL" else "or" complete_rules = filter_complete_rules(ruleset_settings['conditions']) if complete_rules: # Give all plugins a chance to update the data dict again (that we will test the conditions against) @@ -126,11 +112,16 @@ def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_dat if new_execute_data and isinstance(new_execute_data, dict): EXECUTE_DATA.update(new_execute_data) - ruleset = convert_to_jsonlogic(rule_dict=complete_rules) - result = jsonLogic(logic=ruleset, data=EXECUTE_DATA) + # Create the ruleset + ruleset = convert_to_jsonlogic(logic_operator=logic_operator, rule_dict=complete_rules) + + # Pass the custom operations dictionary to jsonLogic + if not jsonLogic(logic=ruleset, data=EXECUTE_DATA, operations=CUSTOM_OPERATIONS): + result = False return result + # Load plugins dynamically for plugin in plugin_manager.get_plugins(): new_ops = plugin.register_operators() diff --git a/changedetectionio/conditions/blueprint.py b/changedetectionio/conditions/blueprint.py new file mode 100644 index 00000000..b9fb762e --- /dev/null +++ b/changedetectionio/conditions/blueprint.py @@ -0,0 +1,78 @@ +# Flask Blueprint Definition +import json + +from flask import Blueprint + +from changedetectionio.conditions import execute_ruleset_against_all_plugins + + +def construct_blueprint(datastore): + from changedetectionio.flask_app import login_optionally_required + + conditions_blueprint = Blueprint('conditions', __name__, template_folder="templates") + + @conditions_blueprint.route("//verify-condition-single-rule", methods=['POST']) + @login_optionally_required + def verify_condition_single_rule(watch_uuid): + """Verify a single condition rule against the current snapshot""" + from changedetectionio.processors.text_json_diff import prepare_filter_prevew + from flask import request, jsonify + from copy import deepcopy + + ephemeral_data = {} + + # Get the watch data + watch = datastore.data['watching'].get(watch_uuid) + if not watch: + return jsonify({'status': 'error', 'message': 'Watch not found'}), 404 + + # First use prepare_filter_prevew to process the form data + # This will return text_after_filter which is after all current form settings are applied + # Create ephemeral data with the text from the current snapshot + + try: + # Call prepare_filter_prevew to get a processed version of the content with current form settings + # We'll ignore the returned response and just use the datastore which is modified by the function + + # this should apply all filters etc so then we can run the CONDITIONS against the final output text + result = prepare_filter_prevew(datastore=datastore, + form_data=request.form, + watch_uuid=watch_uuid) + + ephemeral_data['text'] = result.get('after_filter', '') + # Create a temporary watch data structure with this single rule + tmp_watch_data = deepcopy(datastore.data['watching'].get(watch_uuid)) + + # Override the conditions in the temporary watch + rule_json = request.args.get("rule") + rule = json.loads(rule_json) if rule_json else None + tmp_watch_data['conditions'] = [rule] + tmp_watch_data['conditions_match_logic'] = "ALL" # Single rule, so use ALL + + # Create a temporary application data structure for the rule check + temp_app_data = { + 'watching': { + watch_uuid: tmp_watch_data + } + } + + # Execute the rule against the current snapshot with form data + result = execute_ruleset_against_all_plugins( + current_watch_uuid=watch_uuid, + application_datastruct=temp_app_data, + ephemeral_data=ephemeral_data + ) + + return jsonify({ + 'status': 'success', + 'result': result, + 'message': 'Condition passes' if result else 'Condition does not pass' + }) + + except Exception as e: + return jsonify({ + 'status': 'error', + 'message': f'Error verifying condition: {str(e)}' + }), 500 + + return conditions_blueprint \ No newline at end of file diff --git a/changedetectionio/conditions/default_plugin.py b/changedetectionio/conditions/default_plugin.py index eb0cb004..58346b3d 100644 --- a/changedetectionio/conditions/default_plugin.py +++ b/changedetectionio/conditions/default_plugin.py @@ -1,3 +1,5 @@ +import re + import pluggy from price_parser import Price from loguru import logger @@ -8,14 +10,34 @@ hookimpl = pluggy.HookimplMarker("changedetectionio_conditions") @hookimpl def register_operators(): def starts_with(_, text, prefix): - return text.lower().strip().startswith(prefix.lower()) + return text.lower().strip().startswith(str(prefix).strip().lower()) def ends_with(_, text, suffix): - return text.lower().strip().endswith(suffix.lower()) + return text.lower().strip().endswith(str(suffix).strip().lower()) + + def length_min(_, text, strlen): + return len(text) >= int(strlen) + + def length_max(_, text, strlen): + return len(text) <= int(strlen) + + # ✅ Custom function for case-insensitive regex matching + def contains_regex(_, text, pattern): + """Returns True if `text` contains `pattern` (case-insensitive regex match).""" + return bool(re.search(pattern, str(text), re.IGNORECASE)) + + # ✅ Custom function for NOT matching case-insensitive regex + def not_contains_regex(_, text, pattern): + """Returns True if `text` does NOT contain `pattern` (case-insensitive regex match).""" + return not bool(re.search(pattern, str(text), re.IGNORECASE)) return { + "!contains_regex": not_contains_regex, + "contains_regex": contains_regex, + "ends_with": ends_with, + "length_max": length_max, + "length_min": length_min, "starts_with": starts_with, - "ends_with": ends_with } @hookimpl @@ -23,6 +45,8 @@ def register_operator_choices(): return [ ("starts_with", "Text Starts With"), ("ends_with", "Text Ends With"), + ("length_min", "Length minimum"), + ("length_max", "Length maximum"), ] @hookimpl @@ -32,7 +56,7 @@ def register_field_choices(): # ("meta_description", "Meta Description"), # ("meta_keywords", "Meta Keywords"), ("page_filtered_text", "Page text after 'Filters & Triggers'"), - ("page_title", "Page "), # actual page title <title> + #("page_title", "Page <title>"), # actual page title <title> ] @hookimpl @@ -40,12 +64,13 @@ def add_data(current_watch_uuid, application_datastruct, ephemeral_data): res = {} if 'text' in ephemeral_data: - res['page_text'] = ephemeral_data['text'] + res['page_filtered_text'] = ephemeral_data['text'] # Better to not wrap this in try/except so that the UI can see any errors price = Price.fromstring(ephemeral_data.get('text')) if price and price.amount != None: + # This is slightly misleading, it's extracting a PRICE not a Number.. res['extracted_number'] = float(price.amount) - logger.debug(f"Extracted price result: '{price}' - returning float({res['extracted_number']})") + logger.debug(f"Extracted number result: '{price}' - returning float({res['extracted_number']})") return res diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 81655bf1..0ce50df4 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -1403,8 +1403,10 @@ def changedetection_app(config=None, datastore_o=None): @login_optionally_required def watch_get_preview_rendered(uuid): '''For when viewing the "preview" of the rendered text from inside of Edit''' + from flask import jsonify from .processors.text_json_diff import prepare_filter_prevew - return prepare_filter_prevew(watch_uuid=uuid, datastore=datastore) + result = prepare_filter_prevew(watch_uuid=uuid, form_data=request.form, datastore=datastore) + return jsonify(result) @app.route("/form/add/quickwatch", methods=['POST']) @@ -1705,6 +1707,9 @@ def changedetection_app(config=None, datastore_o=None): import changedetectionio.blueprint.backups as backups app.register_blueprint(backups.construct_blueprint(datastore), url_prefix='/backups') + import changedetectionio.conditions.blueprint as conditions + app.register_blueprint(conditions.construct_blueprint(datastore), url_prefix='/conditions') + # @todo handle ctrl break ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() threading.Thread(target=notification_runner).start() diff --git a/changedetectionio/processors/text_json_diff/__init__.py b/changedetectionio/processors/text_json_diff/__init__.py index a6e018fd..8e5bdfc8 100644 --- a/changedetectionio/processors/text_json_diff/__init__.py +++ b/changedetectionio/processors/text_json_diff/__init__.py @@ -28,13 +28,13 @@ def _task(watch, update_handler): return text_after_filter -def prepare_filter_prevew(datastore, watch_uuid): +def prepare_filter_prevew(datastore, watch_uuid, form_data): '''Used by @app.route("/edit/<string:uuid>/preview-rendered", methods=['POST'])''' from changedetectionio import forms, html_tools from changedetectionio.model.Watch import model as watch_model from concurrent.futures import ProcessPoolExecutor from copy import deepcopy - from flask import request, jsonify + from flask import request import brotli import importlib import os @@ -50,12 +50,12 @@ def prepare_filter_prevew(datastore, watch_uuid): if tmp_watch and tmp_watch.history and os.path.isdir(tmp_watch.watch_data_dir): # Splice in the temporary stuff from the form - form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None, - data=request.form + form = forms.processor_text_json_diff_form(formdata=form_data if request.method == 'POST' else None, + data=form_data ) # Only update vars that came in via the AJAX post - p = {k: v for k, v in form.data.items() if k in request.form.keys()} + p = {k: v for k, v in form.data.items() if k in form_data.keys()} tmp_watch.update(p) blank_watch_no_filters = watch_model() blank_watch_no_filters['url'] = tmp_watch.get('url') @@ -103,13 +103,12 @@ def prepare_filter_prevew(datastore, watch_uuid): logger.trace(f"Parsed in {time.time() - now:.3f}s") - return jsonify( - { + return ({ 'after_filter': text_after_filter, 'before_filter': text_before_filter.decode('utf-8') if isinstance(text_before_filter, bytes) else text_before_filter, 'duration': time.time() - now, 'trigger_line_numbers': trigger_line_numbers, 'ignore_line_numbers': ignore_line_numbers, - } - ) + }) + diff --git a/changedetectionio/static/js/conditions.js b/changedetectionio/static/js/conditions.js index 1b983b51..8c627d63 100644 --- a/changedetectionio/static/js/conditions.js +++ b/changedetectionio/static/js/conditions.js @@ -2,7 +2,7 @@ $(document).ready(function () { // Function to set up button event handlers function setupButtonHandlers() { // Unbind existing handlers first to prevent duplicates - $(".addRuleRow, .removeRuleRow").off("click"); + $(".addRuleRow, .removeRuleRow, .verifyRuleRow").off("click"); // Add row button handler $(".addRuleRow").on("click", function(e) { @@ -34,12 +34,94 @@ $(document).ready(function () { reindexRules(); } }); + + // Verify rule button handler + $(".verifyRuleRow").on("click", function(e) { + e.preventDefault(); + + let row = $(this).closest("tr"); + let field = row.find("select[name$='field']").val(); + let operator = row.find("select[name$='operator']").val(); + let value = row.find("input[name$='value']").val(); + + // Validate that all fields are filled + if (!field || field === "None" || !operator || operator === "None" || !value) { + alert("Please fill in all fields (Field, Operator, and Value) before verifying."); + return; + } + + + // Create a rule object + const rule = { + field: field, + operator: operator, + value: value + }; + + // Show a spinner or some indication that verification is in progress + const $button = $(this); + const originalHTML = $button.html(); + $button.html("⌛").prop("disabled", true); + + // Collect form data - similar to request_textpreview_update() in watch-settings.js + let formData = new FormData(); + $('#edit-text-filter textarea, #edit-text-filter input').each(function() { + const $element = $(this); + const name = $element.attr('name'); + if (name) { + if ($element.is(':checkbox')) { + formData.append(name, $element.is(':checked') ? $element.val() : false); + } else { + formData.append(name, $element.val()); + } + } + }); + + // Also collect select values + $('#edit-text-filter select').each(function() { + const $element = $(this); + const name = $element.attr('name'); + if (name) { + formData.append(name, $element.val()); + } + }); + + + // Send the request to verify the rule + $.ajax({ + url: verify_condition_rule_url+"?"+ new URLSearchParams({ rule: JSON.stringify(rule) }).toString(), + type: "POST", + data: formData, + processData: false, // Prevent jQuery from converting FormData to a string + contentType: false, // Let the browser set the correct content type + success: function (response) { + if (response.status === "success") { + if (response.result) { + alert("✅ Condition PASSES verification against current snapshot!"); + } else { + alert("❌ Condition FAILS verification against current snapshot."); + } + } else { + alert("Error: " + response.message); + } + $button.html(originalHTML).prop("disabled", false); + }, + error: function (xhr) { + let errorMsg = "Error verifying condition."; + if (xhr.responseJSON && xhr.responseJSON.message) { + errorMsg = xhr.responseJSON.message; + } + alert(errorMsg); + $button.html(originalHTML).prop("disabled", false); + } + }); + }); } // Function to reindex form elements and re-setup event handlers function reindexRules() { // Unbind all button handlers first - $(".addRuleRow, .removeRuleRow").off("click"); + $(".addRuleRow, .removeRuleRow, .verifyRuleRow").off("click"); // Reindex all form elements $("#rulesTable tbody tr").each(function(index) { diff --git a/changedetectionio/templates/_helpers.html b/changedetectionio/templates/_helpers.html index d0eec6df..461eb22e 100644 --- a/changedetectionio/templates/_helpers.html +++ b/changedetectionio/templates/_helpers.html @@ -89,6 +89,7 @@ <td> <button type="button" class="addRuleRow">+</button> <button type="button" class="removeRuleRow">-</button> + <button type="button" class="verifyRuleRow" title="Verify this rule against current snapshot">✓</button> </td> </tr> {% endfor %} diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 3f7027fd..92c71967 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -286,9 +286,29 @@ Math: {{ 1 + 1 }}") }} {% if watch['processor'] == 'text_json_diff' %} <div class="tab-pane-inner" id="conditions"> + <script> + const verify_condition_rule_url="{{url_for('conditions.verify_condition_single_rule', watch_uuid=uuid)}}"; + </script> + <style> + .verifyRuleRow { + background-color: #4caf50; + color: white; + border: none; + cursor: pointer; + font-weight: bold; + } + .verifyRuleRow:hover { + background-color: #45a049; + } + </style> <div class="pure-control-group"> {{ render_field(form.conditions_match_logic) }} {{ render_fieldlist_of_formfields_as_table(form.conditions) }} + <div class="pure-form-message-inline"> + <br> + Use the verify (✓) button to test if a condition passes against the current snapshot.<br><br> + Did you know that <strong>conditions</strong> can be extended with your own custom plugin? tutorials coming soon!<br> + </div> </div> </div> <div class="tab-pane-inner" id="filters-and-triggers"> diff --git a/changedetectionio/tests/test_conditions.py b/changedetectionio/tests/test_conditions.py new file mode 100644 index 00000000..ad11884c --- /dev/null +++ b/changedetectionio/tests/test_conditions.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 + +from flask import url_for +from .util import live_server_setup, wait_for_all_checks + +def set_original_response(number="50"): + test_return_data = f"""<html> + <body> + <h1>Test Page for Conditions</h1> + <p>This page contains a number that will be tested with conditions.</p> + <div class="number-container">Current value: {number}</div> + </body> + </html> + """ + + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(test_return_data) + +def set_number_in_range_response(number="75"): + test_return_data = f"""<html> + <body> + <h1>Test Page for Conditions</h1> + <p>This page contains a number that will be tested with conditions.</p> + <div class="number-container">Current value: {number}</div> + </body> + </html> + """ + + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(test_return_data) + +def set_number_out_of_range_response(number="150"): + test_return_data = f"""<html> + <body> + <h1>Test Page for Conditions</h1> + <p>This page contains a number that will be tested with conditions.</p> + <div class="number-container">Current value: {number}</div> + </body> + </html> + """ + + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(test_return_data) + + +def test_conditions_with_text_and_number(client, live_server): + """Test that both text and number conditions work together with AND logic.""" + + set_original_response("50") + live_server_setup(live_server) + + test_url = url_for('test_endpoint', _external=True) + + # Add our URL to the import page + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + wait_for_all_checks(client) + + # Configure the watch with two conditions connected with AND: + # 1. The page filtered text must contain "5" (first digit of value) + # 2. The extracted number should be >= 20 and <= 100 + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "fetch_backend": "html_requests", + "include_filters": ".number-container", + "title": "Number AND Text Condition Test", + "conditions_match_logic": "ALL", # ALL = AND logic + "conditions-0-operator": "in", + "conditions-0-field": "page_filtered_text", + "conditions-0-value": "5", + + "conditions-1-operator": ">=", + "conditions-1-field": "extracted_number", + "conditions-1-value": "20", + + "conditions-2-operator": "<=", + "conditions-2-field": "extracted_number", + "conditions-2-value": "100", + + # So that 'operations' from pluggy discovery are tested + "conditions-3-operator": "length_min", + "conditions-3-field": "page_filtered_text", + "conditions-3-value": "1", + + # So that 'operations' from pluggy discovery are tested + "conditions-4-operator": "length_max", + "conditions-4-field": "page_filtered_text", + "conditions-4-value": "100", + + # So that 'operations' from pluggy discovery are tested + "conditions-5-operator": "contains_regex", + "conditions-5-field": "page_filtered_text", + "conditions-5-value": "\d", + }, + follow_redirects=True + ) + assert b"Updated watch." in res.data + + wait_for_all_checks(client) + client.get(url_for("mark_all_viewed"), follow_redirects=True) + wait_for_all_checks(client) + + # Case 1 + set_number_in_range_response("70.5") + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + # 75 is > 20 and < 100 and contains "5" + res = client.get(url_for("index")) + assert b'unviewed' in res.data + + + # Case 2: Change with one condition violated + # Number out of range (150) but contains '5' + client.get(url_for("mark_all_viewed"), follow_redirects=True) + set_number_out_of_range_response("150.5") + + + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + # Should NOT be marked as having changes since not all conditions are met + res = client.get(url_for("index")) + assert b'unviewed' not in res.data + + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data