diff --git a/changedetectionio/PLUGIN_README.md b/changedetectionio/PLUGIN_README.md new file mode 100644 index 00000000..5c0f0d60 --- /dev/null +++ b/changedetectionio/PLUGIN_README.md @@ -0,0 +1,98 @@ +# Creating Plugins for changedetection.io + +This document describes how to create plugins for changedetection.io. Plugins can be used to extend the functionality of the application in various ways. + +## Plugin Types + +### UI Stats Tab Plugins + +These plugins can add content to the Stats tab in the Edit page. This is useful for adding custom statistics or visualizations about a watch. + +#### Creating a UI Stats Tab Plugin + +1. Create a Python file in a directory that will be loaded by the plugin system. + +2. Use the `global_hookimpl` decorator to implement the `ui_edit_stats_extras` hook: + +```python +import pluggy +from loguru import logger + +global_hookimpl = pluggy.HookimplMarker("changedetectionio") + +@global_hookimpl +def ui_edit_stats_extras(watch): + """Add custom content to the stats tab""" + # Calculate or retrieve your stats + my_stat = calculate_something(watch) + + # Return HTML content as a string + html = f""" +
+

My Plugin Statistics

+

My statistic: {my_stat}

+
+ """ + return html +``` + +3. The HTML you return will be included in the Stats tab. + +## Plugin Loading + +Plugins can be loaded from: + +1. Built-in plugin directories in the codebase +2. External packages using setuptools entry points + +To add a new plugin directory, modify the `plugin_dirs` dictionary in `pluggy_interface.py`. + +## Example Plugin + +Here's a simple example of a plugin that adds a word count statistic to the Stats tab: + +```python +import pluggy +from loguru import logger + +global_hookimpl = pluggy.HookimplMarker("changedetectionio") + +def count_words_in_history(watch): + """Count words in the latest snapshot""" + try: + if not watch.history.keys(): + return 0 + + latest_key = list(watch.history.keys())[-1] + latest_content = watch.get_history_snapshot(latest_key) + return len(latest_content.split()) + except Exception as e: + logger.error(f"Error counting words: {str(e)}") + return 0 + +@global_hookimpl +def ui_edit_stats_extras(watch): + """Add word count to the Stats tab""" + word_count = count_words_in_history(watch) + + html = f""" +
+

Content Analysis

+ + + + + + + +
Word count (latest snapshot){word_count}
+
+ """ + return html +``` + +## Testing Your Plugin + +1. Place your plugin in one of the directories scanned by the plugin system +2. Restart changedetection.io +3. Go to the Edit page of a watch and check the Stats tab to see your content \ No newline at end of file diff --git a/changedetectionio/blueprint/ui/edit.py b/changedetectionio/blueprint/ui/edit.py index 467b4df6..b491d854 100644 --- a/changedetectionio/blueprint/ui/edit.py +++ b/changedetectionio/blueprint/ui/edit.py @@ -233,6 +233,9 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe # Only works reliably with Playwright + # Import the global plugin system + from changedetectionio.pluggy_interface import collect_ui_edit_stats_extras + template_args = { 'available_processors': processors.available_processors(), 'available_timezones': sorted(available_timezones()), @@ -250,6 +253,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe 'settings_application': datastore.data['settings']['application'], 'system_has_playwright_configured': os.getenv('PLAYWRIGHT_DRIVER_URL'), 'system_has_webdriver_configured': os.getenv('WEBDRIVER_URL'), + 'ui_edit_stats_extras': collect_ui_edit_stats_extras(watch), 'visual_selector_data_ready': datastore.visualselector_data_is_ready(watch_uuid=uuid), 'timezone_default_config': datastore.data['settings']['application'].get('timezone'), 'using_global_webdriver_wait': not default['webdriver_delay'], diff --git a/changedetectionio/conditions/__init__.py b/changedetectionio/conditions/__init__.py index ca36532e..05ef53e2 100644 --- a/changedetectionio/conditions/__init__.py +++ b/changedetectionio/conditions/__init__.py @@ -102,12 +102,31 @@ def execute_ruleset_against_all_plugins(current_watch_uuid: str, application_dat if complete_rules: # Give all plugins a chance to update the data dict again (that we will test the conditions against) for plugin in plugin_manager.get_plugins(): - new_execute_data = plugin.add_data(current_watch_uuid=current_watch_uuid, - application_datastruct=application_datastruct, - ephemeral_data=ephemeral_data) - - if new_execute_data and isinstance(new_execute_data, dict): - EXECUTE_DATA.update(new_execute_data) + try: + import concurrent.futures + import time + + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit( + plugin.add_data, + current_watch_uuid=current_watch_uuid, + application_datastruct=application_datastruct, + ephemeral_data=ephemeral_data + ) + + # Set a timeout of 10 seconds + try: + new_execute_data = future.result(timeout=10) + if new_execute_data and isinstance(new_execute_data, dict): + EXECUTE_DATA.update(new_execute_data) + except concurrent.futures.TimeoutError: + # The plugin took too long, abort processing for this watch + raise Exception(f"Plugin {plugin.__class__.__name__} took more than 10 seconds to run.") + except Exception as e: + # Log the error but continue with the next plugin + import logging + logging.error(f"Error executing plugin {plugin.__class__.__name__}: {str(e)}") + continue # Create the ruleset ruleset = convert_to_jsonlogic(logic_operator=logic_operator, rule_dict=complete_rules) @@ -132,3 +151,18 @@ for plugin in plugin_manager.get_plugins(): if isinstance(new_field_choices, list): field_choices.extend(new_field_choices) +def collect_ui_edit_stats_extras(watch): + """Collect and combine HTML content from all plugins that implement ui_edit_stats_extras""" + extras_content = [] + + for plugin in plugin_manager.get_plugins(): + try: + content = plugin.ui_edit_stats_extras(watch=watch) + if content: + extras_content.append(content) + except Exception as e: + # Skip plugins that don't implement the hook or have errors + pass + + return "\n".join(extras_content) if extras_content else "" + diff --git a/changedetectionio/conditions/pluggy_interface.py b/changedetectionio/conditions/pluggy_interface.py index af85776f..f67b9f51 100644 --- a/changedetectionio/conditions/pluggy_interface.py +++ b/changedetectionio/conditions/pluggy_interface.py @@ -1,5 +1,8 @@ import pluggy -from . import default_plugin # Import the default plugin +import os +import importlib +import sys +from . import default_plugin # ✅ Ensure that the namespace in HookspecMarker matches PluginManager PLUGIN_NAMESPACE = "changedetectionio_conditions" @@ -30,6 +33,11 @@ class ConditionsSpec: def add_data(current_watch_uuid, application_datastruct, ephemeral_data): """Add to the datadict""" pass + + @hookspec + def ui_edit_stats_extras(watch): + """Return HTML content to add to the stats tab in the edit view""" + pass # ✅ Set up Pluggy Plugin Manager plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE) @@ -40,5 +48,27 @@ plugin_manager.add_hookspecs(ConditionsSpec) # ✅ Register built-in plugins manually plugin_manager.register(default_plugin, "default_plugin") +# ✅ Load plugins from the plugins directory +def load_plugins_from_directory(): + plugins_dir = os.path.join(os.path.dirname(__file__), 'plugins') + if not os.path.exists(plugins_dir): + return + + # Get all Python files (excluding __init__.py) + for filename in os.listdir(plugins_dir): + if filename.endswith(".py") and filename != "__init__.py": + module_name = filename[:-3] # Remove .py extension + module_path = f"changedetectionio.conditions.plugins.{module_name}" + + try: + module = importlib.import_module(module_path) + # Register the plugin with pluggy + plugin_manager.register(module, module_name) + except (ImportError, AttributeError) as e: + print(f"Error loading plugin {module_name}: {e}") + +# Load plugins from the plugins directory +load_plugins_from_directory() + # ✅ Discover installed plugins from external packages (if any) plugin_manager.load_setuptools_entrypoints(PLUGIN_NAMESPACE) diff --git a/changedetectionio/conditions/plugins/__init__.py b/changedetectionio/conditions/plugins/__init__.py new file mode 100644 index 00000000..5e642ff3 --- /dev/null +++ b/changedetectionio/conditions/plugins/__init__.py @@ -0,0 +1 @@ +# Import plugins package to make them discoverable \ No newline at end of file diff --git a/changedetectionio/conditions/plugins/levenshtein_plugin.py b/changedetectionio/conditions/plugins/levenshtein_plugin.py new file mode 100644 index 00000000..df8341d9 --- /dev/null +++ b/changedetectionio/conditions/plugins/levenshtein_plugin.py @@ -0,0 +1,102 @@ +import pluggy +from loguru import logger + +# Support both plugin systems +conditions_hookimpl = pluggy.HookimplMarker("changedetectionio_conditions") +global_hookimpl = pluggy.HookimplMarker("changedetectionio") + +def levenshtein_ratio_recent_history(watch, incoming_text=None): + try: + from Levenshtein import ratio, distance + k = list(watch.history.keys()) + if len(k) >= 2: + # When called from ui_edit_stats_extras, we don't have incoming_text + if incoming_text is None: + a = watch.get_history_snapshot(timestamp=k[-1]) # Latest snapshot + b = watch.get_history_snapshot(timestamp=k[-2]) # Previous snapshot + else: + a = watch.get_history_snapshot(timestamp=k[-2]) # Second newest, incoming_text will be "newest" + b = incoming_text + + distance_value = distance(a, b) + ratio_value = ratio(a, b) + return { + 'distance': distance_value, + 'ratio': ratio_value, + 'percent_similar': round(ratio_value * 100, 2) + } + except Exception as e: + logger.warning(f"Unable to calc similarity: {str(e)}") + + return '' + +@conditions_hookimpl +def register_operators(): + pass + +@conditions_hookimpl +def register_operator_choices(): + pass + + +@conditions_hookimpl +def register_field_choices(): + return [ + ("levenshtein_ratio", "Levenshtein - Text similarity ratio"), + ("levenshtein_distance", "Levenshtein - Text change distance"), + ] + +@conditions_hookimpl +def add_data(current_watch_uuid, application_datastruct, ephemeral_data): + res = {} + watch = application_datastruct['watching'].get(current_watch_uuid) + # ephemeral_data['text'] will be the current text after filters, they may have edited filters but not saved them yet etc + + if watch and 'text' in ephemeral_data: + lev_data = levenshtein_ratio_recent_history(watch, ephemeral_data['text']) + if isinstance(lev_data, dict): + res['levenshtein_ratio'] = lev_data.get('ratio', 0) + res['levenshtein_similarity'] = lev_data.get('percent_similar', 0) + res['levenshtein_distance'] = lev_data.get('distance', 0) + + return res + +@global_hookimpl +def ui_edit_stats_extras(watch): + """Add Levenshtein stats to the UI using the global plugin system""" + """Generate the HTML for Levenshtein stats - shared by both plugin systems""" + if len(watch.history.keys()) < 2: + return "

Not enough history to calculate Levenshtein metrics

" + + try: + lev_data = levenshtein_ratio_recent_history(watch) + if not lev_data or not isinstance(lev_data, dict): + return "

Unable to calculate Levenshtein metrics

" + + html = f""" +
+

Levenshtein Text Similarity Details

+ + + + + + + + + + + + + + + +
Raw distance (edits needed){lev_data['distance']}
Similarity ratio{lev_data['ratio']:.4f}
Percent similar{lev_data['percent_similar']}%
+

Levenshtein metrics compare the last two snapshots, measuring how many character edits are needed to transform one into the other.

+
+ """ + return html + except Exception as e: + logger.error(f"Error generating Levenshtein UI extras: {str(e)}") + return "

Error calculating Levenshtein metrics

" + diff --git a/changedetectionio/conditions/plugins/wordcount_plugin.py b/changedetectionio/conditions/plugins/wordcount_plugin.py new file mode 100644 index 00000000..a19d3353 --- /dev/null +++ b/changedetectionio/conditions/plugins/wordcount_plugin.py @@ -0,0 +1,82 @@ +import pluggy +from loguru import logger + +# Support both plugin systems +conditions_hookimpl = pluggy.HookimplMarker("changedetectionio_conditions") +global_hookimpl = pluggy.HookimplMarker("changedetectionio") + +def count_words_in_history(watch, incoming_text=None): + """Count words in snapshot text""" + try: + if incoming_text is not None: + # When called from add_data with incoming text + return len(incoming_text.split()) + elif watch.history.keys(): + # When called from UI extras to count latest snapshot + latest_key = list(watch.history.keys())[-1] + latest_content = watch.get_history_snapshot(latest_key) + return len(latest_content.split()) + return 0 + except Exception as e: + logger.error(f"Error counting words: {str(e)}") + return 0 + +# Implement condition plugin hooks +@conditions_hookimpl +def register_operators(): + # No custom operators needed + return {} + +@conditions_hookimpl +def register_operator_choices(): + # No custom operator choices needed + return [] + +@conditions_hookimpl +def register_field_choices(): + # Add a field that will be available in conditions + return [ + ("word_count", "Word count of content"), + ] + +@conditions_hookimpl +def add_data(current_watch_uuid, application_datastruct, ephemeral_data): + """Add word count data for conditions""" + result = {} + watch = application_datastruct['watching'].get(current_watch_uuid) + + if watch and 'text' in ephemeral_data: + word_count = count_words_in_history(watch, ephemeral_data['text']) + result['word_count'] = word_count + + return result + +def _generate_stats_html(watch): + """Generate the HTML content for the stats tab""" + word_count = count_words_in_history(watch) + + html = f""" +
+

Content Analysis

+ + + + + + + +
Word count (latest snapshot){word_count}
+

Word count is a simple measure of content length, calculated by splitting text on whitespace.

+
+ """ + return html + +@conditions_hookimpl +def ui_edit_stats_extras(watch): + """Add word count stats to the UI through conditions plugin system""" + return _generate_stats_html(watch) + +@global_hookimpl +def ui_edit_stats_extras(watch): + """Add word count stats to the UI using the global plugin system""" + return _generate_stats_html(watch) \ No newline at end of file diff --git a/changedetectionio/pluggy_interface.py b/changedetectionio/pluggy_interface.py new file mode 100644 index 00000000..fe2f7182 --- /dev/null +++ b/changedetectionio/pluggy_interface.py @@ -0,0 +1,82 @@ +import pluggy +import os +import importlib +import sys + +# Global plugin namespace for changedetection.io +PLUGIN_NAMESPACE = "changedetectionio" + +hookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE) +hookimpl = pluggy.HookimplMarker(PLUGIN_NAMESPACE) + + +class ChangeDetectionSpec: + """Hook specifications for extending changedetection.io functionality.""" + + @hookspec + def ui_edit_stats_extras(watch): + """Return HTML content to add to the stats tab in the edit view. + + Args: + watch: The watch object being edited + + Returns: + str: HTML content to be inserted in the stats tab + """ + pass + + +# Set up Plugin Manager +plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE) + +# Register hookspecs +plugin_manager.add_hookspecs(ChangeDetectionSpec) + +# Load plugins from subdirectories +def load_plugins_from_directories(): + # Dictionary of directories to scan for plugins + plugin_dirs = { + 'conditions': os.path.join(os.path.dirname(__file__), 'conditions', 'plugins'), + # Add more plugin directories here as needed + } + + # Note: Removed the direct import of example_word_count_plugin as it's now in the conditions/plugins directory + + for dir_name, dir_path in plugin_dirs.items(): + if not os.path.exists(dir_path): + continue + + # Get all Python files (excluding __init__.py) + for filename in os.listdir(dir_path): + if filename.endswith(".py") and filename != "__init__.py": + module_name = filename[:-3] # Remove .py extension + module_path = f"changedetectionio.{dir_name}.plugins.{module_name}" + + try: + module = importlib.import_module(module_path) + # Register the plugin with pluggy + plugin_manager.register(module, module_name) + except (ImportError, AttributeError) as e: + print(f"Error loading plugin {module_name}: {e}") + +# Load plugins +load_plugins_from_directories() + +# Discover installed plugins from external packages (if any) +plugin_manager.load_setuptools_entrypoints(PLUGIN_NAMESPACE) + +# Helper function to collect UI stats extras from all plugins +def collect_ui_edit_stats_extras(watch): + """Collect and combine HTML content from all plugins that implement ui_edit_stats_extras""" + extras_content = [] + + # Get all plugins that implement the ui_edit_stats_extras hook + results = plugin_manager.hook.ui_edit_stats_extras(watch=watch) + + # If we have results, add them to our content + if results: + for result in results: + if result: # Skip empty results + extras_content.append(result) + + return "\n".join(extras_content) if extras_content else "" \ No newline at end of file diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 35b50197..82478d1a 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -450,6 +450,13 @@ Math: {{ 1 + 1 }}") }} + + {% if ui_edit_stats_extras %} +
+ {{ ui_edit_stats_extras|safe }} +
+ {% endif %} + {% if watch.history_n %}

Download latest HTML snapshot diff --git a/changedetectionio/tests/test_conditions.py b/changedetectionio/tests/test_conditions.py index b0e55631..fdb6d880 100644 --- a/changedetectionio/tests/test_conditions.py +++ b/changedetectionio/tests/test_conditions.py @@ -45,11 +45,15 @@ def set_number_out_of_range_response(number="150"): f.write(test_return_data) +def test_setup(client, live_server): + """Test that both text and number conditions work together with AND logic.""" + live_server_setup(live_server) + def test_conditions_with_text_and_number(client, live_server): """Test that both text and number conditions work together with AND logic.""" set_original_response("50") - live_server_setup(live_server) + #live_server_setup(live_server) test_url = url_for('test_endpoint', _external=True) @@ -195,3 +199,40 @@ def test_condition_validate_rule_row(client, live_server): + +# If there was only a change in the whitespacing, then we shouldnt have a change detected +def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage): + #live_server_setup(live_server) + + test_return_data = """ + + Some initial text
+

Which is across multiple lines

+
+ So let's see what happens.
+ + + """ + + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(test_return_data) + + # Add our URL to the import page + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("imports.import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + # Give the thread time to pick it up + wait_for_all_checks(client) + + # Check it saved + res = client.get( + url_for("ui.ui_edit.edit_page", uuid="first"), + ) + + # Assert the word count is counted correctly + assert b'13' in res.data \ No newline at end of file