From 9a073fc9aa99bbdb4b8ef75f61fd64545b78cd37 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 19 Mar 2025 15:24:02 +0100 Subject: [PATCH] WIP - pluggy refactor --- .../blueprint/imports/__init__.py | 2 +- .../blueprint/imports/importer.py | 2 +- .../blueprint/price_data_follower/__init__.py | 10 +- .../blueprint/price_data_follower/flags.py | 3 + .../blueprint/settings/__init__.py | 40 +++ .../settings/templates/settings.html | 3 +- changedetectionio/blueprint/ui/edit.py | 36 +- changedetectionio/blueprint/ui/views.py | 2 +- changedetectionio/flask_app.py | 5 +- changedetectionio/forms.py | 52 ++- changedetectionio/model/App.py | 1 + changedetectionio/processors/__init__.py | 304 +++++++++++++---- changedetectionio/processors/constants.py | 5 + .../processors/example_processor_plugin.py | 162 +++++++++ .../processors/pluggy_interface.py | 64 ++++ .../processors/test_plugin_example.py | 46 +++ .../processors/text_json_diff/processor.py | 2 +- changedetectionio/static/js/plugins.js | 18 + changedetectionio/store.py | 2 +- changedetectionio/templates/_helpers.html | 4 + changedetectionio/templates/settings.html | 310 ------------------ .../tests/test_processor_plugins.py | 120 +++++++ changedetectionio/update_worker.py | 22 +- 23 files changed, 774 insertions(+), 441 deletions(-) create mode 100644 changedetectionio/blueprint/price_data_follower/flags.py create mode 100644 changedetectionio/processors/constants.py create mode 100644 changedetectionio/processors/example_processor_plugin.py create mode 100644 changedetectionio/processors/pluggy_interface.py create mode 100644 changedetectionio/processors/test_plugin_example.py delete mode 100644 changedetectionio/templates/settings.html create mode 100644 changedetectionio/tests/test_processor_plugins.py diff --git a/changedetectionio/blueprint/imports/__init__.py b/changedetectionio/blueprint/imports/__init__.py index e0dd12bd..bc695f74 100644 --- a/changedetectionio/blueprint/imports/__init__.py +++ b/changedetectionio/blueprint/imports/__init__.py @@ -63,7 +63,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) # Could be some remaining, or we could be on GET - form = forms.importForm(formdata=request.form if request.method == 'POST' else None) + form = forms.importForm(formdata=request.form if request.method == 'POST' else None, datastore=datastore) output = render_template("import.html", form=form, import_url_list_remaining="\n".join(remaining_urls), diff --git a/changedetectionio/blueprint/imports/importer.py b/changedetectionio/blueprint/imports/importer.py index 4824d138..1a6bd3d7 100644 --- a/changedetectionio/blueprint/imports/importer.py +++ b/changedetectionio/blueprint/imports/importer.py @@ -3,7 +3,6 @@ import time from wtforms import ValidationError from loguru import logger -from changedetectionio.forms import validate_url class Importer(): @@ -151,6 +150,7 @@ class import_xlsx_wachete(Importer): self.new_uuids = [] from openpyxl import load_workbook + from changedetectionio.forms import validate_url try: wb = load_workbook(data) diff --git a/changedetectionio/blueprint/price_data_follower/__init__.py b/changedetectionio/blueprint/price_data_follower/__init__.py index 6011303a..018d54fb 100644 --- a/changedetectionio/blueprint/price_data_follower/__init__.py +++ b/changedetectionio/blueprint/price_data_follower/__init__.py @@ -1,18 +1,14 @@ -from changedetectionio.strtobool import strtobool from flask import Blueprint, flash, redirect, url_for from flask_login import login_required -from changedetectionio.store import ChangeDetectionStore -from changedetectionio import queuedWatchMetaData from queue import PriorityQueue +from changedetectionio import queuedWatchMetaData +from changedetectionio.processors.constants import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT -PRICE_DATA_TRACK_ACCEPT = 'accepted' -PRICE_DATA_TRACK_REJECT = 'rejected' -def construct_blueprint(datastore: ChangeDetectionStore, update_q: PriorityQueue): +def construct_blueprint(datastore, update_q: PriorityQueue): price_data_follower_blueprint = Blueprint('price_data_follower', __name__) - @login_required @price_data_follower_blueprint.route("//accept", methods=['GET']) def accept(uuid): diff --git a/changedetectionio/blueprint/price_data_follower/flags.py b/changedetectionio/blueprint/price_data_follower/flags.py new file mode 100644 index 00000000..652587f7 --- /dev/null +++ b/changedetectionio/blueprint/price_data_follower/flags.py @@ -0,0 +1,3 @@ + +PRICE_DATA_TRACK_ACCEPT = 'accepted' +PRICE_DATA_TRACK_REJECT = 'rejected' \ No newline at end of file diff --git a/changedetectionio/blueprint/settings/__init__.py b/changedetectionio/blueprint/settings/__init__.py index 5375b565..2614794a 100644 --- a/changedetectionio/blueprint/settings/__init__.py +++ b/changedetectionio/blueprint/settings/__init__.py @@ -84,6 +84,45 @@ def construct_blueprint(datastore: ChangeDetectionStore): # Convert to ISO 8601 format, all date/time relative events stored as UTC time utc_time = datetime.now(ZoneInfo("UTC")).isoformat() + + # Get processor plugins info + from changedetectionio.processors import get_all_plugins_info + plugins_info = get_all_plugins_info() + + # Create/update form with plugins info + default = deepcopy(datastore.data['settings']) + form = forms.globalSettingsForm( + formdata=request.form if request.method == 'POST' else None, + data=default, + extra_notification_tokens=datastore.get_unique_notification_tokens_available(), + plugins_info=plugins_info + ) + + # Process settings including plugin toggles + if request.method == 'POST' and form.validate(): + # Process the main form data + app_update = dict(deepcopy(form.data['application'])) + + # Don't update password with '' or False (Added by wtforms when not in submission) + if 'password' in app_update and not app_update['password']: + del (app_update['password']) + + datastore.data['settings']['application'].update(app_update) + datastore.data['settings']['requests'].update(form.data['requests']) + + # Update plugin settings from the dynamically created fields + enabled_plugins = {} + if hasattr(form, 'plugins'): + for field_name, field in form.plugins._fields.items(): + if field_name.startswith('plugin_'): + plugin_name = field_name.replace('plugin_', '') + enabled_plugins[plugin_name] = field.data + + # Update the datastore with plugin settings + datastore.data['settings']['application']['enabled_plugins'] = enabled_plugins + + datastore.needs_write_urgent = True + flash("Settings updated.") output = render_template("settings.html", api_key=datastore.data['settings']['application'].get('api_access_token'), @@ -93,6 +132,7 @@ def construct_blueprint(datastore: ChangeDetectionStore): form=form, hide_remove_pass=os.getenv("SALTED_PASS", False), min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)), + plugins_info=plugins_info, settings_application=datastore.data['settings']['application'], timezone_default_config=datastore.data['settings']['application'].get('timezone'), utc_time=utc_time, diff --git a/changedetectionio/blueprint/settings/templates/settings.html b/changedetectionio/blueprint/settings/templates/settings.html index 1dfeba0d..cc79f0fd 100644 --- a/changedetectionio/blueprint/settings/templates/settings.html +++ b/changedetectionio/blueprint/settings/templates/settings.html @@ -9,6 +9,7 @@ const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}'); {% endif %} + @@ -300,7 +301,7 @@ nav
{{ render_button(form.save_button) }} Back - Clear Snapshot History + Clear Snapshot History
diff --git a/changedetectionio/blueprint/ui/edit.py b/changedetectionio/blueprint/ui/edit.py index 73cd7853..3a9d4532 100644 --- a/changedetectionio/blueprint/ui/edit.py +++ b/changedetectionio/blueprint/ui/edit.py @@ -24,7 +24,6 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ? def edit_page(uuid): - from changedetectionio import forms from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config from changedetectionio import processors import importlib @@ -43,7 +42,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe switch_processor = request.args.get('switch_processor') if switch_processor: - for p in processors.available_processors(): + for p in processors.available_processors(datastore): if p[0] == switch_processor: datastore.data['watching'][uuid]['processor'] = switch_processor flash(f"Switched to mode - {p[1]}.") @@ -61,31 +60,19 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe default['proxy'] = '' # proxy_override set to the json/text list of the items - # Does it use some custom form? does one exist? - processor_name = datastore.data['watching'][uuid].get('processor', '') - processor_classes = next((tpl for tpl in processors.find_processors() if tpl[1] == processor_name), None) - if not processor_classes: - flash(f"Cannot load the edit form for processor/plugin '{processor_classes[1]}', plugin missing?", 'error') + # Get the appropriate form class for this processor using the pluggy system + processor_name = datastore.data['watching'][uuid].get('processor', 'text_json_diff') + form_class = processors.get_form_class_for_processor(processor_name) + + if not form_class: + flash(f"Cannot load the edit form for processor/plugin '{processor_name}', plugin missing?", 'error') return redirect(url_for('index')) - parent_module = processors.get_parent_module(processor_classes[0]) - - try: - # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code) - forms_module = importlib.import_module(f"{parent_module.__name__}.forms") - # Access the 'processor_settings_form' class from the 'forms' module - form_class = getattr(forms_module, 'processor_settings_form') - except ModuleNotFoundError as e: - # .forms didnt exist - form_class = forms.processor_text_json_diff_form - except AttributeError as e: - # .forms exists but no useful form - form_class = forms.processor_text_json_diff_form - form = form_class(formdata=request.form if request.method == 'POST' else None, data=default, extra_notification_tokens=default.extra_notification_token_values(), - default_system_settings=datastore.data['settings'] + default_system_settings=datastore.data['settings'], + datastore=datastore ) # For the form widget tag UUID back to "string name" for the field @@ -165,7 +152,8 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe datastore.data['watching'][uuid]['tags'] = [] # Recast it if need be to right data Watch handler - watch_class = processors.get_custom_watch_obj_for_processor(form.data.get('processor')) + processor_name = form.data.get('processor') + watch_class = processors.get_watch_model_for_processor(processor_name) datastore.data['watching'][uuid] = watch_class(datastore_path=datastore.datastore_path, default=datastore.data['watching'][uuid]) flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.") @@ -236,7 +224,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe # Only works reliably with Playwright template_args = { - 'available_processors': processors.available_processors(), + 'available_processors': processors.available_processors(datastore), 'available_timezones': sorted(available_timezones()), 'browser_steps_config': browser_step_ui_config, 'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), diff --git a/changedetectionio/blueprint/ui/views.py b/changedetectionio/blueprint/ui/views.py index 903a4c77..b48a25d6 100644 --- a/changedetectionio/blueprint/ui/views.py +++ b/changedetectionio/blueprint/ui/views.py @@ -191,7 +191,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe @login_optionally_required def form_quick_watch_add(): from changedetectionio import forms - form = forms.quickWatchForm(request.form) + form = forms.quickWatchForm(request.form, datastore=datastore) if not form.validate(): for widget, l in form.errors.items(): diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 232ad944..f4d9f0df 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -75,6 +75,7 @@ if os.getenv('FLASK_SERVER_NAME'): # Disables caching of the templates app.config['TEMPLATES_AUTO_RELOAD'] = True app.jinja_env.add_extension('jinja2.ext.loopcontrols') +app.jinja_env.globals.update(hasattr=hasattr) csrf = CSRFProtect() csrf.init_app(app) notification_debug_log=[] @@ -343,7 +344,7 @@ def changedetection_app(config=None, datastore_o=None): @login_optionally_required def index(): global datastore - from changedetectionio import forms + from changedetectionio.forms import quickWatchForm active_tag_req = request.args.get('tag', '').lower().strip() active_tag_uuid = active_tag = None @@ -394,7 +395,7 @@ def changedetection_app(config=None, datastore_o=None): else: sorted_watches.append(watch) - form = forms.quickWatchForm(request.form) + form = quickWatchForm(request.form, datastore=datastore) page = request.args.get(get_page_parameter(), type=int, default=1) total_count = len(sorted_watches) diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 3fd199bb..3d204c11 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -23,7 +23,7 @@ from wtforms import ( from flask_wtf.file import FileField, FileAllowed from wtforms.fields import FieldList -from wtforms.validators import ValidationError +from wtforms.validators import ValidationError, Optional from validators.url import url as url_validator @@ -508,8 +508,17 @@ class quickWatchForm(Form): url = fields.URLField('URL', validators=[validateURL()]) tags = StringTagUUID('Group tag', [validators.Optional()]) watch_submit_button = SubmitField('Watch', render_kw={"class": "pure-button pure-button-primary"}) - processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff") + processor = RadioField(u'Processor', default="text_json_diff") edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"}) + + def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs): + super().__init__(formdata, obj, prefix, data, meta, **kwargs) + # Set processor choices based on datastore if available + datastore = kwargs.get('datastore') + if datastore: + self.processor.choices = self.processors.available_processors(datastore) + else: + self.processor.choices = self.processors.available_processors() @@ -522,6 +531,13 @@ class commonSettingsForm(Form): self.notification_body.extra_notification_tokens = kwargs.get('extra_notification_tokens', {}) self.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {}) self.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {}) + + # Set processor choices based on datastore if available + datastore = kwargs.get('datastore') + if datastore: + self.processor.choices = self.processors.available_processors(datastore) + else: + self.processor.choices = self.processors.available_processors() extract_title_as_title = BooleanField('Extract from document and use as watch title', default=False) fetch_backend = RadioField(u'Fetch Method', choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()]) @@ -529,17 +545,26 @@ class commonSettingsForm(Form): notification_format = SelectField('Notification format', choices=valid_notification_formats.keys()) notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()]) notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()]) - processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff") + processor = RadioField( label=u"Processor - What do you want to achieve?", default="text_json_diff") timezone = StringField("Timezone for watch schedule", render_kw={"list": "timezones"}, validators=[validateTimeZoneName()]) webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")]) class importForm(Form): from . import processors - processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff") + processor = RadioField(u'Processor', default="text_json_diff") urls = TextAreaField('URLs') xlsx_file = FileField('Upload .xlsx file', validators=[FileAllowed(['xlsx'], 'Must be .xlsx file!')]) file_mapping = SelectField('File mapping', [validators.DataRequired()], choices={('wachete', 'Wachete mapping'), ('custom','Custom mapping')}) + + def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs): + super().__init__(formdata, obj, prefix, data, meta, **kwargs) + # Set processor choices based on datastore if available + datastore = kwargs.get('datastore') + if datastore: + self.processor.choices = self.processors.available_processors(datastore) + else: + self.processor.choices = self.processors.available_processors() class SingleBrowserStep(Form): @@ -714,11 +739,12 @@ class globalSettingsRequestForm(Form): default_ua = FormField(DefaultUAInputForm, label="Default User-Agent overrides") def validate_extra_proxies(self, extra_validators=None): - for e in self.data['extra_proxies']: - if e.get('proxy_name') or e.get('proxy_url'): - if not e.get('proxy_name','').strip() or not e.get('proxy_url','').strip(): - self.extra_proxies.errors.append('Both a name, and a Proxy URL is required.') - return False + if self.data.get('extra_proxies'): + for e in self.data['extra_proxies']: + if e.get('proxy_name') or e.get('proxy_url'): + if not e.get('proxy_name','').strip() or not e.get('proxy_url','').strip(): + self.extra_proxies.errors.append('Both a name, and a Proxy URL is required.') + return False # datastore.data['settings']['application'].. @@ -749,6 +775,14 @@ class globalSettingsApplicationForm(commonSettingsForm): validators=[validators.NumberRange(min=0, message="Should contain zero or more attempts")]) + # Create plugins form and add it as an attribute +# plugin_form = PluginsManagementForm( +# formdata=formdata, +# plugins_info=plugins_info, +# enabled_plugins=enabled_plugins +# ) + + class globalSettingsForm(Form): # Define these as FormFields/"sub forms", this way it matches the JSON storage diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py index 4c9c34fe..b4a55fd7 100644 --- a/changedetectionio/model/App.py +++ b/changedetectionio/model/App.py @@ -54,6 +54,7 @@ class model(dict): 'webdriver_delay': None , # Extra delay in seconds before extracting text 'tags': {}, #@todo use Tag.model initialisers 'timezone': None, # Default IANA timezone name + 'enabled_plugins': {} # Dictionary of plugin names and their enabled status } } } diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index adf08a18..299a2ebe 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -9,6 +9,8 @@ import inspect import os import pkgutil import re +import sys +from .pluggy_interface import plugin_manager, hookimpl class difference_detection_processor(): @@ -172,83 +174,245 @@ class difference_detection_processor(): return changed_detected, update_obj, ''.encode('utf-8') -def find_sub_packages(package_name): +def get_all_plugins_info(): """ - Find all sub-packages within the given package. - - :param package_name: The name of the base package to scan for sub-packages. - :return: A list of sub-package names. + Get information about all registered processor plugins + :return: A list of dictionaries with plugin info """ - package = importlib.import_module(package_name) - return [name for _, name, is_pkg in pkgutil.iter_modules(package.__path__) if is_pkg] + plugins_info = [] + + # Collect from all registered plugins + for plugin in plugin_manager.get_plugins(): + if hasattr(plugin, "get_processor_name") and hasattr(plugin, "get_processor_description"): + processor_name = plugin.get_processor_name() + description = plugin.get_processor_description() + + # Get version if available + version = "N/A" + if hasattr(plugin, "get_processor_version"): + plugin_version = plugin.get_processor_version() + if plugin_version: + version = plugin_version + + if processor_name and description: + plugins_info.append({ + "name": processor_name, + "description": description, + "version": version + }) + + # Fallback if no plugins registered + if not plugins_info: + plugins_info = [ + {"name": "text_json_diff", "description": "Webpage Text/HTML, JSON and PDF changes", "version": "1.0.0"}, + {"name": "restock_diff", "description": "Re-stock & Price detection for single product pages", "version": "1.0.0"} + ] + + return plugins_info - -def find_processors(): - """ - Find all subclasses of DifferenceDetectionProcessor in the specified package. - - :param package_name: The name of the package to scan for processor modules. - :return: A list of (module, class) tuples. - """ - package_name = "changedetectionio.processors" # Name of the current package/module - - processors = [] - sub_packages = find_sub_packages(package_name) - - for sub_package in sub_packages: - module_name = f"{package_name}.{sub_package}.processor" - try: - module = importlib.import_module(module_name) - - # Iterate through all classes in the module - for name, obj in inspect.getmembers(module, inspect.isclass): - if issubclass(obj, difference_detection_processor) and obj is not difference_detection_processor: - processors.append((module, sub_package)) - except (ModuleNotFoundError, ImportError) as e: - logger.warning(f"Failed to import module {module_name}: {e} (find_processors())") - - return processors - - -def get_parent_module(module): - module_name = module.__name__ - if '.' not in module_name: - return None # Top-level module has no parent - parent_module_name = module_name.rsplit('.', 1)[0] - try: - return importlib.import_module(parent_module_name) - except Exception as e: - pass - - return False - - - -def get_custom_watch_obj_for_processor(processor_name): - from changedetectionio.model import Watch - watch_class = Watch.model - processor_classes = find_processors() - custom_watch_obj = next((tpl for tpl in processor_classes if tpl[1] == processor_name), None) - if custom_watch_obj: - # Parent of .processor.py COULD have its own Watch implementation - parent_module = get_parent_module(custom_watch_obj[0]) - if hasattr(parent_module, 'Watch'): - watch_class = parent_module.Watch - - return watch_class - - -def available_processors(): +def available_processors(datastore=None): """ Get a list of processors by name and description for the UI elements - :return: A list :) + Filtered by enabled_plugins setting if datastore is provided + :return: A list of tuples (processor_name, description) """ + plugins_info = get_all_plugins_info() + processor_list = [] + + # If datastore is provided, filter by enabled_plugins + if datastore: + # Make sure enabled_plugins exists in datastore + if 'enabled_plugins' not in datastore.data['settings']['application']: + datastore.data['settings']['application']['enabled_plugins'] = {} + + enabled_plugins = datastore.data['settings']['application']['enabled_plugins'] + + # Scan for any new plugins that aren't in the enabled_plugins dict yet + # Default built-in processors to enabled, third-party to disabled + plugins_updated = False + for plugin in plugins_info: + if plugin["name"] not in enabled_plugins: + # Built-in processors are enabled by default + if plugin["name"] in ["text_json_diff", "restock_diff"]: + enabled_plugins[plugin["name"]] = True + else: + # Third-party plugins are disabled by default + enabled_plugins[plugin["name"]] = False + plugins_updated = True + + # Save changes if we added new plugins + if plugins_updated: + datastore.needs_write = True + + # Only include enabled plugins + for plugin in plugins_info: + if enabled_plugins.get(plugin["name"], False): + processor_list.append((plugin["name"], plugin["description"])) + else: + # No datastore provided, include all plugins + for plugin in plugins_info: + processor_list.append((plugin["name"], plugin["description"])) + + return processor_list - processor_classes = find_processors() +def get_processor_handler(processor_name, datastore, watch_uuid): + """ + Get the processor handler for the specified processor name + :return: The processor handler instance + """ + # Try each plugin in turn + for plugin in plugin_manager.get_plugins(): + if hasattr(plugin, "perform_site_check"): + handler = plugin.perform_site_check(datastore=datastore, watch_uuid=watch_uuid) + if handler: + return handler + + # If no plugins handled it, use the appropriate built-in processor + watch = datastore.data['watching'].get(watch_uuid) + if watch and watch.get('processor') == 'restock_diff': + from .restock_diff.processor import perform_site_check + return perform_site_check(datastore=datastore, watch_uuid=watch_uuid) + else: + # Default to text_json_diff + from .text_json_diff.processor import perform_site_check + return perform_site_check(datastore=datastore, watch_uuid=watch_uuid) - available = [] - for package, processor_class in processor_classes: - available.append((processor_class, package.name)) +def get_form_class_for_processor(processor_name): + """ + Get the form class for the specified processor name + :return: The form class + """ + # Try each plugin in turn + for plugin in plugin_manager.get_plugins(): + if hasattr(plugin, "get_form_class"): + form_class = plugin.get_form_class(processor_name=processor_name) + if form_class: + return form_class + + # If no plugins provided a form class, use the appropriate built-in form + if processor_name == 'restock_diff': + try: + from .restock_diff.forms import processor_settings_form + return processor_settings_form + except ImportError: + pass + + # Default to text_json_diff form + from changedetectionio import forms + return forms.processor_text_json_diff_form - return available +def get_watch_model_for_processor(processor_name): + """ + Get the Watch model class for the specified processor name + :return: The Watch model class + """ + # Try each plugin in turn + for plugin in plugin_manager.get_plugins(): + if hasattr(plugin, "get_watch_model_class"): + model_class = plugin.get_watch_model_class(processor_name=processor_name) + if model_class: + return model_class + + # Default to standard Watch model + from changedetectionio.model import Watch + return Watch.model +# Define plugin implementations for the built-in processors +class TextJsonDiffPlugin: + @hookimpl + def get_processor_name(self): + return "text_json_diff" + + @hookimpl + def get_processor_description(self): + from .text_json_diff.processor import name + return name + + @hookimpl + def get_processor_version(self): + return "1.0.0" + + @hookimpl + def perform_site_check(self, datastore, watch_uuid): + watch = datastore.data['watching'].get(watch_uuid) + if watch and watch.get('processor', 'text_json_diff') == 'text_json_diff': + from .text_json_diff.processor import perform_site_check + return perform_site_check(datastore=datastore, watch_uuid=watch_uuid) + return None + + @hookimpl + def get_form_class(self, processor_name): + if processor_name == 'text_json_diff': + from changedetectionio import forms + return forms.processor_text_json_diff_form + return None + + @hookimpl + def get_watch_model_class(self, processor_name): + if processor_name == 'text_json_diff': + from changedetectionio.model import Watch + return Watch.model + return None + +class RestockDiffPlugin: + @hookimpl + def get_processor_name(self): + return "restock_diff" + + @hookimpl + def get_processor_description(self): + from .restock_diff.processor import name + return name + + @hookimpl + def get_processor_version(self): + return "1.0.0" + + @hookimpl + def perform_site_check(self, datastore, watch_uuid): + watch = datastore.data['watching'].get(watch_uuid) + if watch and watch.get('processor') == 'restock_diff': + from .restock_diff.processor import perform_site_check + return perform_site_check(datastore=datastore, watch_uuid=watch_uuid) + return None + + @hookimpl + def get_form_class(self, processor_name): + if processor_name == 'restock_diff': + try: + from .restock_diff.forms import processor_settings_form + return processor_settings_form + except ImportError: + pass + return None + + @hookimpl + def get_watch_model_class(self, processor_name): + if processor_name == 'restock_diff': + # Currently uses default watch model, could be customized in the future + from changedetectionio.model import Watch + return Watch.model + return None + +# Import our example plugins +from .example_processor_plugin import ExampleProcessorPlugin + +# For backward compatibility +def get_custom_watch_obj_for_processor(processor_name): + return get_watch_model_for_processor(processor_name) + +# Register the built-in processor plugins +plugin_manager.register(TextJsonDiffPlugin()) +plugin_manager.register(RestockDiffPlugin()) +plugin_manager.register(ExampleProcessorPlugin()) + +# Check for test plugin and conditionally register it +try: + # This avoids circular imports + from .test_plugin_example import ExampleProcessorPlugin as TestExampleProcessorPlugin + test_plugin_instance = TestExampleProcessorPlugin() + # Only register if it has a different name than the regular example plugin + if test_plugin_instance.get_processor_name() != "example_processor": + plugin_manager.register(test_plugin_instance) +except (ImportError, AttributeError): + pass \ No newline at end of file diff --git a/changedetectionio/processors/constants.py b/changedetectionio/processors/constants.py new file mode 100644 index 00000000..fd6ff16f --- /dev/null +++ b/changedetectionio/processors/constants.py @@ -0,0 +1,5 @@ +# Common constants used across processors + +# Price data tracking constants +PRICE_DATA_TRACK_ACCEPT = 'accepted' +PRICE_DATA_TRACK_REJECT = 'rejected' \ No newline at end of file diff --git a/changedetectionio/processors/example_processor_plugin.py b/changedetectionio/processors/example_processor_plugin.py new file mode 100644 index 00000000..058fe9a1 --- /dev/null +++ b/changedetectionio/processors/example_processor_plugin.py @@ -0,0 +1,162 @@ +""" +Example plugin to demonstrate how to create a new processor plugin +""" +from .pluggy_interface import hookimpl +import importlib + +class ExampleProcessorPlugin: + """ + Example processor plugin that extends the text_json_diff processor + """ + + @hookimpl + def get_processor_name(self): + return "example_processor" + + @hookimpl + def get_processor_description(self): + return "Example Processor Plugin - For demonstration purposes" + + @hookimpl + def get_processor_version(self): + return "0.1.0-beta" + + @hookimpl + def perform_site_check(self, datastore, watch_uuid): + watch = datastore.data['watching'].get(watch_uuid) + if watch and watch.get('processor') == 'example_processor': + # Log that we're using our special example processor + from loguru import logger + + # Check if the example mode is enabled + if watch.is_example_mode_enabled(): + # Get the threshold value for our plugin + threshold = watch.get_example_threshold() + logger.info(f"Example processor using mode: {watch.get('example_settings', {}).get('mode')} with threshold: {threshold}") + + # Check if advanced features are enabled + advanced_features = watch.get('example_settings', {}).get('example_toggle', False) + if advanced_features: + logger.info("Example processor advanced features are enabled") + else: + logger.info("Example processor is in OFF mode, using standard processing") + + # Import here to avoid circular imports + from changedetectionio.processors.text_json_diff.processor import perform_site_check + return perform_site_check(datastore=datastore, watch_uuid=watch_uuid) + return None + + @hookimpl + def get_form_class(self, processor_name): + if processor_name == 'example_processor': + # Import here to avoid circular imports + from changedetectionio import forms + from wtforms import StringField, BooleanField, TextAreaField, RadioField, FloatField + from wtforms.validators import Optional, NumberRange + from wtforms.fields.form import FormField + from wtforms.form import Form + + # Create a settings form for the example plugin + class ExampleSettingsForm(Form): + mode = RadioField(label='Example Mode', choices=[ + ('mode_a', "Mode A - Default behavior"), + ('mode_b', "Mode B - Alternative behavior"), + ('off', "Off - Disable example functionality"), + ], default="mode_a") + + threshold = FloatField('Threshold value', [ + Optional(), + NumberRange(min=0, max=100, message="Should be between 0 and 100") + ], render_kw={"placeholder": "0", "size": "5"}) + + example_toggle = BooleanField('Enable advanced features', default=False) + example_notes = TextAreaField('Notes', validators=[Optional()]) + + # Create the main form by extending the base form + class ExampleProcessorForm(forms.processor_text_json_diff_form): + example_settings = FormField(ExampleSettingsForm) + + def extra_tab_content(self): + return 'Example Plugin' + + def extra_form_content(self): + output = "" + + # Show warning if tag overrides settings (similar to restock plugin) + if getattr(self, 'watch', None) and getattr(self, 'datastore'): + for tag_uuid in self.watch.get('tags'): + tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {}) + if tag.get('overrides_watch'): + output = f"""<p><strong>Note! A Group tag overrides the example plugin settings here.</strong></p><style>#example-fieldset-group {{ opacity: 0.6; }}</style>""" + + output += """ + {% from '_helpers.html' import render_field, render_checkbox_field, render_button %} + <script> + $(document).ready(function () { + toggleOpacity('#example_settings-example_toggle', '.example-advanced-settings', true); + }); + </script> + + <fieldset id="example-fieldset-group"> + <div class="pure-control-group"> + <fieldset class="pure-group inline-radio"> + {{ render_field(form.example_settings.mode) }} + </fieldset> + <fieldset class="pure-group"> + {{ render_checkbox_field(form.example_settings.example_toggle) }} + <span class="pure-form-message-inline">Enable advanced example features</span> + </fieldset> + <fieldset class="pure-group example-advanced-settings"> + {{ render_field(form.example_settings.threshold) }} + <span class="pure-form-message-inline">Set the threshold percentage for this example plugin</span> + <span class="pure-form-message-inline">For example, 5% means the plugin will only activate when changes exceed 5% of the content</span> + </fieldset> + <fieldset class="pure-group example-advanced-settings"> + {{ render_field(form.example_settings.example_notes, rows=3, placeholder="Add any notes here...") }} + <span class="pure-form-message-inline">Additional notes for this watch</span> + </fieldset> + </div> + </fieldset> + """ + return output + + return ExampleProcessorForm + return None + + @hookimpl + def get_watch_model_class(self, processor_name): + if processor_name == 'example_processor': + # Import here to avoid circular imports + from changedetectionio.model import Watch + + # Create a custom Watch model class for the example plugin + class ExampleWatchModel(Watch.model): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Initialize example plugin settings if not present + if not self.get('example_settings'): + self['example_settings'] = { + 'mode': 'mode_a', + 'threshold': 0, + 'example_toggle': False, + 'example_notes': '' + } + + # Add any custom methods for the example plugin + def get_example_threshold(self): + """Get the threshold value or return the default""" + settings = self.get('example_settings', {}) + return settings.get('threshold', 0) + + def is_example_mode_enabled(self): + """Check if the example plugin is enabled""" + settings = self.get('example_settings', {}) + return settings.get('mode') != 'off' + + return ExampleWatchModel + return None + +# This function would be called by the setup.py entry_points +def register_plugin(plugin_manager): + plugin_manager.register(ExampleProcessorPlugin()) \ No newline at end of file diff --git a/changedetectionio/processors/pluggy_interface.py b/changedetectionio/processors/pluggy_interface.py new file mode 100644 index 00000000..4ab9692a --- /dev/null +++ b/changedetectionio/processors/pluggy_interface.py @@ -0,0 +1,64 @@ +import pluggy + +# Ensure that the namespace in HookspecMarker matches PluginManager +PLUGIN_NAMESPACE = "changedetectionio_processors" + +hookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE) +hookimpl = pluggy.HookimplMarker(PLUGIN_NAMESPACE) + + +class ProcessorSpec: + """Hook specifications for difference detection processors.""" + + @hookspec + def get_processor_name(): + """Return the processor name for selection in the UI.""" + pass + + @hookspec + def get_processor_description(): + """Return a human-readable description of the processor.""" + pass + + @hookspec + def get_processor_version(): + """Return the processor plugin version.""" + pass + + @hookspec + def perform_site_check(datastore, watch_uuid): + """Return the processor handler class or None if not applicable. + + Each plugin should check if it's the right processor for this watch + and return None if it's not. + + Should return an instance of a class that implements: + - call_browser(preferred_proxy_id=None): Fetch the content + - run_changedetection(watch): Analyze for changes and return tuple of (changed_detected, update_obj, contents) + """ + pass + + @hookspec + def get_form_class(processor_name): + """Return the WTForms form class for the processor settings or None if not applicable. + + Each plugin should check if it's the right processor and return None if not. + """ + pass + + @hookspec + def get_watch_model_class(processor_name): + """Return a custom Watch model class if needed or None if not applicable. + + Each plugin should check if it's the right processor and return None if not. + """ + pass + +# Set up Pluggy Plugin Manager +plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE) + +# Register hookspecs +plugin_manager.add_hookspecs(ProcessorSpec) + +# Discover installed plugins from external packages (if any) +plugin_manager.load_setuptools_entrypoints(PLUGIN_NAMESPACE) \ No newline at end of file diff --git a/changedetectionio/processors/test_plugin_example.py b/changedetectionio/processors/test_plugin_example.py new file mode 100644 index 00000000..6a12bf7c --- /dev/null +++ b/changedetectionio/processors/test_plugin_example.py @@ -0,0 +1,46 @@ +""" +Example plugin to demonstrate how to create a new processor plugin +""" +from .pluggy_interface import hookimpl +from .text_json_diff.processor import perform_site_check as text_json_diff_perform_site_check +from changedetectionio import forms + +class ExampleProcessorPlugin: + """ + Example processor plugin that extends the text_json_diff processor + """ + + @hookimpl + def get_processor_name(self): + return "example_processor" + + @hookimpl + def get_processor_description(self): + return "Example Processor Plugin - For demonstration purposes" + + @hookimpl + def perform_site_check(self, datastore, watch_uuid): + watch = datastore.data['watching'].get(watch_uuid) + if watch and watch.get('processor') == 'example_processor': + # This processor is just a wrapper around text_json_diff for demonstration + return text_json_diff_perform_site_check(datastore=datastore, watch_uuid=watch_uuid) + return None + + @hookimpl + def get_form_class(self, processor_name): + if processor_name == 'example_processor': + # Use the default form for this example + return forms.processor_text_json_diff_form + return None + + @hookimpl + def get_watch_model_class(self, processor_name): + if processor_name == 'example_processor': + # Use the default Watch model for this example + from changedetectionio.model import Watch + return Watch.model + return None + +# This function would be called by the setup.py entry_points +def register_plugin(plugin_manager): + plugin_manager.register(ExampleProcessorPlugin()) \ No newline at end of file diff --git a/changedetectionio/processors/text_json_diff/processor.py b/changedetectionio/processors/text_json_diff/processor.py index faeab5d2..5ccc8ce5 100644 --- a/changedetectionio/processors/text_json_diff/processor.py +++ b/changedetectionio/processors/text_json_diff/processor.py @@ -10,7 +10,7 @@ from changedetectionio.conditions import execute_ruleset_against_all_plugins from changedetectionio.processors import difference_detection_processor from changedetectionio.html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text, TRANSLATE_WHITESPACE_TABLE from changedetectionio import html_tools, content_fetchers -from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT +from changedetectionio.processors.constants import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT from loguru import logger urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) diff --git a/changedetectionio/static/js/plugins.js b/changedetectionio/static/js/plugins.js index 1eecef6c..1dea43fe 100644 --- a/changedetectionio/static/js/plugins.js +++ b/changedetectionio/static/js/plugins.js @@ -1,4 +1,22 @@ (function ($) { + // Initialize plugin management UI when the DOM is ready + $(document).ready(function() { + // Add event handlers for plugin checkboxes + $("#plugins-table input[type='checkbox']").on('change', function() { + const isEnabled = $(this).is(':checked'); + + // For visual feedback, fade the row when disabled + if (isEnabled) { + $(this).closest('tr').removeClass('disabled-plugin'); + } else { + $(this).closest('tr').addClass('disabled-plugin'); + } + + const pluginName = $(this).closest('tr').find('td:nth-child(2)').text().trim(); + console.log(`Plugin ${pluginName} ${isEnabled ? 'enabled' : 'disabled'}`); + }); + }); + /** * debounce * @param {integer} milliseconds This param indicates the number of milliseconds diff --git a/changedetectionio/store.py b/changedetectionio/store.py index efc29275..9d5c7086 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -6,7 +6,7 @@ from flask import ( from .html_tools import TRANSLATE_WHITESPACE_TABLE from . model import App, Watch -from copy import deepcopy, copy +from copy import deepcopy from os import path, unlink from threading import Lock import json diff --git a/changedetectionio/templates/_helpers.html b/changedetectionio/templates/_helpers.html index 2ed75a30..0bc23684 100644 --- a/changedetectionio/templates/_helpers.html +++ b/changedetectionio/templates/_helpers.html @@ -1,3 +1,7 @@ +{% macro hasattr(obj, name) -%} + {{ obj is defined and name in obj.__dict__ }} +{%- endmacro %} + {% macro render_field(field) %} <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div> <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }} diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html deleted file mode 100644 index 2e651a01..00000000 --- a/changedetectionio/templates/settings.html +++ /dev/null @@ -1,310 +0,0 @@ -{% extends 'base.html' %} - -{% block content %} -{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %} -{% from '_common_fields.html' import render_common_settings_form %} -<script> - const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="global-settings")}}"; -{% if emailprefix %} - const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}'); -{% endif %} -</script> -<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script> -<script src="{{url_for('static_content', group='js', filename='plugins.js')}}" defer></script> -<script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script> -<script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script> -<script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script> -<script src="{{url_for('static_content', group='js', filename='scheduler.js')}}" defer></script> -<div class="edit-form"> - <div class="tabs collapsable"> - <ul> - <li class="tab" id=""><a href="#general">General</a></li> - <li class="tab"><a href="#notifications">Notifications</a></li> - <li class="tab"><a href="#fetching">Fetching</a></li> - <li class="tab"><a href="#filters">Global Filters</a></li> - <li class="tab"><a href="#api">API</a></li> - <li class="tab"><a href="#timedate">Time & Date</a></li> - <li class="tab"><a href="#proxies">CAPTCHA & Proxies</a></li> - </ul> - </div> - <div class="box-wrap inner"> - <form class="pure-form pure-form-stacked settings" action="{{url_for('settings.settings_page')}}" method="POST"> - <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" > - <div class="tab-pane-inner" id="general"> - <fieldset> - <div class="pure-control-group"> - {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }} - <span class="pure-form-message-inline">Default recheck time for all watches, current system minimum is <i>{{min_system_recheck_seconds}}</i> seconds (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Misc-system-settings#enviroment-variables">more info</a>).</span> - <div id="time-between-check-schedule"> - <!-- Start Time and End Time --> - <div id="limit-between-time"> - {{ render_time_schedule_form(form.requests, available_timezones, timezone_default_config) }} - </div> - </div> - </div> - <div class="pure-control-group"> - {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }} - <span class="pure-form-message-inline">Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later</span> - </div> - <div class="pure-control-group"> - {{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class="filter_failure_notification_threshold_attempts") }} - <span class="pure-form-message-inline">After this many consecutive times that the CSS/xPath filter is missing, send a notification - <br> - Set to <strong>0</strong> to disable - </span> - </div> - <div class="pure-control-group"> - {% if not hide_remove_pass %} - {% if current_user.is_authenticated %} - {{ render_button(form.application.form.removepassword_button) }} - {% else %} - {{ render_field(form.application.form.password) }} - <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span> - {% endif %} - {% else %} - <span class="pure-form-message-inline">Password is locked.</span> - {% endif %} - </div> - - <div class="pure-control-group"> - {{ render_checkbox_field(form.application.form.shared_diff_access, class="shared_diff_access") }} - <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page) - </span> - </div> - <div class="pure-control-group"> - {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }} - </div> - <div class="pure-control-group"> - {{ render_field(form.application.form.pager_size) }} - <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span> - </div> - - <div class="pure-control-group"> - {{ render_checkbox_field(form.application.form.extract_title_as_title) }} - <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span> - </div> - <div class="pure-control-group"> - {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }} - <span class="pure-form-message-inline">When a request returns no content, or the HTML does not contain any text, is this considered a change?</span> - </div> - {% if form.requests.proxy %} - <div class="pure-control-group inline-radio"> - {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }} - <span class="pure-form-message-inline"> - Choose a default proxy for all watches - </span> - </div> - {% endif %} - </fieldset> - </div> - - <div class="tab-pane-inner" id="notifications"> - <fieldset> - <div class="field-group"> - {{ render_common_settings_form(form.application.form, emailprefix, settings_application, extra_notification_token_placeholder_info) }} - </div> - </fieldset> - <div class="pure-control-group" id="notification-base-url"> - {{ render_field(form.application.form.base_url, class="m-d") }} - <span class="pure-form-message-inline"> - Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notification links.<br> - Default value is the system environment variable '<code>BASE_URL</code>' - <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>. - </span> - </div> - </div> - - <div class="tab-pane-inner" id="fetching"> - <div class="pure-control-group inline-radio"> - {{ render_field(form.application.form.fetch_backend, class="fetch-backend") }} - <span class="pure-form-message-inline"> - <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p> - <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p> - </span> - </div> - <fieldset class="pure-group" id="webdriver-override-options" data-visible-for="application-fetch_backend=html_webdriver"> - <div class="pure-form-message-inline"> - <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong> - <br> - This will wait <i>n</i> seconds before extracting the text. - </div> - <div class="pure-control-group"> - {{ render_field(form.application.form.webdriver_delay) }} - </div> - </fieldset> - <div class="pure-control-group inline-radio"> - {{ render_field(form.requests.form.default_ua) }} - <span class="pure-form-message-inline"> - Applied to all requests.<br><br> - Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it's important to consider <a href="https://changedetection.io/tutorial/what-are-main-types-anti-robot-mechanisms">all of the ways that the browser is detected</a>. - </span> - </div> - <div class="pure-control-group"> - <br> - Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a> - - </div> - </div> - - <div class="tab-pane-inner" id="filters"> - - <fieldset class="pure-group"> - {{ render_checkbox_field(form.application.form.ignore_whitespace) }} - <span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br> - <i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc. - </span> - </fieldset> - <fieldset class="pure-group"> - {{ render_checkbox_field(form.application.form.render_anchor_tag_content) }} - <span class="pure-form-message-inline">Render anchor tag content, default disabled, when enabled renders links as <code>(link text)[https://somesite.com]</code> - <br> - <i>Note:</i> Changing this could affect the content of your existing watches, possibly trigger alerts etc. - </span> - </fieldset> - <fieldset class="pure-group"> - {{ render_field(form.application.form.global_subtractive_selectors, rows=5, placeholder="header -footer -nav -.stockticker -//*[contains(text(), 'Advertisement')]") }} - <span class="pure-form-message-inline"> - <ul> - <li> Remove HTML element(s) by CSS and XPath selectors before text conversion. </li> - <li> Don't paste HTML here, use only CSS and XPath selectors </li> - <li> Add multiple elements, CSS or XPath selectors per line to ignore multiple parts of the HTML. </li> - </ul> - </span> - </fieldset> - <fieldset class="pure-group"> - {{ render_field(form.application.form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line -/some.regex\d{2}/ for case-INsensitive regex - ") }} - <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br> - <span class="pure-form-message-inline"> - <ul> - <li>Matching text will be <strong>ignored</strong> in the text snapshot (you can still see it but it wont trigger a change)</li> - <li>Note: This is applied globally in addition to the per-watch rules.</li> - <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li> - <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li> - <li>Changing this will affect the comparison checksum which may trigger an alert</li> - </ul> - </span> - </fieldset> - </div> - - <div class="tab-pane-inner" id="api"> - <h4>API Access</h4> - <p>Drive your changedetection.io via API, More about <a href="https://github.com/dgtlmoon/changedetection.io/wiki/API-Reference">API access here</a></p> - - <div class="pure-control-group"> - {{ render_checkbox_field(form.application.form.api_access_token_enabled) }} - <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header - required for the Chrome Extension to work</div><br> - <div class="pure-form-message-inline"><br>API Key <span id="api-key">{{api_key}}</span> - <span style="display:none;" id="api-key-copy" >copy</span> - </div> - </div> - <div class="pure-control-group"> - <a href="{{url_for('settings.settings_reset_api_key')}}" class="pure-button button-small button-cancel">Regenerate API key</a> - </div> - <div class="pure-control-group"> - <h4>Chrome Extension</h4> - <p>Easily add any web-page to your changedetection.io installation from within Chrome.</p> - <strong>Step 1</strong> Install the extension, <strong>Step 2</strong> Navigate to this page, - <strong>Step 3</strong> Open the extension from the toolbar and click "<i>Sync API Access</i>" - <p> - <a id="chrome-extension-link" - title="Try our new Chrome Extension!" - href="https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop"> - <img alt="Chrome store icon" src="{{ url_for('static_content', group='images', filename='Google-Chrome-icon.png') }}" alt="Chrome"> - Chrome Webstore - </a> - </p> - </div> - </div> - <div class="tab-pane-inner" id="timedate"> - <div class="pure-control-group"> - Ensure the settings below are correct, they are used to manage the time schedule for checking your web page watches. - </div> - <div class="pure-control-group"> - <p><strong>UTC Time & Date from Server:</strong> <span id="utc-time" >{{ utc_time }}</span></p> - <p><strong>Local Time & Date in Browser:</strong> <span class="local-time" data-utc="{{ utc_time }}"></span></p> - <p> - {{ render_field(form.application.form.timezone) }} - <datalist id="timezones" style="display: none;"> - {% for tz_name in available_timezones %} - <option value="{{ tz_name }}">{{ tz_name }}</option> - {% endfor %} - </datalist> - </p> - </div> - </div> - <div class="tab-pane-inner" id="proxies"> - <div id="recommended-proxy"> - <div> - <img style="height: 2em;" src="{{url_for('static_content', group='images', filename='brightdata.svg')}}" alt="BrightData Proxy Provider"> - <p>BrightData offer world-class proxy services, "Data Center" proxies are a very affordable way to proxy your requests, whilst <strong><a href="https://brightdata.grsm.io/n0r16zf7eivq">WebUnlocker</a></strong> can help solve most CAPTCHAs.</p> - <p> - BrightData offer many <a href="https://brightdata.com/proxy-types" target="new">many different types of proxies</a>, it is worth reading about what is best for your use-case. - </p> - - <p> - When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the access Proxy URL into the "Extra Proxies" boxes below.<br> - </p> - <p> - The Proxy URL with BrightData should start with <code>http://brd-customer...</code> - </p> - <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p> - </div> - <div> - <img style="height: 2em;" - src="{{url_for('static_content', group='images', filename='oxylabs.svg')}}" - alt="Oxylabs Proxy Provider"> - <p> - Collect public data at scale with industry-leading web scraping solutions and the world’s - largest ethical proxy network. - </p> - <p> - Oxylabs also provide a <a href="https://oxylabs.io/products/web-unblocker"><strong>WebUnlocker</strong></a> - proxy that bypasses sophisticated anti-bot systems, so you don’t have to.<br> - </p> - <p> - Serve over <a href="https://oxylabs.io/location-proxy">195 countries</a>, providing <a - href="https://oxylabs.io/products/residential-proxy-pool">Residential</a>, <a - href="https://oxylabs.io/products/mobile-proxies">Mobile</a> and <a - href="https://oxylabs.io/products/rotating-isp-proxies">ISP proxies</a> and much more. - </p> - <p> - Use the promo code <strong>boost35</strong> with this link <a href="https://oxylabs.go2cloud.org/SH2d">https://oxylabs.go2cloud.org/SH2d</a> for 35% off Residential, Mobile proxies, Web Unblocker, and Scraper APIs. Built-in proxies enable you to access data from all around the world and help overcome anti-bot solutions. - - </p> - - - </div> - </div> - - <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites. - - <div class="pure-control-group" id="extra-proxies-setting"> - {{ render_field(form.requests.form.extra_proxies) }} - <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span><br> - <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span> - </div> - <div class="pure-control-group" id="extra-browsers-setting"> - <p> - <span class="pure-form-message-inline"><i>Extra Browsers</i> can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.</span><br> - <span class="pure-form-message-inline">Simply paste the connection address into the box, <a href="https://changedetection.io/tutorial/using-bright-datas-scraping-browser-pass-captchas-and-other-protection-when-monitoring">More instructions and examples here</a> </span> - </p> - {{ render_field(form.requests.form.extra_browsers) }} - </div> - </div> - <div id="actions"> - <div class="pure-control-group"> - {{ render_button(form.save_button) }} - <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a> - <a href="{{url_for('ui.clear_all_history')}}" class="pure-button button-small button-error">Clear Snapshot History</a> - </div> - </div> - </form> - </div> -</div> - -{% endblock %} diff --git a/changedetectionio/tests/test_processor_plugins.py b/changedetectionio/tests/test_processor_plugins.py new file mode 100644 index 00000000..7b66ff6c --- /dev/null +++ b/changedetectionio/tests/test_processor_plugins.py @@ -0,0 +1,120 @@ +import pytest +from time import sleep +from copy import deepcopy +from ..processors import pluggy_interface +from ..processors.pluggy_interface import PLUGIN_NAMESPACE +from ..processors import get_all_plugins_info, available_processors, get_form_class_for_processor +from ..processors.text_json_diff.processor import perform_site_check + +def test_plugin_interfaces(): + """Test that the plugin interface is functioning correctly""" + # The plugin manager should be already set up + assert pluggy_interface.plugin_manager is not None + assert pluggy_interface.plugin_manager.get_namespace() == PLUGIN_NAMESPACE + + # Check that we can get plugins + plugins = pluggy_interface.plugin_manager.get_plugins() + assert len(plugins) >= 3 # Should have at least the 3 built-in plugins + + # Check that the TextJsonDiffPlugin is registered + for plugin in plugins: + if hasattr(plugin, "get_processor_name") and plugin.get_processor_name() == "text_json_diff": + assert plugin.get_processor_description() is not None + assert plugin.get_processor_version() is not None + break + else: + assert False, "TextJsonDiffPlugin not found" + + # Check plugin info collection + plugin_info = get_all_plugins_info() + assert len(plugin_info) >= 3 + + # Check processor list generation + processor_list = available_processors() + assert len(processor_list) >= 3 + + # Ensure each processor has a name and description + for name, description in processor_list: + assert name is not None + assert description is not None + +def test_plugin_form_and_model_handling(): + """Test that plugin form and model handling works""" + # Test getting the form class for text_json_diff + form_class = get_form_class_for_processor("text_json_diff") + assert form_class is not None + + # Test getting the form class for a non-existent processor + form_class = get_form_class_for_processor("non_existent_processor") + assert form_class is not None # Should return the default text_json_diff form + +def test_plugin_enabled_filters(client, live_server): + """Test that enabled plugins filter works""" + # Create a fake datastore with plugin settings and tracking for writes + datastore = type('MockDatastore', (object,), { + 'data': { + 'settings': { + 'application': { + 'enabled_plugins': { + 'text_json_diff': True, + 'restock_diff': False, + 'example_processor': True + } + } + } + }, + 'needs_write': False + }) + + # Get processors filtered by enabled status + processor_list = available_processors(datastore) + + # Should have text_json_diff and example_processor, but not restock_diff + processor_names = [name for name, desc in processor_list] + assert 'text_json_diff' in processor_names + assert 'example_processor' in processor_names + assert 'restock_diff' not in processor_names + + # Test with empty enabled_plugins (should auto-populate with defaults) + datastore.data['settings']['application']['enabled_plugins'] = {} + processor_list = available_processors(datastore) + + # Check that it detected and auto-populated missing plugins + assert len(datastore.data['settings']['application']['enabled_plugins']) >= 3 + assert datastore.needs_write == True + + # Built-in processors should be enabled by default + assert datastore.data['settings']['application']['enabled_plugins']['text_json_diff'] == True + assert datastore.data['settings']['application']['enabled_plugins']['restock_diff'] == True + + # Third-party processors should be disabled by default + assert datastore.data['settings']['application']['enabled_plugins']['example_processor'] == False + + # Only enabled processors should be in the list + processor_names = [name for name, desc in processor_list] + assert 'text_json_diff' in processor_names + assert 'restock_diff' in processor_names + assert 'example_processor' not in processor_names + +def test_plugin_example_implementation(): + """Test the example plugin implementation""" + from ..processors.example_processor_plugin import ExampleProcessorPlugin + + plugin = ExampleProcessorPlugin() + assert plugin.get_processor_name() == "example_processor" + assert "Example Processor Plugin" in plugin.get_processor_description() + assert plugin.get_processor_version() is not None + + # Test the form class + form_class = plugin.get_form_class(processor_name="example_processor") + assert form_class is not None + assert hasattr(form_class, "example_settings") + + # Test the model class + model_class = plugin.get_watch_model_class(processor_name="example_processor") + assert model_class is not None + + # Create an instance of the model and check its methods + model_instance = model_class() + assert hasattr(model_instance, "get_example_threshold") + assert hasattr(model_instance, "is_example_mode_enabled") \ No newline at end of file diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 28647bad..2845b7a5 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -270,20 +270,16 @@ class update_worker(threading.Thread): logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}") try: + # Get processor handler from pluggy plugin system + from changedetectionio.processors import get_processor_handler + # Processor is what we are using for detecting the "Change" - processor = watch.get('processor', 'text_json_diff') - - # Init a new 'difference_detection_processor', first look in processors - processor_module_name = f"changedetectionio.processors.{processor}.processor" - try: - processor_module = importlib.import_module(processor_module_name) - except ModuleNotFoundError as e: - print(f"Processor module '{processor}' not found.") - raise e - - update_handler = processor_module.perform_site_check(datastore=self.datastore, - watch_uuid=uuid - ) + processor_name = watch.get('processor', 'text_json_diff') + + # Get the handler via the plugin system + update_handler = get_processor_handler(processor_name=processor_name, + datastore=self.datastore, + watch_uuid=uuid) update_handler.call_browser()