From 9dbe91e470b6abda3bbfbeb49372bc739d00aafc Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 17 Mar 2025 19:17:18 +0100 Subject: [PATCH] WIP --- .../processors/processor_registry.py | 194 ++++-------------- 1 file changed, 36 insertions(+), 158 deletions(-) diff --git a/changedetectionio/processors/processor_registry.py b/changedetectionio/processors/processor_registry.py index abdd03e9..a3cba847 100644 --- a/changedetectionio/processors/processor_registry.py +++ b/changedetectionio/processors/processor_registry.py @@ -4,22 +4,13 @@ from .pluggy_interface import plugin_manager from typing import Dict, Any, List, Tuple, Optional, TypeVar, Type import functools -# Clear global cache to ensure clean state -_plugin_name_map = {} - -# Import both plugins first to avoid registration order issues +# Import and register internal plugins from . import whois_plugin from . import test_plugin -# Now register them after all imports +# Register plugins plugin_manager.register(whois_plugin) -logger.debug("Registered whois_plugin") plugin_manager.register(test_plugin) -logger.debug("Registered test_plugin") - -# Log the plugins list after registration -all_plugins = plugin_manager.get_plugins() -logger.debug(f"Plugin manager has {len(all_plugins)} plugins after registration") # Load any setuptools entrypoints plugin_manager.load_setuptools_entrypoints("changedetectionio_processors") @@ -32,7 +23,6 @@ ProcessorWatchModel = TypeVar('ProcessorWatchModel') ProcessorInstance = TypeVar('ProcessorInstance') # Cache for plugin name mapping to improve performance -# This will be populated after the first call to _get_plugin_name_map _plugin_name_map: Dict[str, Any] = {} def register_plugin(plugin_module): @@ -48,9 +38,8 @@ def _get_plugin_name_map() -> Dict[str, Any]: """ global _plugin_name_map - # Return cached map if available - but only if it's not empty - if _plugin_name_map and len(_plugin_name_map) > 0: - logger.debug(f"Using cached plugin map: {list(_plugin_name_map.keys())}") + # Return cached map if available + if _plugin_name_map: return _plugin_name_map # Build the map @@ -58,10 +47,8 @@ def _get_plugin_name_map() -> Dict[str, Any]: # Get all plugins from the plugin manager all_plugins = list(plugin_manager.get_plugins()) - logger.debug(f"Processing {len(all_plugins)} plugins from plugin manager: {all_plugins}") - # First, directly check for known plugins to ensure they're included - # This is a backup strategy in case the hook mechanism fails + # First register known internal plugins by name for reliability known_plugins = { 'whois': whois_plugin, 'test': test_plugin @@ -69,73 +56,31 @@ def _get_plugin_name_map() -> Dict[str, Any]: for name, plugin in known_plugins.items(): if plugin in all_plugins: - logger.debug(f"Found known plugin '{name}' in plugin list") result[name] = plugin - # Now process all plugins through the hook system + # Then process remaining plugins through the hook system for plugin in all_plugins: - try: - # For debugging, log the plugin's module name - module_name = getattr(plugin, '__name__', str(plugin)) - logger.debug(f"Processing plugin: {module_name}") + if plugin in known_plugins.values(): + continue # Skip plugins we've already registered - # Call get_processor_name individually for each plugin - try: - # Use a direct attribute call if it exists (more reliable) - if hasattr(plugin, 'get_processor_name'): - plugin_name = plugin.get_processor_name() - logger.debug(f"Direct call to get_processor_name returned: {plugin_name}") - else: - # Fall back to the hook system - name_results = plugin_manager.hook.get_processor_name(plugin=plugin) - if name_results: - plugin_name = name_results[0] - logger.debug(f"Hook call to get_processor_name returned: {plugin_name}") - else: - logger.error(f"Plugin {module_name} did not return a name via hook") - continue + try: + # Get the processor name from this plugin + name_results = plugin_manager.hook.get_processor_name(plugin=plugin) + + if name_results: + plugin_name = name_results[0] - # Add to result if we got a name - if plugin_name: - # Check for collisions - if plugin_name in result and result[plugin_name] != plugin: - logger.warning(f"Plugin name collision: '{plugin_name}' is already registered. " - f"The latest registration will overwrite the previous one.") + # Check for name collisions + if plugin_name in result: + logger.warning(f"Plugin name collision: '{plugin_name}' is already registered") + continue - result[plugin_name] = plugin - logger.debug(f"Successfully registered processor plugin: '{plugin_name}'") - except Exception as e: - logger.error(f"Error getting name for plugin {module_name}: {str(e)}") - + result[plugin_name] = plugin except Exception as e: - import traceback - logger.error(f"Error processing plugin: {str(e)}") - logger.error(traceback.format_exc()) - - # Verify we have all the required plugins - if 'whois' not in result: - logger.error("Critical error: whois plugin is missing from the plugin map!") - # Try to manually add it if it's in the all_plugins list - if whois_plugin in all_plugins: - logger.warning("Manually adding whois_plugin to map") - result['whois'] = whois_plugin - - if 'test' not in result: - logger.error("Critical error: test plugin is missing from the plugin map!") - # Try to manually add it if it's in the all_plugins list - if test_plugin in all_plugins: - logger.warning("Manually adding test_plugin to map") - result['test'] = test_plugin - - # Log all registered plugins for debugging - logger.debug(f"Registered plugins ({len(result)} total): {list(result.keys())}") - - # Cache the map only if we have at least all the expected plugins - if len(result) >= 2: - _plugin_name_map = result - else: - logger.error(f"Not caching plugin map - expected at least 2 plugins but found {len(result)}") + logger.error(f"Error getting processor name from plugin: {str(e)}") + # Cache the map + _plugin_name_map = result return result def _get_plugin_by_name(processor_name: str) -> Optional[Any]: @@ -143,13 +88,7 @@ def _get_plugin_by_name(processor_name: str) -> Optional[Any]: :param processor_name: Name of the processor :return: Plugin object or None """ - plugin_map = _get_plugin_name_map() - plugin = plugin_map.get(processor_name) - - if plugin is None: - logger.error(f"Plugin not found: '{processor_name}'. Available plugins: {list(plugin_map.keys())}") - - return plugin + return _get_plugin_name_map().get(processor_name) def _call_hook_for_plugin(plugin: Any, hook_name: str, default_value: T = None, **kwargs) -> Optional[T]: """Call a hook for a specific plugin and handle exceptions @@ -160,31 +99,16 @@ def _call_hook_for_plugin(plugin: Any, hook_name: str, default_value: T = None, :return: Result of the hook call or default value """ if not plugin: - logger.debug(f"Cannot call hook {hook_name}: plugin is None") return default_value try: - # Ensure the hook exists - if not hasattr(plugin_manager.hook, hook_name): - logger.error(f"Hook {hook_name} does not exist in the plugin manager") - return default_value - hook = getattr(plugin_manager.hook, hook_name) - logger.debug(f"Calling hook {hook_name} for plugin {plugin}") - - # Call the hook with the plugin results = hook(plugin=plugin, **kwargs) - if not results: - logger.debug(f"Hook {hook_name} returned no results") - return default_value - - logger.debug(f"Hook {hook_name} returned {len(results)} results: {results}") - return results[0] + if results: + return results[0] except Exception as e: logger.error(f"Error calling {hook_name} for plugin: {str(e)}") - import traceback - logger.error(traceback.format_exc()) return default_value @@ -194,38 +118,11 @@ def get_all_processors() -> List[Tuple[str, str]]: """ processors = [] - # Get the plugin map - plugin_map = _get_plugin_name_map() - logger.debug(f"Getting descriptions for processors: {list(plugin_map.keys())}") + for processor_name, plugin in _get_plugin_name_map().items(): + description = _call_hook_for_plugin(plugin, 'get_processor_description') + if description: + processors.append((processor_name, description)) - # Process each plugin - for processor_name, plugin in plugin_map.items(): - try: - # Try direct attribute access first (more reliable) - if hasattr(plugin, 'get_processor_description'): - try: - description = plugin.get_processor_description() - logger.debug(f"Got description for {processor_name} via direct call: {description}") - except Exception as e: - logger.error(f"Error calling get_processor_description directly on {processor_name}: {str(e)}") - description = None - else: - # Fall back to hook system - description = _call_hook_for_plugin(plugin, 'get_processor_description') - logger.debug(f"Got description for {processor_name} via hook: {description}") - - # Check if both name and description were returned - if description: - processors.append((processor_name, description)) - logger.debug(f"Added processor {processor_name} with description: {description}") - else: - logger.error(f"No description found for processor {processor_name}") - except Exception as e: - import traceback - logger.error(f"Error getting processor info for {processor_name}: {str(e)}") - logger.error(traceback.format_exc()) - - logger.debug(f"Returning {len(processors)} processors: {processors}") return processors def get_processor_class(processor_name: str) -> Optional[Type[ProcessorClass]]: @@ -241,22 +138,8 @@ def get_processor_form(processor_name: str) -> Optional[Type[ProcessorForm]]: :param processor_name: Name of the processor :return: Processor form class or None """ - # Force clear the plugin cache to ensure we have fresh data - global _plugin_name_map - _plugin_name_map = {} - plugin = _get_plugin_by_name(processor_name) - - if plugin is None: - logger.error(f"Cannot get form for processor '{processor_name}': plugin not found") - return None - - form = _call_hook_for_plugin(plugin, 'get_processor_form') - - if form is None: - logger.error(f"Form class not found for processor '{processor_name}'") - - return form + return _call_hook_for_plugin(plugin, 'get_processor_form') def get_processor_watch_model(processor_name: str) -> Type[ProcessorWatchModel]: """Get processor watch model by name @@ -322,23 +205,18 @@ def get_plugin_processor_modules() -> List[Tuple[Any, str]]: # Import base modules once to avoid repeated imports from changedetectionio.processors.text_json_diff import processor as text_json_diff_processor - - # For each plugin, create a fake module that can be used with find_processors + + # For each plugin, map to a suitable module for find_processors for processor_name, plugin in _get_plugin_name_map().items(): try: - # Get the processor class for this plugin processor_class = _call_hook_for_plugin(plugin, 'get_processor_class') if processor_class: - # Check if this processor inherits from TextJsonDiffProcessor - from changedetectionio.processors.text_json_diff.processor import TextJsonDiffProcessor - if issubclass(processor_class, TextJsonDiffProcessor) or 'TextJsonDiffProcessor' in str(processor_class.__bases__): + # Check if this processor extends the text_json_diff processor + base_class_name = str(processor_class.__bases__[0].__name__) + if base_class_name == 'perform_site_check' or 'TextJsonDiffProcessor' in base_class_name: result.append((text_json_diff_processor, processor_name)) - else: - # For non-inherited processors, could create a mapping to their base module - # Future enhancement: dynamically determine base module based on inheritance - logger.debug(f"Processor {processor_name} does not inherit from TextJsonDiffProcessor") except Exception as e: - logger.error(f"Error determining processor module for {processor_name}: {str(e)}") + logger.error(f"Error mapping processor module for {processor_name}: {str(e)}") return result \ No newline at end of file