lee/changedetection.io
1
0
Fork 0
mirror of https://github.com/dgtlmoon/changedetection.io.git synced 2026-05-01 23:30:33 +00:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
dgtlmoon
2025-03-17 19:17:18 +01:00
parent 51bd8cd2d7
commit 9dbe91e470
1 changed files with 36 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
changedetectionio/processors/processor_registry.py
+36 -158
View File
@@ -4,22 +4,13 @@ from .pluggy_interface import plugin_manager
from typing import Dict, Any, List, Tuple, Optional, TypeVar, Type
import functools
# Clear global cache to ensure clean state
_plugin_name_map = {}
# Import both plugins first to avoid registration order issues
# Import and register internal plugins
from . import whois_plugin
from . import test_plugin
# Now register them after all imports
# Register plugins
plugin_manager.register(whois_plugin)
logger.debug("Registered whois_plugin")
plugin_manager.register(test_plugin)
logger.debug("Registered test_plugin")
# Log the plugins list after registration
all_plugins = plugin_manager.get_plugins()
logger.debug(f"Plugin manager has {len(all_plugins)} plugins after registration")
# Load any setuptools entrypoints
plugin_manager.load_setuptools_entrypoints("changedetectionio_processors")
@@ -32,7 +23,6 @@ ProcessorWatchModel = TypeVar('ProcessorWatchModel')
ProcessorInstance = TypeVar('ProcessorInstance')
# Cache for plugin name mapping to improve performance
# This will be populated after the first call to _get_plugin_name_map
_plugin_name_map: Dict[str, Any] = {}
def register_plugin(plugin_module):
@@ -48,9 +38,8 @@ def _get_plugin_name_map() -> Dict[str, Any]:
"""
global _plugin_name_map
# Return cached map if available - but only if it's not empty
if _plugin_name_map and len(_plugin_name_map) > 0:
logger.debug(f"Using cached plugin map: {list(_plugin_name_map.keys())}")
# Return cached map if available
if _plugin_name_map:
return _plugin_name_map
# Build the map
@@ -58,10 +47,8 @@ def _get_plugin_name_map() -> Dict[str, Any]:
# Get all plugins from the plugin manager
all_plugins = list(plugin_manager.get_plugins())
logger.debug(f"Processing {len(all_plugins)} plugins from plugin manager: {all_plugins}")
# First, directly check for known plugins to ensure they're included
# This is a backup strategy in case the hook mechanism fails
# First register known internal plugins by name for reliability
known_plugins = {
'whois': whois_plugin,
'test': test_plugin
@@ -69,73 +56,31 @@ def _get_plugin_name_map() -> Dict[str, Any]:
for name, plugin in known_plugins.items():
if plugin in all_plugins:
logger.debug(f"Found known plugin '{name}' in plugin list")
result[name] = plugin
# Now process all plugins through the hook system
# Then process remaining plugins through the hook system
for plugin in all_plugins:
try:
# For debugging, log the plugin's module name
module_name = getattr(plugin, '__name__', str(plugin))
logger.debug(f"Processing plugin: {module_name}")
if plugin in known_plugins.values():
continue # Skip plugins we've already registered
# Call get_processor_name individually for each plugin
try:
# Use a direct attribute call if it exists (more reliable)
if hasattr(plugin, 'get_processor_name'):
plugin_name = plugin.get_processor_name()
logger.debug(f"Direct call to get_processor_name returned: {plugin_name}")
else:
# Fall back to the hook system
name_results = plugin_manager.hook.get_processor_name(plugin=plugin)
if name_results:
plugin_name = name_results[0]
logger.debug(f"Hook call to get_processor_name returned: {plugin_name}")
else:
logger.error(f"Plugin {module_name} did not return a name via hook")
continue
try:
# Get the processor name from this plugin
name_results = plugin_manager.hook.get_processor_name(plugin=plugin)
if name_results:
plugin_name = name_results[0]
# Add to result if we got a name
if plugin_name:
# Check for collisions
if plugin_name in result and result[plugin_name] != plugin:
logger.warning(f"Plugin name collision: '{plugin_name}' is already registered. "
f"The latest registration will overwrite the previous one.")
# Check for name collisions
if plugin_name in result:
logger.warning(f"Plugin name collision: '{plugin_name}' is already registered")
continue
result[plugin_name] = plugin
logger.debug(f"Successfully registered processor plugin: '{plugin_name}'")
except Exception as e:
logger.error(f"Error getting name for plugin {module_name}: {str(e)}")
result[plugin_name] = plugin
except Exception as e:
import traceback
logger.error(f"Error processing plugin: {str(e)}")
logger.error(traceback.format_exc())
# Verify we have all the required plugins
if 'whois' not in result:
logger.error("Critical error: whois plugin is missing from the plugin map!")
# Try to manually add it if it's in the all_plugins list
if whois_plugin in all_plugins:
logger.warning("Manually adding whois_plugin to map")
result['whois'] = whois_plugin
if 'test' not in result:
logger.error("Critical error: test plugin is missing from the plugin map!")
# Try to manually add it if it's in the all_plugins list
if test_plugin in all_plugins:
logger.warning("Manually adding test_plugin to map")
result['test'] = test_plugin
# Log all registered plugins for debugging
logger.debug(f"Registered plugins ({len(result)} total): {list(result.keys())}")
# Cache the map only if we have at least all the expected plugins
if len(result) >= 2:
_plugin_name_map = result
else:
logger.error(f"Not caching plugin map - expected at least 2 plugins but found {len(result)}")
logger.error(f"Error getting processor name from plugin: {str(e)}")
# Cache the map
_plugin_name_map = result
return result
def _get_plugin_by_name(processor_name: str) -> Optional[Any]:
@@ -143,13 +88,7 @@ def _get_plugin_by_name(processor_name: str) -> Optional[Any]:
:param processor_name: Name of the processor
:return: Plugin object or None
"""
plugin_map = _get_plugin_name_map()
plugin = plugin_map.get(processor_name)
if plugin is None:
logger.error(f"Plugin not found: '{processor_name}'. Available plugins: {list(plugin_map.keys())}")
return plugin
return _get_plugin_name_map().get(processor_name)
def _call_hook_for_plugin(plugin: Any, hook_name: str, default_value: T = None, **kwargs) -> Optional[T]:
"""Call a hook for a specific plugin and handle exceptions
@@ -160,31 +99,16 @@ def _call_hook_for_plugin(plugin: Any, hook_name: str, default_value: T = None,
:return: Result of the hook call or default value
"""
if not plugin:
logger.debug(f"Cannot call hook {hook_name}: plugin is None")
return default_value
try:
# Ensure the hook exists
if not hasattr(plugin_manager.hook, hook_name):
logger.error(f"Hook {hook_name} does not exist in the plugin manager")
return default_value
hook = getattr(plugin_manager.hook, hook_name)
logger.debug(f"Calling hook {hook_name} for plugin {plugin}")
# Call the hook with the plugin
results = hook(plugin=plugin, **kwargs)
if not results:
logger.debug(f"Hook {hook_name} returned no results")
return default_value
logger.debug(f"Hook {hook_name} returned {len(results)} results: {results}")
return results[0]
if results:
return results[0]
except Exception as e:
logger.error(f"Error calling {hook_name} for plugin: {str(e)}")
import traceback
logger.error(traceback.format_exc())
return default_value
@@ -194,38 +118,11 @@ def get_all_processors() -> List[Tuple[str, str]]:
"""
processors = []
# Get the plugin map
plugin_map = _get_plugin_name_map()
logger.debug(f"Getting descriptions for processors: {list(plugin_map.keys())}")
for processor_name, plugin in _get_plugin_name_map().items():
description = _call_hook_for_plugin(plugin, 'get_processor_description')
if description:
processors.append((processor_name, description))
# Process each plugin
for processor_name, plugin in plugin_map.items():
try:
# Try direct attribute access first (more reliable)
if hasattr(plugin, 'get_processor_description'):
try:
description = plugin.get_processor_description()
logger.debug(f"Got description for {processor_name} via direct call: {description}")
except Exception as e:
logger.error(f"Error calling get_processor_description directly on {processor_name}: {str(e)}")
description = None
else:
# Fall back to hook system
description = _call_hook_for_plugin(plugin, 'get_processor_description')
logger.debug(f"Got description for {processor_name} via hook: {description}")
# Check if both name and description were returned
if description:
processors.append((processor_name, description))
logger.debug(f"Added processor {processor_name} with description: {description}")
else:
logger.error(f"No description found for processor {processor_name}")
except Exception as e:
import traceback
logger.error(f"Error getting processor info for {processor_name}: {str(e)}")
logger.error(traceback.format_exc())
logger.debug(f"Returning {len(processors)} processors: {processors}")
return processors
def get_processor_class(processor_name: str) -> Optional[Type[ProcessorClass]]:
@@ -241,22 +138,8 @@ def get_processor_form(processor_name: str) -> Optional[Type[ProcessorForm]]:
:param processor_name: Name of the processor
:return: Processor form class or None
"""
# Force clear the plugin cache to ensure we have fresh data
global _plugin_name_map
_plugin_name_map = {}
plugin = _get_plugin_by_name(processor_name)
if plugin is None:
logger.error(f"Cannot get form for processor '{processor_name}': plugin not found")
return None
form = _call_hook_for_plugin(plugin, 'get_processor_form')
if form is None:
logger.error(f"Form class not found for processor '{processor_name}'")
return form
return _call_hook_for_plugin(plugin, 'get_processor_form')
def get_processor_watch_model(processor_name: str) -> Type[ProcessorWatchModel]:
"""Get processor watch model by name
@@ -322,23 +205,18 @@ def get_plugin_processor_modules() -> List[Tuple[Any, str]]:
# Import base modules once to avoid repeated imports
from changedetectionio.processors.text_json_diff import processor as text_json_diff_processor
# For each plugin, create a fake module that can be used with find_processors
# For each plugin, map to a suitable module for find_processors
for processor_name, plugin in _get_plugin_name_map().items():
try:
# Get the processor class for this plugin
processor_class = _call_hook_for_plugin(plugin, 'get_processor_class')
if processor_class:
# Check if this processor inherits from TextJsonDiffProcessor
from changedetectionio.processors.text_json_diff.processor import TextJsonDiffProcessor
if issubclass(processor_class, TextJsonDiffProcessor) or 'TextJsonDiffProcessor' in str(processor_class.__bases__):
# Check if this processor extends the text_json_diff processor
base_class_name = str(processor_class.__bases__[0].__name__)
if base_class_name == 'perform_site_check' or 'TextJsonDiffProcessor' in base_class_name:
result.append((text_json_diff_processor, processor_name))
else:
# For non-inherited processors, could create a mapping to their base module
# Future enhancement: dynamically determine base module based on inheritance
logger.debug(f"Processor {processor_name} does not inherit from TextJsonDiffProcessor")
except Exception as e:
logger.error(f"Error determining processor module for {processor_name}: {str(e)}")
logger.error(f"Error mapping processor module for {processor_name}: {str(e)}")
return result