changedetection.io/changedetectionio/processors/__init__.py

from abc import abstractmethod
from changedetectionio.content_fetchers.base import Fetcher
from changedetectionio.strtobool import strtobool
from copy import deepcopy
from loguru import logger

import hashlib
import importlib
import inspect
import os
import pkgutil
import re

# Import the plugin manager
from .pluggy_interface import plugin_manager


class difference_detection_processor():

    browser_steps = None
    datastore = None
    fetcher = None
    screenshot = None
    watch = None
    xpath_data = None
    preferred_proxy = None

    def __init__(self, *args, datastore, watch_uuid, **kwargs):
        super().__init__(*args, **kwargs)
        self.datastore = datastore
        self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
        # Generic fetcher that should be extended (requests, playwright etc)
        self.fetcher = Fetcher()
        
    def _get_proxy_for_watch(self, preferred_proxy_id=None):
        """Get proxy configuration based on watch settings and preferred proxy ID
        
        Args:
            preferred_proxy_id: Optional explicit proxy ID to use
            
        Returns:
            dict: Proxy configuration or None if no proxy should be used
            str: Proxy URL or None if no proxy should be used
        """
        # Default to no proxy config
        proxy_config = None
        proxy_url = None
        
        # Check if datastore is available and has get_preferred_proxy_for_watch method
        if hasattr(self, 'datastore') and self.datastore:
            try:
                # Get preferred proxy ID if not provided
                if not preferred_proxy_id and hasattr(self.datastore, 'get_preferred_proxy_for_watch'):
                    # Get the watch UUID if available
                    watch_uuid = None
                    if hasattr(self.watch, 'get'):
                        watch_uuid = self.watch.get('uuid')
                    elif hasattr(self.watch, 'uuid'):
                        watch_uuid = self.watch.uuid
                    
                    if watch_uuid:
                        preferred_proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
                
                # Check if we have a proxy list and a valid proxy ID
                if preferred_proxy_id and hasattr(self.datastore, 'proxy_list') and self.datastore.proxy_list:
                    proxy_info = self.datastore.proxy_list.get(preferred_proxy_id)
                    
                    if proxy_info and 'url' in proxy_info:
                        proxy_url = proxy_info.get('url')
                        logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}'")
                        
                        # Parse the proxy URL to build a proxy dict for requests
                        import urllib.parse
                        parsed_proxy = urllib.parse.urlparse(proxy_url)
                        proxy_type = parsed_proxy.scheme
                        
                        # Extract credentials if present
                        username = None
                        password = None
                        if parsed_proxy.username:
                            username = parsed_proxy.username
                            if parsed_proxy.password:
                                password = parsed_proxy.password
                        
                        # Build the proxy URL without credentials for the proxy dict
                        netloc = parsed_proxy.netloc
                        if '@' in netloc:
                            netloc = netloc.split('@')[1]
                        
                        proxy_addr = f"{proxy_type}://{netloc}"
                        
                        # Create the proxy configuration
                        proxy_config = {
                            'http': proxy_addr,
                            'https': proxy_addr
                        }
                        
                        # Add credentials if present
                        if username:
                            proxy_config['username'] = username
                            if password:
                                proxy_config['password'] = password
            except Exception as e:
                # Log the error but continue without a proxy
                logger.error(f"Error setting up proxy: {str(e)}")
                proxy_config = None
                proxy_url = None
                
        return proxy_config, proxy_url

    def call_browser(self, preferred_proxy_id=None):
        """Fetch content using the appropriate browser/fetcher
        
        This method will:
        1. Determine the appropriate fetcher to use based on watch settings
        2. Set up proxy configuration if needed
        3. Initialize the fetcher with the correct parameters
        4. Configure any browser steps if needed
        
        Args:
            preferred_proxy_id: Optional explicit proxy ID to use
        """
        from requests.structures import CaseInsensitiveDict

        url = self.watch.link

        # Protect against file:, file:/, file:// access, check the real "link" without any meta "source:" etc prepended.
        if re.search(r'^file:', url.strip(), re.IGNORECASE):
            if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
                raise Exception(
                    "file:// type access is denied for security reasons."
                )

        # Requests, playwright, other browser via wss:// etc, fetch_extra_something
        prefer_fetch_backend = self.watch.get('fetch_backend', 'system')

        # Get proxy configuration
        proxy_config, proxy_url = self._get_proxy_for_watch(preferred_proxy_id)

        # Pluggable content self.fetcher
        if not prefer_fetch_backend or prefer_fetch_backend == 'system':
            prefer_fetch_backend = self.datastore.data['settings']['application'].get('fetch_backend')

        # In the case that the preferred fetcher was a browser config with custom connection URL..
        # @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..)
        custom_browser_connection_url = None
        if prefer_fetch_backend.startswith('extra_browser_'):
            (t, key) = prefer_fetch_backend.split('extra_browser_')
            connection = list(
                filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', [])))
            if connection:
                prefer_fetch_backend = 'html_webdriver'
                custom_browser_connection_url = connection[0].get('browser_connection_url')

        # PDF should be html_requests because playwright will serve it up (so far) in a embedded page
        # @todo https://github.com/dgtlmoon/changedetection.io/issues/2019
        # @todo needs test to or a fix
        if self.watch.is_pdf:
           prefer_fetch_backend = "html_requests"

        # Grab the right kind of 'fetcher', (playwright, requests, etc)
        from changedetectionio import content_fetchers
        if hasattr(content_fetchers, prefer_fetch_backend):
            # @todo TEMPORARY HACK - SWITCH BACK TO PLAYWRIGHT FOR BROWSERSTEPS
            if prefer_fetch_backend == 'html_webdriver' and self.watch.has_browser_steps:
                # This is never supported in selenium anyway
                logger.warning("Using playwright fetcher override for possible puppeteer request in browsersteps, because puppetteer:browser steps is incomplete.")
                from changedetectionio.content_fetchers.playwright import fetcher as playwright_fetcher
                fetcher_obj = playwright_fetcher
            else:
                fetcher_obj = getattr(content_fetchers, prefer_fetch_backend)
        else:
            # What it referenced doesnt exist, Just use a default
            fetcher_obj = getattr(content_fetchers, "html_requests")

        # Custom browser endpoints should NOT have a proxy added
        if proxy_url and prefer_fetch_backend.startswith('extra_browser_'):
            logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified.")
            proxy_url = None

        # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
        # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
        self.fetcher = fetcher_obj(proxy_override=proxy_url,
                                   custom_browser_connection_url=custom_browser_connection_url
                                   )

        if self.watch.has_browser_steps:
            self.fetcher.browser_steps = self.watch.get('browser_steps', [])
            self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))

        # Tweak the base config with the per-watch ones
        from changedetectionio.safe_jinja import render as jinja_render
        request_headers = CaseInsensitiveDict()

        ua = self.datastore.data['settings']['requests'].get('default_ua')
        if ua and ua.get(prefer_fetch_backend):
            request_headers.update({'User-Agent': ua.get(prefer_fetch_backend)})

        request_headers.update(self.watch.get('headers', {}))
        request_headers.update(self.datastore.get_all_base_headers())
        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=self.watch.get('uuid')))

        # https://github.com/psf/requests/issues/4525
        # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
        # do this by accident.
        if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
            request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')

        for header_name in request_headers:
            request_headers.update({header_name: jinja_render(template_str=request_headers.get(header_name))})

        timeout = self.datastore.data['settings']['requests'].get('timeout')

        request_body = self.watch.get('body')
        if request_body:
            request_body = jinja_render(template_str=self.watch.get('body'))
        
        request_method = self.watch.get('method')
        ignore_status_codes = self.watch.get('ignore_status_codes', False)

        # Configurable per-watch or global extra delay before extracting text (for webDriver types)
        system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)
        if self.watch.get('webdriver_delay'):
            self.fetcher.render_extract_delay = self.watch.get('webdriver_delay')
        elif system_webdriver_delay is not None:
            self.fetcher.render_extract_delay = system_webdriver_delay

        if self.watch.get('webdriver_js_execute_code') is not None and self.watch.get('webdriver_js_execute_code').strip():
            self.fetcher.webdriver_js_execute_code = self.watch.get('webdriver_js_execute_code')

        # Requests for PDF's, images etc should be passwd the is_binary flag
        is_binary = self.watch.is_pdf

        # And here we go! call the right browser with browser-specific settings
        empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)

        self.fetcher.run(url=url,
                         timeout=timeout,
                         request_headers=request_headers,
                         request_body=request_body,
                         request_method=request_method,
                         ignore_status_codes=ignore_status_codes,
                         current_include_filters=self.watch.get('include_filters'),
                         is_binary=is_binary,
                         empty_pages_are_a_change=empty_pages_are_a_change
                         )

        #@todo .quit here could go on close object, so we can run JS if change-detected
        self.fetcher.quit()

        # After init, call run_changedetection() which will do the actual change-detection

    @abstractmethod
    def run_changedetection(self, watch):
        update_obj = {'last_notification_error': False, 'last_error': False}
        some_data = 'xxxxx'
        update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()
        changed_detected = False
        return changed_detected, update_obj, ''.encode('utf-8')


def find_sub_packages(package_name):
    """
    Find all sub-packages within the given package.

    :param package_name: The name of the base package to scan for sub-packages.
    :return: A list of sub-package names.
    """
    package = importlib.import_module(package_name)
    return [name for _, name, is_pkg in pkgutil.iter_modules(package.__path__) if is_pkg]


def find_processors():
    """
    Find all subclasses of DifferenceDetectionProcessor in the specified package
    and also include processors from the plugin system.

    :return: A list of (module, class) tuples.
    """
    package_name = "changedetectionio.processors"  # Name of the current package/module

    processors = []
    sub_packages = find_sub_packages(package_name)

    # Find traditional processors
    for sub_package in sub_packages:
        module_name = f"{package_name}.{sub_package}.processor"
        try:
            module = importlib.import_module(module_name)

            # Iterate through all classes in the module
            for name, obj in inspect.getmembers(module, inspect.isclass):
                if issubclass(obj, difference_detection_processor) and obj is not difference_detection_processor:
                    processors.append((module, sub_package))
        except (ModuleNotFoundError, ImportError) as e:
            logger.warning(f"Failed to import module {module_name}: {e} (find_processors())")

    # Also include processors from the plugin system
    try:
        from .processor_registry import get_plugin_processor_modules
        plugin_modules = get_plugin_processor_modules()
        if plugin_modules:
            processors.extend(plugin_modules)
    except (ImportError, ModuleNotFoundError) as e:
        logger.warning(f"Failed to import plugin modules: {e} (find_processors())")

    return processors


def get_parent_module(module):
    module_name = module.__name__
    if '.' not in module_name:
        return None  # Top-level module has no parent
    parent_module_name = module_name.rsplit('.', 1)[0]
    try:
        return importlib.import_module(parent_module_name)
    except Exception as e:
        pass

    return False


def get_custom_watch_obj_for_processor(processor_name):
    """
    Get the custom watch object for a processor
    :param processor_name: Name of the processor
    :return: Watch class or None
    """
    # First, try to get the watch model from the pluggy system
    try:
        from .processor_registry import get_processor_watch_model
        watch_model = get_processor_watch_model(processor_name)
        if watch_model:
            return watch_model
    except Exception as e:
        logger.warning(f"Error getting processor watch model from pluggy: {e}")

    # Fall back to the traditional approach
    from changedetectionio.model import Watch
    watch_class = Watch.model
    processor_classes = find_processors()
    custom_watch_obj = next((tpl for tpl in processor_classes if tpl[1] == processor_name), None)
    if custom_watch_obj:
        # Parent of .processor.py COULD have its own Watch implementation
        parent_module = get_parent_module(custom_watch_obj[0])
        if hasattr(parent_module, 'Watch'):
            watch_class = parent_module.Watch

    return watch_class


def available_processors():
    """
    Get a list of processors by name and description for the UI elements
    :return: A list of tuples (processor_name, description)
    """
    # Get processors from the pluggy system
    pluggy_processors = []
    try:
        from .processor_registry import get_all_processors
        pluggy_processors = get_all_processors()
    except Exception as e:
        logger.error(f"Error getting processors from pluggy: {str(e)}")
    
    # Get processors from the traditional file-based system
    traditional_processors = []
    try:
        # Let's not use find_processors() directly since it now also includes pluggy processors
        package_name = "changedetectionio.processors"
        sub_packages = find_sub_packages(package_name)
        
        for sub_package in sub_packages:
            module_name = f"{package_name}.{sub_package}.processor"
            try:
                module = importlib.import_module(module_name)
                # Get the name and description from the module if available
                name = getattr(module, 'name', f"Traditional processor: {sub_package}")
                description = getattr(module, 'description', sub_package)
                traditional_processors.append((sub_package, name))
            except (ModuleNotFoundError, ImportError, AttributeError) as e:
                logger.warning(f"Failed to import module {module_name}: {e} (available_processors())")
    except Exception as e:
        logger.error(f"Error getting traditional processors: {str(e)}")
    
    # Combine the lists, ensuring no duplicates
    # Pluggy processors take precedence
    all_processors = []
    
    # Add all pluggy processors
    all_processors.extend(pluggy_processors)
    
    # Add traditional processors that aren't already registered via pluggy
    pluggy_processor_names = [name for name, _ in pluggy_processors]
    for processor_class, name in traditional_processors:
        if processor_class not in pluggy_processor_names:
            all_processors.append((processor_class, name))
    
    return all_processors