diff --git a/.github/workflows/test-stack-reusable-workflow.yml b/.github/workflows/test-stack-reusable-workflow.yml index 959b4c657..67cb0535e 100644 --- a/.github/workflows/test-stack-reusable-workflow.yml +++ b/.github/workflows/test-stack-reusable-workflow.yml @@ -587,6 +587,10 @@ jobs: run: | docker run -e EXTRA_PACKAGES=changedetection.io-osint-processor test-changedetectionio bash -c 'cd changedetectionio;pytest -vvv -s tests/plugins/test_processor.py::test_check_plugin_processor' + - name: Plugin get_html_head_extras hook injects into base.html + run: | + docker run test-changedetectionio bash -c 'cd changedetectionio;pytest -vvv -s tests/plugins/test_html_head_extras.py' + # Container startup tests container-tests: runs-on: ubuntu-latest diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index e734556c5..8a0cc8862 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -2,7 +2,7 @@ # Read more https://github.com/dgtlmoon/changedetection.io/wiki # Semver means never use .01, or 00. Should be .1. -__version__ = '0.54.6' +__version__ = '0.54.7' from changedetectionio.strtobool import strtobool from json.decoder import JSONDecodeError diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 5812409ed..0faa263db 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -212,6 +212,11 @@ def _is_safe_valid_url(test_url): from .validate_url import is_safe_valid_url return is_safe_valid_url(test_url) +@app.template_global('get_html_head_extras') +def _get_html_head_extras(): + from .pluggy_interface import collect_html_head_extras + return collect_html_head_extras() + @app.template_filter('format_number_locale') def _jinja2_filter_format_number_locale(value: float) -> str: diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index ff580b1f5..4a045306a 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -653,9 +653,11 @@ class ValidateCSSJSONXPATHInput(object): # `jq` requires full compilation in windows and so isn't generally available raise ValidationError("jq not support not found") + from changedetectionio.html_tools import validate_jq_expression input = line.replace('jq:', '') try: + validate_jq_expression(input) jq.compile(input) except (ValueError) as e: message = field.gettext('\'%s\' is not a valid jq expression. (%s)') @@ -981,7 +983,7 @@ class globalSettingsApplicationForm(commonSettingsForm): render_kw={"placeholder": "0.1", "style": "width: 8em;"} ) - password = SaltyPasswordField(_l('Password')) + password = SaltyPasswordField(_l('Password'), render_kw={"autocomplete": "new-password"}) pager_size = IntegerField(_l('Pager size'), render_kw={"style": "width: 5em;"}, validators=[validators.NumberRange(min=0, diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index 6175282de..f029ad7a3 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -4,6 +4,7 @@ from loguru import logger from typing import List import html import json +import os import re # HTML added to be sure each result matching a filter (.example) gets converted to a new line by Inscriptis @@ -13,6 +14,45 @@ PERL_STYLE_REGEX = r'^/(.*?)/([a-z]*)?$' TITLE_RE = re.compile(r"]*>(.*?)", re.I | re.S) META_CS = re.compile(r']+charset=["\']?\s*([a-z0-9_\-:+.]+)', re.I) + +# jq builtins that can leak sensitive data or cause harm when user-supplied expressions are executed. +# env/$ENV reads all process environment variables (passwords, API keys, etc.) +# include/import can read arbitrary files from disk +# input/inputs reads beyond the supplied JSON data +# debug/stderr leaks data to stderr +# halt/halt_error terminates the process (DoS) +_JQ_BLOCKED_PATTERNS = [ + (re.compile(r'\benv\b'), 'env (reads environment variables)'), + (re.compile(r'\$ENV\b'), '$ENV (reads environment variables)'), + (re.compile(r'\binclude\b'), 'include (reads files from disk)'), + (re.compile(r'\bimport\b'), 'import (reads files from disk)'), + (re.compile(r'\binputs?\b'), 'input/inputs (reads beyond provided data)'), + (re.compile(r'\bdebug\b'), 'debug (leaks data to stderr)'), + (re.compile(r'\bstderr\b'), 'stderr (leaks data to stderr)'), + (re.compile(r'\bhalt(?:_error)?\b'), 'halt/halt_error (terminates the process)'), + (re.compile(r'\$__loc__\b'), '$__loc__ (leaks file path information)'), + (re.compile(r'\bbuiltins\b'), 'builtins (enumerates available functions)'), + (re.compile(r'\bmodulemeta\b'), 'modulemeta (leaks module information)'), + (re.compile(r'\$JQ_BUILD_CONFIGURATION\b'), '$JQ_BUILD_CONFIGURATION (leaks build information)'), +] + +def validate_jq_expression(expression: str) -> None: + """Raise ValueError if the jq expression uses any dangerous builtin. + + User-supplied jq expressions are executed server-side. Without this check, + builtins like `env` expose every process environment variable (SALTED_PASS, + proxy credentials, API keys, etc.) as watch output. + """ + from changedetectionio.strtobool import strtobool + if strtobool(os.getenv('JQ_ALLOW_RISKY_EXPRESSIONS', 'false')): + return + + for pattern, description in _JQ_BLOCKED_PATTERNS: + if pattern.search(expression): + msg = f"jq expression uses disallowed builtin: {description}" + logger.critical(f"Security: blocked jq expression containing '{description}' - expression: {expression!r}") + raise ValueError(msg) + META_CT = re.compile(r']+http-equiv=["\']?content-type["\']?[^>]*content=["\'][^>]*charset=([a-z0-9_\-:+.]+)', re.I) # 'price' , 'lowPrice', 'highPrice' are usually under here @@ -30,6 +70,12 @@ _DEFAULT_UNSAFE_XPATH3_FUNCTIONS = [ 'unparsed-text-available', 'doc', 'doc-available', + 'json-doc', + 'json-doc-available', + 'collection', # XPath 2.0+: loads XML node collections from arbitrary URIs + 'uri-collection', # XPath 3.0+: enumerates URIs from resource collections + 'transform', # XPath 3.1: XSLT transformation (currently raises, block proactively) + 'load-xquery-module', # XPath 3.1: loads XQuery modules (currently raises, block proactively) 'environment-variable', 'available-environment-variables', ] @@ -378,12 +424,16 @@ def _parse_json(json_data, json_filter): raise Exception("jq not support not found") if json_filter.startswith("jq:"): - jq_expression = jq.compile(json_filter.removeprefix("jq:")) + expr = json_filter.removeprefix("jq:") + validate_jq_expression(expr) + jq_expression = jq.compile(expr) match = jq_expression.input(json_data).all() return _get_stripped_text_from_json_match(match) if json_filter.startswith("jqraw:"): - jq_expression = jq.compile(json_filter.removeprefix("jqraw:")) + expr = json_filter.removeprefix("jqraw:") + validate_jq_expression(expr) + jq_expression = jq.compile(expr) match = jq_expression.input(json_data).all() return '\n'.join(str(item) for item in match) diff --git a/changedetectionio/pluggy_interface.py b/changedetectionio/pluggy_interface.py index 07cd46727..1e7f40d85 100644 --- a/changedetectionio/pluggy_interface.py +++ b/changedetectionio/pluggy_interface.py @@ -174,6 +174,64 @@ class ChangeDetectionSpec: """ pass + @hookspec + def get_html_head_extras(): + """Return HTML to inject into the of every page via base.html. + + Plugins can use this to add ' + ) + + For larger assets, register your own lightweight Flask routes in the plugin + module and point to them with url_for() so the sub-path prefix is handled + automatically:: + + from flask import url_for, Response + from changedetectionio.pluggy_interface import hookimpl + from changedetectionio.flask_app import app as _app + + MY_CSS = ".my-module-example { color: red; }" + MY_JS = "console.log('my_module_content loaded');" + + @_app.route('/my_module_content/css') + def my_module_content_css(): + return Response(MY_CSS, mimetype='text/css', + headers={'Cache-Control': 'max-age=3600'}) + + @_app.route('/my_module_content/js') + def my_module_content_js(): + return Response(MY_JS, mimetype='application/javascript', + headers={'Cache-Control': 'max-age=3600'}) + + @hookimpl + def get_html_head_extras(self): + css = url_for('my_module_content_css') + js = url_for('my_module_content_js') + return ( + f'\\n' + f'' + ) + + Returns: + str or None: Raw HTML string to inject inside , or None + """ + pass + # Set up Plugin Manager plugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE) @@ -606,4 +664,20 @@ def apply_update_finalize(update_handler, watch, datastore, processing_exception except Exception as e: # Don't let plugin errors crash the worker logger.error(f"Error in update_finalize hook: {e}") - logger.exception(f"update_finalize hook exception details:") \ No newline at end of file + logger.exception(f"update_finalize hook exception details:") + + +def collect_html_head_extras(): + """Collect and combine HTML head extras from all plugins. + + Called from a Flask template global so it always runs inside a request context. + This means url_for() works correctly in plugin implementations, including when the + app is deployed under a sub-path via USE_X_SETTINGS / X-Forwarded-Prefix (ProxyFix + sets SCRIPT_NAME so url_for() automatically prepends the prefix). + + Returns: + str: Combined HTML string to inject inside , or empty string + """ + results = plugin_manager.hook.get_html_head_extras() + parts = [r for r in results if r] + return "\n".join(parts) if parts else "" \ No newline at end of file diff --git a/changedetectionio/templates/base.html b/changedetectionio/templates/base.html index aa49b8a1b..b323155f8 100644 --- a/changedetectionio/templates/base.html +++ b/changedetectionio/templates/base.html @@ -45,6 +45,10 @@ {% endif %} + {%- set _html_head_extras = get_html_head_extras() -%} + {%- if _html_head_extras %} + {{ _html_head_extras | safe }} + {%- endif %} diff --git a/changedetectionio/tests/plugins/test_html_head_extras.py b/changedetectionio/tests/plugins/test_html_head_extras.py new file mode 100644 index 000000000..84cebb81c --- /dev/null +++ b/changedetectionio/tests/plugins/test_html_head_extras.py @@ -0,0 +1,83 @@ +"""Test that plugins can inject HTML into base.html via get_html_head_extras hookimpl.""" +import pytest +from flask import url_for, Response + +from changedetectionio.pluggy_interface import hookimpl, plugin_manager + +_MY_JS = "console.log('my_module_content loaded');" +_MY_CSS = ".my-module-example { color: red; }" + + +class _HeadExtrasPlugin: + """Test plugin that injects tags pointing at its own Flask routes.""" + + @hookimpl + def get_html_head_extras(self): + css_url = url_for('test_plugin_my_module_content_css') + js_url = url_for('test_plugin_my_module_content_js') + return ( + f'\n' + f'' + ) + + +@pytest.fixture(scope='module') +def plugin_routes(live_server): + """Register plugin asset routes once per module (Flask routes can't be added twice).""" + app = live_server.app + + @app.route('/test-plugin/my_module_content/css') + def test_plugin_my_module_content_css(): + return Response(_MY_CSS, mimetype='text/css', + headers={'Cache-Control': 'max-age=3600'}) + + @app.route('/test-plugin/my_module_content/js') + def test_plugin_my_module_content_js(): + return Response(_MY_JS, mimetype='application/javascript', + headers={'Cache-Control': 'max-age=3600'}) + + +@pytest.fixture +def head_extras_plugin(plugin_routes): + """Register the hookimpl for one test then unregister it — function-scoped for clean isolation.""" + plugin = _HeadExtrasPlugin() + plugin_manager.register(plugin, name="test_head_extras") + yield plugin + plugin_manager.unregister(name="test_head_extras") + + +def test_plugin_html_injected_into_head(client, live_server, measure_memory_usage, datastore_path, head_extras_plugin): + """get_html_head_extras output must appear inside in the rendered page.""" + res = client.get(url_for("watchlist.index"), follow_redirects=True) + assert res.status_code == 200 + assert b'id="test-head-extra-css"' in res.data, "Plugin tag missing from rendered page" + assert b'id="test-head-extra-js"' in res.data, "Plugin