diff --git a/changedetectionio/api/Import.py b/changedetectionio/api/Import.py index 997593e8c..88626640c 100644 --- a/changedetectionio/api/Import.py +++ b/changedetectionio/api/Import.py @@ -3,7 +3,7 @@ from flask_restful import abort, Resource from flask import request from functools import wraps from . import auth, validate_openapi_request -from ..html_tools import is_safe_valid_url +from ..validate_url import is_safe_valid_url def default_content_type(content_type='text/plain'): diff --git a/changedetectionio/api/Watch.py b/changedetectionio/api/Watch.py index b31a13542..d63d3f4c5 100644 --- a/changedetectionio/api/Watch.py +++ b/changedetectionio/api/Watch.py @@ -1,6 +1,6 @@ import os -from changedetectionio.html_tools import is_safe_valid_url +from changedetectionio.validate_url import is_safe_valid_url from flask_expects_json import expects_json from changedetectionio import queuedWatchMetaData diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 413547d41..399f568c1 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -135,7 +135,7 @@ def get_socketio_path(): @app.template_global('is_safe_valid_url') def _is_safe_valid_url(test_url): - from .html_tools import is_safe_valid_url + from .validate_url import is_safe_valid_url return is_safe_valid_url(test_url) diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 183a6cf91..d169e8bcd 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -538,7 +538,7 @@ class validateURL(object): def validate_url(test_url): - from changedetectionio.html_tools import is_safe_valid_url + from changedetectionio.validate_url import is_safe_valid_url if not is_safe_valid_url(test_url): # This should be wtforms.validators. raise ValidationError('Watch protocol is not permitted or invalid URL format') diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index 46c0d0f9a..ba84207ee 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -22,56 +22,6 @@ class JSONNotFound(ValueError): def __init__(self, msg): ValueError.__init__(self, msg) -@lru_cache(maxsize=10000) -def is_safe_valid_url(test_url): - from changedetectionio import strtobool - from changedetectionio.jinja2_custom import render as jinja_render - from urllib.parse import urlparse, parse_qs - import os - import re - import validators - - allow_file_access = strtobool(os.getenv('ALLOW_FILE_URI', 'false')) - safe_protocol_regex = '^(http|https|ftp|file):' if allow_file_access else '^(http|https|ftp):' - - # See https://github.com/dgtlmoon/changedetection.io/issues/1358 - - # Remove 'source:' prefix so we dont get 'source:javascript:' etc - # 'source:' is a valid way to tell us to return the source - - r = re.compile('^source:', re.IGNORECASE) - test_url = r.sub('', test_url) - - # Check the actual rendered URL in case of any Jinja markup - try: - test_url = jinja_render(test_url) - except Exception as e: - logger.error(f'URL "{test_url}" is not correct Jinja2? {str(e)}') - return False - - # Be sure the protocol is safe (no file, etcetc) - pattern = re.compile(os.getenv('SAFE_PROTOCOL_REGEX', safe_protocol_regex), re.IGNORECASE) - if not pattern.match(test_url.strip()): - logger.warning(f'URL "{test_url}" is not safe, aborting.') - return False - - # Check query parameters and fragment - if re.search(r'[<>]', test_url): - logger.warning(f'URL "{test_url}" contains suspicious characters') - return False - - # If hosts that only contain alphanumerics are allowed ("localhost" for example) - allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False')) - try: - if not validators.url(test_url, simple_host=allow_simplehost): - logger.warning(f'URLf "{test_url}" failed validation, aborting.') - return False - except validators.ValidationError: - logger.warning(f'URL f"{test_url}" failed validation, aborting.') - return False - - return True - # Doesn't look like python supports forward slash auto enclosure in re.findall # So convert it to inline flag "(?i)foobar" type configuration def perl_style_slash_enclosed_regex_to_options(regex): diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 98dc0094d..399aa1167 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -1,5 +1,5 @@ from blinker import signal -from changedetectionio.html_tools import is_safe_valid_url +from changedetectionio.validate_url import is_safe_valid_url from changedetectionio.strtobool import strtobool from changedetectionio.jinja2_custom import render as jinja_render diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 859181cda..ec0ba823e 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -1,6 +1,6 @@ from changedetectionio.strtobool import strtobool -from changedetectionio.html_tools import is_safe_valid_url +from changedetectionio.validate_url import is_safe_valid_url from flask import ( flash diff --git a/changedetectionio/validate_url.py b/changedetectionio/validate_url.py new file mode 100644 index 000000000..f27b8d7ed --- /dev/null +++ b/changedetectionio/validate_url.py @@ -0,0 +1,110 @@ +from functools import lru_cache +from loguru import logger +from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode + + +def normalize_url_encoding(url): + """ + Safely encode a URL's query parameters, regardless of whether they're already encoded. + + Why this is necessary: + URLs can arrive in various states - some with already encoded query parameters (%20 for spaces), + some with unencoded parameters (literal spaces), or a mix of both. The validators.url() function + requires proper encoding, but simply encoding an already-encoded URL would double-encode it + (e.g., %20 would become %2520). + + This function solves the problem by: + 1. Parsing the URL to extract query parameters + 2. parse_qsl() automatically decodes parameters if they're encoded + 3. urlencode() re-encodes them properly + 4. Returns a consistently encoded URL that will pass validation + + Example: + - Input: "http://example.com/test?time=2025-10-28 09:19" (space not encoded) + - Output: "http://example.com/test?time=2025-10-28+09%3A19" (properly encoded) + + - Input: "http://example.com/test?time=2025-10-28%2009:19" (already encoded) + - Output: "http://example.com/test?time=2025-10-28+09%3A19" (properly encoded) + + Returns a properly encoded URL string. + """ + try: + # Parse the URL into components (scheme, netloc, path, params, query, fragment) + parsed = urlparse(url) + + # Parse query string - this automatically decodes it if encoded + # parse_qsl handles both encoded and unencoded query strings gracefully + query_params = parse_qsl(parsed.query, keep_blank_values=True) + + # Re-encode the query string properly using standard URL encoding + encoded_query = urlencode(query_params, safe='') + + # Reconstruct the URL with properly encoded query string + normalized = urlunparse(( + parsed.scheme, + parsed.netloc, + parsed.path, + parsed.params, + encoded_query, # Use the re-encoded query + parsed.fragment + )) + + return normalized + except Exception as e: + # If parsing fails for any reason, return original URL + logger.debug(f"URL normalization failed for '{url}': {e}") + return url + + +@lru_cache(maxsize=10000) +def is_safe_valid_url(test_url): + from changedetectionio import strtobool + from changedetectionio.jinja2_custom import render as jinja_render + from urllib.parse import urlparse, parse_qs + import os + import re + import validators + + allow_file_access = strtobool(os.getenv('ALLOW_FILE_URI', 'false')) + safe_protocol_regex = '^(http|https|ftp|file):' if allow_file_access else '^(http|https|ftp):' + + # See https://github.com/dgtlmoon/changedetection.io/issues/1358 + + # Remove 'source:' prefix so we dont get 'source:javascript:' etc + # 'source:' is a valid way to tell us to return the source + + r = re.compile('^source:', re.IGNORECASE) + test_url = r.sub('', test_url) + + # Check the actual rendered URL in case of any Jinja markup + try: + test_url = jinja_render(test_url) + except Exception as e: + logger.error(f'URL "{test_url}" is not correct Jinja2? {str(e)}') + return False + + # Normalize URL encoding - handle both encoded and unencoded query parameters + test_url = normalize_url_encoding(test_url) + + # Be sure the protocol is safe (no file, etcetc) + pattern = re.compile(os.getenv('SAFE_PROTOCOL_REGEX', safe_protocol_regex), re.IGNORECASE) + if not pattern.match(test_url.strip()): + logger.warning(f'URL "{test_url}" is not safe, aborting.') + return False + + # Check query parameters and fragment + if re.search(r'[<>]', test_url): + logger.warning(f'URL "{test_url}" contains suspicious characters') + return False + + # If hosts that only contain alphanumerics are allowed ("localhost" for example) + allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False')) + try: + if not validators.url(test_url, simple_host=allow_simplehost): + logger.warning(f'URL "{test_url}" failed validation, aborting.') + return False + except validators.ValidationError: + logger.warning(f'URL f"{test_url}" failed validation, aborting.') + return False + + return True