refactor and handle spaces etc

changedetectionio/api/Import.py

@@ -3,7 +3,7 @@ from flask_restful import abort, Resource
 from flask import request
 from functools import wraps
 from . import auth, validate_openapi_request
-from ..html_tools import is_safe_valid_url
+from ..validate_url import is_safe_valid_url
 
 
 def default_content_type(content_type='text/plain'):

changedetectionio/api/Watch.py

@@ -1,6 +1,6 @@
 import os
 
-from changedetectionio.html_tools import is_safe_valid_url
+from changedetectionio.validate_url import is_safe_valid_url
 
 from flask_expects_json import expects_json
 from changedetectionio import queuedWatchMetaData

changedetectionio/flask_app.py

@@ -135,7 +135,7 @@ def get_socketio_path():
 
 @app.template_global('is_safe_valid_url')
 def _is_safe_valid_url(test_url):
-    from .html_tools import is_safe_valid_url
+    from .validate_url import is_safe_valid_url
     return is_safe_valid_url(test_url)
 
 

changedetectionio/forms.py

@@ -538,7 +538,7 @@ class validateURL(object):
 
 
 def validate_url(test_url):
-    from changedetectionio.html_tools import is_safe_valid_url
+    from changedetectionio.validate_url import is_safe_valid_url
     if not is_safe_valid_url(test_url):
         # This should be wtforms.validators.
         raise ValidationError('Watch protocol is not permitted or invalid URL format')

changedetectionio/html_tools.py

@@ -22,56 +22,6 @@ class JSONNotFound(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
 
-@lru_cache(maxsize=10000)
-def is_safe_valid_url(test_url):
-    from changedetectionio import strtobool
-    from changedetectionio.jinja2_custom import render as jinja_render
-    from urllib.parse import urlparse, parse_qs
-    import os
-    import re
-    import validators
-
-    allow_file_access = strtobool(os.getenv('ALLOW_FILE_URI', 'false'))
-    safe_protocol_regex = '^(http|https|ftp|file):' if allow_file_access else '^(http|https|ftp):'
-
-    # See https://github.com/dgtlmoon/changedetection.io/issues/1358
-
-    # Remove 'source:' prefix so we dont get 'source:javascript:' etc
-    # 'source:' is a valid way to tell us to return the source
-
-    r = re.compile('^source:', re.IGNORECASE)
-    test_url = r.sub('', test_url)
-
-    # Check the actual rendered URL in case of any Jinja markup
-    try:
-        test_url = jinja_render(test_url)
-    except Exception as e:
-        logger.error(f'URL "{test_url}" is not correct Jinja2? {str(e)}')
-        return False
-
-    # Be sure the protocol is safe (no file, etcetc)
-    pattern = re.compile(os.getenv('SAFE_PROTOCOL_REGEX', safe_protocol_regex), re.IGNORECASE)
-    if not pattern.match(test_url.strip()):
-        logger.warning(f'URL "{test_url}" is not safe, aborting.')
-        return False
-
-    # Check query parameters and fragment
-    if re.search(r'[<>]', test_url):
-        logger.warning(f'URL "{test_url}" contains suspicious characters')
-        return False
-
-    # If hosts that only contain alphanumerics are allowed ("localhost" for example)
-    allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
-    try:
-        if not validators.url(test_url, simple_host=allow_simplehost):
-            logger.warning(f'URLf "{test_url}" failed validation, aborting.')
-            return False
-    except validators.ValidationError:
-        logger.warning(f'URL f"{test_url}" failed validation, aborting.')
-        return False
-
-    return True
-
 # Doesn't look like python supports forward slash auto enclosure in re.findall
 # So convert it to inline flag "(?i)foobar" type configuration
 def perl_style_slash_enclosed_regex_to_options(regex):

changedetectionio/model/Watch.py

@@ -1,5 +1,5 @@
 from blinker import signal
-from changedetectionio.html_tools import is_safe_valid_url
+from changedetectionio.validate_url import is_safe_valid_url
 
 from changedetectionio.strtobool import strtobool
 from changedetectionio.jinja2_custom import render as jinja_render

changedetectionio/store.py

@@ -1,6 +1,6 @@
 from changedetectionio.strtobool import strtobool
 
-from changedetectionio.html_tools import is_safe_valid_url
+from changedetectionio.validate_url import is_safe_valid_url
 
 from flask import (
     flash

changedetectionio/validate_url.py

@@ -0,0 +1,110 @@
+from functools import lru_cache
+from loguru import logger
+from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
+
+
+def normalize_url_encoding(url):
+    """
+    Safely encode a URL's query parameters, regardless of whether they're already encoded.
+
+    Why this is necessary:
+    URLs can arrive in various states - some with already encoded query parameters (%20 for spaces),
+    some with unencoded parameters (literal spaces), or a mix of both. The validators.url() function
+    requires proper encoding, but simply encoding an already-encoded URL would double-encode it
+    (e.g., %20 would become %2520).
+
+    This function solves the problem by:
+    1. Parsing the URL to extract query parameters
+    2. parse_qsl() automatically decodes parameters if they're encoded
+    3. urlencode() re-encodes them properly
+    4. Returns a consistently encoded URL that will pass validation
+
+    Example:
+    - Input:  "http://example.com/test?time=2025-10-28 09:19"  (space not encoded)
+    - Output: "http://example.com/test?time=2025-10-28+09%3A19" (properly encoded)
+
+    - Input:  "http://example.com/test?time=2025-10-28%2009:19" (already encoded)
+    - Output: "http://example.com/test?time=2025-10-28+09%3A19" (properly encoded)
+
+    Returns a properly encoded URL string.
+    """
+    try:
+        # Parse the URL into components (scheme, netloc, path, params, query, fragment)
+        parsed = urlparse(url)
+
+        # Parse query string - this automatically decodes it if encoded
+        # parse_qsl handles both encoded and unencoded query strings gracefully
+        query_params = parse_qsl(parsed.query, keep_blank_values=True)
+
+        # Re-encode the query string properly using standard URL encoding
+        encoded_query = urlencode(query_params, safe='')
+
+        # Reconstruct the URL with properly encoded query string
+        normalized = urlunparse((
+            parsed.scheme,
+            parsed.netloc,
+            parsed.path,
+            parsed.params,
+            encoded_query,  # Use the re-encoded query
+            parsed.fragment
+        ))
+
+        return normalized
+    except Exception as e:
+        # If parsing fails for any reason, return original URL
+        logger.debug(f"URL normalization failed for '{url}': {e}")
+        return url
+
+
+@lru_cache(maxsize=10000)
+def is_safe_valid_url(test_url):
+    from changedetectionio import strtobool
+    from changedetectionio.jinja2_custom import render as jinja_render
+    from urllib.parse import urlparse, parse_qs
+    import os
+    import re
+    import validators
+
+    allow_file_access = strtobool(os.getenv('ALLOW_FILE_URI', 'false'))
+    safe_protocol_regex = '^(http|https|ftp|file):' if allow_file_access else '^(http|https|ftp):'
+
+    # See https://github.com/dgtlmoon/changedetection.io/issues/1358
+
+    # Remove 'source:' prefix so we dont get 'source:javascript:' etc
+    # 'source:' is a valid way to tell us to return the source
+
+    r = re.compile('^source:', re.IGNORECASE)
+    test_url = r.sub('', test_url)
+
+    # Check the actual rendered URL in case of any Jinja markup
+    try:
+        test_url = jinja_render(test_url)
+    except Exception as e:
+        logger.error(f'URL "{test_url}" is not correct Jinja2? {str(e)}')
+        return False
+
+    # Normalize URL encoding - handle both encoded and unencoded query parameters
+    test_url = normalize_url_encoding(test_url)
+
+    # Be sure the protocol is safe (no file, etcetc)
+    pattern = re.compile(os.getenv('SAFE_PROTOCOL_REGEX', safe_protocol_regex), re.IGNORECASE)
+    if not pattern.match(test_url.strip()):
+        logger.warning(f'URL "{test_url}" is not safe, aborting.')
+        return False
+
+    # Check query parameters and fragment
+    if re.search(r'[<>]', test_url):
+        logger.warning(f'URL "{test_url}" contains suspicious characters')
+        return False
+
+    # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+    allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+    try:
+        if not validators.url(test_url, simple_host=allow_simplehost):
+            logger.warning(f'URL "{test_url}" failed validation, aborting.')
+            return False
+    except validators.ValidationError:
+        logger.warning(f'URL f"{test_url}" failed validation, aborting.')
+        return False
+
+    return True