Ensure JSON is always nicely formatted

unused var
Re #3482 - Ensure JSON is always correctly reformatted with padding
2025-10-30 14:17:40 +00:00 · 2025-10-10 15:35:01 +02:00 · 2025-10-10 13:48:35 +02:00 · 2025-10-10 13:45:55 +02:00 · 2025-10-10 13:44:49 +02:00
3 changed files with 130 additions and 122 deletions
--- a/changedetectionio/html_tools.py
+++ b/changedetectionio/html_tools.py
@@ -1,5 +1,4 @@
 from loguru import logger
-from lxml import etree
 from typing import List
 import html
 import json
@@ -58,13 +57,17 @@ def include_filters(include_filters, html_content, append_pretty_line_formatting

    return html_block

-def subtractive_css_selector(css_selector, html_content):
+def subtractive_css_selector(css_selector, content):
    from bs4 import BeautifulSoup
-    soup = BeautifulSoup(html_content, "html.parser")
+    soup = BeautifulSoup(content, "html.parser")

    # So that the elements dont shift their index, build a list of elements here which will be pointers to their place in the DOM
    elements_to_remove = soup.select(css_selector)

+    if not elements_to_remove:
+        # Better to return the original that rebuild with BeautifulSoup
+        return content
+
    # Then, remove them in a separate loop
    for item in elements_to_remove:
        item.decompose()
@@ -72,6 +75,7 @@ def subtractive_css_selector(css_selector, html_content):
    return str(soup)

 def subtractive_xpath_selector(selectors: List[str], html_content: str) -> str:
+    from lxml import etree
    # Parse the HTML content using lxml
    html_tree = etree.HTML(html_content)

@@ -83,6 +87,10 @@ def subtractive_xpath_selector(selectors: List[str], html_content: str) -> str:
        # Collect elements for each selector
        elements_to_remove.extend(html_tree.xpath(selector))

+    # If no elements were found, return the original HTML content
+    if not elements_to_remove:
+        return html_content
+
    # Then, remove them in a separate loop
    for element in elements_to_remove:
        if element.getparent() is not None:  # Ensure the element has a parent before removing
@@ -295,70 +303,92 @@ def _get_stripped_text_from_json_match(match):

    return stripped_text_from_html

+def extract_json_blob_from_html(content, ensure_is_ldjson_info_type, json_filter):
+    from bs4 import BeautifulSoup
+    stripped_text_from_html = ''
+
+    # Foreach <script json></script> blob.. just return the first that matches json_filter
+    # As a last resort, try to parse the whole <body>
+    soup = BeautifulSoup(content, 'html.parser')
+
+    if ensure_is_ldjson_info_type:
+        bs_result = soup.find_all('script', {"type": "application/ld+json"})
+    else:
+        bs_result = soup.find_all('script')
+    bs_result += soup.find_all('body')
+
+    bs_jsons = []
+
+    for result in bs_result:
+        # result.text is how bs4 magically strips JSON from the body
+        content_start = result.text.lstrip("\ufeff").strip()[:100] if result.text else ''
+        # Skip empty tags, and things that dont even look like JSON
+        if not result.text or not (content_start[0] == '{' or content_start[0] == '['):
+            continue
+        try:
+            json_data = json.loads(result.text)
+            bs_jsons.append(json_data)
+        except json.JSONDecodeError:
+            # Skip objects which cannot be parsed
+            continue
+
+    if not bs_jsons:
+        raise JSONNotFound("No parsable JSON found in this document")
+
+    for json_data in bs_jsons:
+        stripped_text_from_html = _parse_json(json_data, json_filter)
+
+        if ensure_is_ldjson_info_type:
+            # Could sometimes be list, string or something else random
+            if isinstance(json_data, dict):
+                # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
+                # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
+                # @type could also be a list although non-standard ("@type": ["Product", "SubType"],)
+                # LD_JSON auto-extract also requires some content PLUS the ldjson to be present
+                # 1833 - could be either str or dict, should not be anything else
+
+                t = json_data.get('@type')
+                if t and stripped_text_from_html:
+
+                    if isinstance(t, str) and t.lower() == ensure_is_ldjson_info_type.lower():
+                        break
+                    # The non-standard part, some have a list
+                    elif isinstance(t, list):
+                        if ensure_is_ldjson_info_type.lower() in [x.lower().strip() for x in t]:
+                            break
+
+        elif stripped_text_from_html:
+            break
+
+    return stripped_text_from_html
+
 # content - json
 # json_filter - ie json:$..price
 # ensure_is_ldjson_info_type - str "product", optional, "@type == product" (I dont know how to do that as a json selector)
 def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None):
-    from bs4 import BeautifulSoup

    stripped_text_from_html = False
 # https://github.com/dgtlmoon/changedetection.io/pull/2041#issuecomment-1848397161w
    # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags
-    try:
-        # .lstrip("\ufeff") strings ByteOrderMark from UTF8 and still lets the UTF work
-        stripped_text_from_html = _parse_json(json.loads(content.lstrip("\ufeff") ), json_filter)
-    except json.JSONDecodeError as e:
-        logger.warning(str(e))

-        # Foreach <script json></script> blob.. just return the first that matches json_filter
-        # As a last resort, try to parse the whole <body>
-        soup = BeautifulSoup(content, 'html.parser')
+    # Looks like clean JSON, dont bother extracting from HTML

-        if ensure_is_ldjson_info_type:
-            bs_result = soup.find_all('script', {"type": "application/ld+json"})
-        else:
-            bs_result = soup.find_all('script')
-        bs_result += soup.find_all('body')
+    content_start = content.lstrip("\ufeff").strip()[:100]

-        bs_jsons = []
-        for result in bs_result:
-            # Skip empty tags, and things that dont even look like JSON
-            if not result.text or '{' not in result.text:
-                continue
-            try:
-                json_data = json.loads(result.text)
-                bs_jsons.append(json_data)
-            except json.JSONDecodeError:
-                # Skip objects which cannot be parsed
-                continue
-
-        if not bs_jsons:
-            raise JSONNotFound("No parsable JSON found in this document")
-        
-        for json_data in bs_jsons:
-            stripped_text_from_html = _parse_json(json_data, json_filter)
-
-            if ensure_is_ldjson_info_type:
-                # Could sometimes be list, string or something else random
-                if isinstance(json_data, dict):
-                    # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
-                    # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
-                    # @type could also be a list although non-standard ("@type": ["Product", "SubType"],)
-                    # LD_JSON auto-extract also requires some content PLUS the ldjson to be present
-                    # 1833 - could be either str or dict, should not be anything else
-
-                    t = json_data.get('@type')
-                    if t and stripped_text_from_html:
-
-                        if isinstance(t, str) and t.lower() == ensure_is_ldjson_info_type.lower():
-                            break
-                        # The non-standard part, some have a list
-                        elif isinstance(t, list):
-                            if ensure_is_ldjson_info_type.lower() in [x.lower().strip() for x in t]:
-                                break
-
-            elif stripped_text_from_html:
-                break
+    if content_start[0] == '{' or content_start[0] == '[':
+        try:
+            # .lstrip("\ufeff") strings ByteOrderMark from UTF8 and still lets the UTF work
+            stripped_text_from_html = _parse_json(json.loads(content.lstrip("\ufeff")), json_filter)
+        except json.JSONDecodeError as e:
+            logger.warning(f"Error processing JSON {content[:20]}...{str(e)})")
+    else:
+        # Probably something else, go fish inside for it
+        try:
+            stripped_text_from_html = extract_json_blob_from_html(content=content,
+                                                                  ensure_is_ldjson_info_type=ensure_is_ldjson_info_type,
+                                                                  json_filter=json_filter                                                                  )
+        except json.JSONDecodeError as e:
+            logger.warning(f"Error processing JSON while extracting JSON from HTML blob {content[:20]}...{str(e)})")

    if not stripped_text_from_html:
        # Re 265 - Just return an empty string when filter not found
--- a/changedetectionio/processors/text_json_diff/processor.py
+++ b/changedetectionio/processors/text_json_diff/processor.py
@@ -20,7 +20,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 name = 'Webpage Text/HTML, JSON and PDF changes'
 description = 'Detects all text changes where possible'

-json_filter_prefixes = ['json:', 'jq:', 'jqraw:']
+JSON_FILTER_PREFIXES = ['json:', 'jq:', 'jqraw:']

 # Assume it's this type if the server says nothing on content-type
 DEFAULT_WHEN_NO_CONTENT_TYPE_HEADER = 'text/html'
@@ -99,6 +99,10 @@ class FilterConfig:
    def has_include_filters(self):
        return bool(self.include_filters) and bool(self.include_filters[0].strip())

+    @property
+    def has_include_json_filters(self):
+        return any(f.strip().startswith(prefix) for f in self.include_filters for prefix in JSON_FILTER_PREFIXES)
+
    @property
    def has_subtractive_selectors(self):
        return bool(self.subtractive_selectors) and bool(self.subtractive_selectors[0].strip())
@@ -255,15 +259,14 @@ class ContentProcessor:
        )
        return html_content.replace('</body>', metadata + '</body>')

-    def preprocess_json(self, content, has_filters):
+    def preprocess_json(self, raw_content):
        """Format and sort JSON content."""
-        # Force reformat if no filters specified
-        if not has_filters:
-            content = html_tools.extract_json_as_string(content=content, json_filter="json:$")
+        # Then we re-format it, else it does have filters (later on) which will reformat it anyway
+        content = html_tools.extract_json_as_string(content=raw_content, json_filter="json:$")

        # Sort JSON to avoid false alerts from reordering
        try:
-            content = json.dumps(json.loads(content), sort_keys=True)
+            content = json.dumps(json.loads(content), sort_keys=True, indent=4)
        except Exception:
            # Might be malformed JSON, continue anyway
            pass
@@ -294,7 +297,7 @@ class ContentProcessor:
                )

            # JSON filters
-            elif any(filter_rule.startswith(prefix) for prefix in json_filter_prefixes):
+            elif any(filter_rule.startswith(prefix) for prefix in JSON_FILTER_PREFIXES):
                filtered_content += html_tools.extract_json_as_string(
                    content=content,
                    json_filter=filter_rule
@@ -387,9 +390,12 @@ class perform_site_check(difference_detection_processor):
            content = content_processor.preprocess_pdf(raw_content=self.fetcher.raw_content)
            stream_content_type.is_html = True

-        # JSON preprocessing
+        # JSON - Always reformat it nicely for consistency.
+
        if stream_content_type.is_json:
-            content = content_processor.preprocess_json(content, filter_config.has_include_filters)
+            if not filter_config.has_include_json_filters:
+                content = content_processor.preprocess_json(raw_content=content)
+        #else, otherwise it gets sorted/formatted in the filter stage anyway

        # HTML obfuscation workarounds
        if stream_content_type.is_html:
@@ -404,6 +410,8 @@ class perform_site_check(difference_detection_processor):
        html_content = content

        # Apply include filters (CSS, XPath, JSON)
+        # Except for plaintext (incase they tried to confuse the system, it will HTML escape
+        #if not stream_content_type.is_plaintext:
        if filter_config.has_include_filters:
            html_content = content_processor.apply_include_filters(content, stream_content_type)

--- a/changedetectionio/tests/test_jsonpath_jq_selector.py
+++ b/changedetectionio/tests/test_jsonpath_jq_selector.py
@@ -113,14 +113,8 @@ def set_original_ext_response():
    return None

 def set_modified_ext_response():
-    data = """
-    [
-    {
-        "isPriceLowered": false,
-        "status": "Sold",
-        "statusOrig": "sold"
-    },
-    {
+    # This should get reformatted
+    data = """ [ { "isPriceLowered": false,  "status": "Sold",  "statusOrig": "sold" }, {
        "_id": "5e7b3e1fb3262d306323ff1e",
        "listingsType": "consumer",
        "isPriceLowered": false,
@@ -230,30 +224,15 @@ def check_json_filter(json_filter, client, live_server):

    # Add our URL to the import page
    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
-    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
+    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url, extras={"include_filters": json_filter.splitlines()})
    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    wait_for_all_checks(client)

-    # Goto the edit page, add our ignore text
-    # Add our URL to the import page
-    res = client.post(
-        url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"include_filters": json_filter,
-              "url": test_url,
-              "tags": "",
-              "headers": "",
-              "fetch_backend": "html_requests",
-              "time_between_check_use_default": "y"
-              },
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
    # Check it saved
    res = client.get(
-        url_for("ui.ui_edit.edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid=uuid),
    )
    assert bytes(escape(json_filter).encode('utf-8')) in res.data

@@ -272,7 +251,7 @@ def check_json_filter(json_filter, client, live_server):
    assert b'has-unread-changes' in res.data

    # Should not see this, because its not in the JSONPath we entered
-    res = client.get(url_for("ui.ui_views.diff_history_page", uuid="first"))
+    res = client.get(url_for("ui.ui_views.diff_history_page", uuid=uuid))

    # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
    # And #462 - check we see the proper utf-8 string there
@@ -294,32 +273,12 @@ def test_check_jqraw_filter(client, live_server, measure_memory_usage):
 def check_json_filter_bool_val(json_filter, client, live_server):
    set_original_response()

-    # Give the endpoint time to spin up
-    time.sleep(1)
-
    test_url = url_for('test_endpoint', content_type="application/json", _external=True)

-    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
+    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url, extras={"include_filters": [json_filter]})
    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-
    wait_for_all_checks(client)
-    # Goto the edit page, add our ignore text
-    # Add our URL to the import page
-    res = client.post(
-        url_for("ui.ui_edit.edit_page", uuid="first"),
-        data={"include_filters": json_filter,
-              "url": test_url,
-              "tags": "",
-              "headers": "",
-              "fetch_backend": "html_requests",
-              "time_between_check_use_default": "y"
-              },
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data

-    # Give the thread time to pick it up
-    wait_for_all_checks(client)
    #  Make a change
    set_modified_response()

@@ -353,21 +312,16 @@ def test_check_jqraw_filter_bool_val(client, live_server, measure_memory_usage):
 def check_json_ext_filter(json_filter, client, live_server):
    set_original_ext_response()

-    # Give the endpoint time to spin up
-    time.sleep(1)
-
    # Add our URL to the import page
    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
    wait_for_all_checks(client)

    # Goto the edit page, add our ignore text
    # Add our URL to the import page
    res = client.post(
-        url_for("ui.ui_edit.edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid=uuid),
        data={"include_filters": json_filter,
              "url": test_url,
              "tags": "",
@@ -381,7 +335,7 @@ def check_json_ext_filter(json_filter, client, live_server):

    # Check it saved
    res = client.get(
-        url_for("ui.ui_edit.edit_page", uuid="first"),
+        url_for("ui.ui_edit.edit_page", uuid=uuid),
    )
    assert bytes(escape(json_filter).encode('utf-8')) in res.data

@@ -395,6 +349,12 @@ def check_json_ext_filter(json_filter, client, live_server):
    # Give the thread time to pick it up
    wait_for_all_checks(client)

+    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
+    dates = list(watch.history.keys())
+    snapshot_contents = watch.get_history_snapshot(dates[0])
+
+    assert snapshot_contents[0] == '['
+
    # It should have 'has-unread-changes'
    res = client.get(url_for("watchlist.index"))
    assert b'has-unread-changes' in res.data
@@ -456,7 +416,7 @@ def test_correct_header_detect(client, live_server, measure_memory_usage):
    # Like in https://github.com/dgtlmoon/changedetection.io/pull/1593
    # Specify extra html that JSON is sometimes wrapped in - when using SockpuppetBrowser / Puppeteer / Playwrightetc
    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write('<html><body>{"hello" : 123, "world": 123}')
+        f.write('<html><body>{ "world": 123, "hello" : 123}')

    # Add our URL to the import page
    # Check weird casing is cleaned up and detected also
@@ -474,8 +434,18 @@ def test_correct_header_detect(client, live_server, measure_memory_usage):
        follow_redirects=True
    )

-    assert b'&#34;hello&#34;: 123,' in res.data
-    assert b'&#34;world&#34;: 123' in res.data
+
+    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
+    dates = list(watch.history.keys())
+    snapshot_contents = watch.get_history_snapshot(dates[0])
+
+    assert b'&#34;hello&#34;: 123,' in res.data # properly html escaped in the front end
+
+    # Should be correctly formatted and sorted,  ("world" goes to end)
+    assert snapshot_contents == """{
+    "hello": 123,
+    "world": 123
+}"""

    delete_all_watches(client)
Author	SHA1	Message	Date
dgtlmoon	9d22c86e9d	Ensure JSON is always nicely formatted Some checks failed Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Build distribution 📦 (push) Has been cancelled Details ChangeDetection.io App Test / lint-code (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Test the built 📦 package works basically. (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Publish Python 🐍 distribution 📦 to PyPI (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-10 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-11 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-12 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-13 (push) Has been cancelled Details	2025-10-10 15:35:01 +02:00
dgtlmoon	7fe504f3e9	unused var	2025-10-10 13:48:35 +02:00
dgtlmoon	ca140c559e	Re #3482 - Ensure JSON is always correctly reformatted with padding	2025-10-10 13:45:55 +02:00
dgtlmoon	80be1a30f2	No need to reformat/reprocess content in the case that no filters were found (#3484 , #3483 ) Some checks failed Build and push containers / metadata (push) Has been cancelled Details Build and push containers / build-push-containers (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Build distribution 📦 (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Test the built 📦 package works basically. (push) Has been cancelled Details Publish Python 🐍distribution 📦 to PyPI and TestPyPI / Publish Python 🐍 distribution 📦 to PyPI (push) Has been cancelled Details ChangeDetection.io App Test / lint-code (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-10 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-11 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-12 (push) Has been cancelled Details ChangeDetection.io App Test / test-application-3-13 (push) Has been cancelled Details	2025-10-10 13:44:49 +02:00