#!/usr/bin/env python3
# coding=utf-8

import time
from flask import url_for
from markupsafe import escape
from . util import live_server_setup, wait_for_all_checks, delete_all_watches
import pytest
import os
jq_support = True

try:
    import jq
except ModuleNotFoundError:
    jq_support = False



def test_jsonp_treated_as_plaintext():
    from ..processors.magic import guess_stream_type

    # JSONP content (server wrongly claims application/json) should be detected as plaintext
    # Callback names are arbitrary identifiers, not always 'cb'
    jsonp_content = 'jQuery123456({ "version": "8.0.41", "url": "https://example.com/app.apk" })'
    result = guess_stream_type(http_content_header="application/json", content=jsonp_content)
    assert result.is_json is False
    assert result.is_plaintext is True

    # Variation with dotted callback name e.g. jQuery.cb(...)
    jsonp_dotted = 'some.callback({ "version": "1.0" })'
    result = guess_stream_type(http_content_header="application/json", content=jsonp_dotted)
    assert result.is_json is False
    assert result.is_plaintext is True

    # Real JSON should still be detected as JSON
    json_content = '{ "version": "8.0.41", "url": "https://example.com/app.apk" }'
    result = guess_stream_type(http_content_header="application/json", content=json_content)
    assert result.is_json is True
    assert result.is_plaintext is False


def test_jsonp_json_filter_extraction():
    from .. import html_tools

    # Tough case: dotted namespace callback, trailing semicolon, deeply nested content with arrays
    jsonp_content = 'weixin.update.callback({"platforms": {"android": {"variants": [{"arch": "arm64", "versionName": "8.0.68", "url": "https://example.com/app-arm64.apk"}, {"arch": "arm32", "versionName": "8.0.41", "url": "https://example.com/app-arm32.apk"}]}}});'

    # Deep nested jsonpath filter into array element
    text = html_tools.extract_json_as_string(jsonp_content, "json:$.platforms.android.variants[0].versionName")
    assert text == '"8.0.68"'

    # Filter that selects the second array element
    text = html_tools.extract_json_as_string(jsonp_content, "json:$.platforms.android.variants[1].arch")
    assert text == '"arm32"'

    if jq_support:
        text = html_tools.extract_json_as_string(jsonp_content, "jq:.platforms.android.variants[0].versionName")
        assert text == '"8.0.68"'

        text = html_tools.extract_json_as_string(jsonp_content, "jqraw:.platforms.android.variants[1].url")
        assert text == "https://example.com/app-arm32.apk"


def test_unittest_inline_html_extract():
    # So lets pretend that the JSON we want is inside some HTML
    content="""
    <html>
    
    food and stuff and more
    <script>
    alert('nothing really good here');
    </script>
    
    <script type="application/ld+json">
  xx {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","description":"During the first year of life, nutrition is critical for your baby. NAN OPTIPRO 1 is tailored to ensure your formula fed infant receives balanced, high quality nutrition.<br />Starter infant formula. The age optimised protein source (whey dominant) is from cow’s milk.<br />Backed by more than 150 years of Nestlé expertise.<br />For hygiene and convenience, it is available in an innovative packaging format with a separate storage area for the scoop, and a semi-transparent window which allows you to see how much powder is left in the can without having to open it.","image":"https://cdn0.woolworths.media/content/wowproductimages/large/155536.jpg","brand":{"@context":"http://schema.org","@type":"Organization","name":"Nan"},"gtin13":"7613287517388","offers":{"@context":"http://schema.org","@type":"Offer","potentialAction":{"@context":"http://schema.org","@type":"BuyAction"},"availability":"http://schema.org/InStock","itemCondition":"http://schema.org/NewCondition","price":23.5,"priceCurrency":"AUD"},"review":[],"sku":"155536"}
</script>
<body>
and it can also be repeated
<script type="application/ld+json">
  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","description":"During the first year of life, nutrition is critical for your baby. NAN OPTIPRO 1 is tailored to ensure your formula fed infant receives balanced, high quality nutrition.<br />Starter infant formula. The age optimised protein source (whey dominant) is from cow’s milk.<br />Backed by more than 150 years of Nestlé expertise.<br />For hygiene and convenience, it is available in an innovative packaging format with a separate storage area for the scoop, and a semi-transparent window which allows you to see how much powder is left in the can without having to open it.","image":"https://cdn0.woolworths.media/content/wowproductimages/large/155536.jpg","brand":{"@context":"http://schema.org","@type":"Organization","name":"Nan"},"gtin13":"7613287517388","offers":{"@context":"http://schema.org","@type":"Offer","potentialAction":{"@context":"http://schema.org","@type":"BuyAction"},"availability":"http://schema.org/InStock","itemCondition":"http://schema.org/NewCondition","price":23.5,"priceCurrency":"AUD"},"review":[],"sku":"155536"}
</script>
<h4>ok</h4>
</body>
</html>

    """
    from .. import html_tools

    # See that we can find the second <script> one, which is not broken, and matches our filter
    text = html_tools.extract_json_as_string(content, "json:$.offers.priceCurrency")
    assert text == '"AUD"'

    text = html_tools.extract_json_as_string('{"id":5}', "json:$.id")
    assert text == "5"

    # also check for jq
    if jq_support:
        text = html_tools.extract_json_as_string(content, "jq:.offers.priceCurrency")
        assert text == '"AUD"'

        text = html_tools.extract_json_as_string('{"id":5}', "jq:.id")
        assert text == "5"

        text = html_tools.extract_json_as_string(content, "jqraw:.offers.priceCurrency")
        assert text == "AUD"

        text = html_tools.extract_json_as_string('{"id":5}', "jqraw:.id")
        assert text == "5"


    # When nothing at all is found, it should throw JSONNOTFound
    # Which is caught and shown to the user in the watch-overview table
    with pytest.raises(html_tools.JSONNotFound) as e_info:
        html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "json:$.id")

    if jq_support:
        with pytest.raises(html_tools.JSONNotFound) as e_info:
            html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jq:.id")

        with pytest.raises(html_tools.JSONNotFound) as e_info:
            html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jqraw:.id")


def test_blank_content_does_not_raise_indexerror():
    """Empty / whitespace-only content must take the normal JSONNotFound path, not crash.

    The content-sniffing peeked at content_start[0] to see if the body looked like JSON.
    A blank body left content_start empty, so that peek raised IndexError instead of
    letting the caller report 'no JSON found' the way any other non-JSON body does.
    See https://github.com/dgtlmoon/changedetection.io/issues/4531
    """
    from .. import html_tools

    # Blank bodies with a json: filter set - the same as any other non-JSON content
    for blank in ('', '   ', '\n', '\r\n\t '):
        with pytest.raises(html_tools.JSONNotFound):
            html_tools.extract_json_as_string(blank, "json:$.id")

    # Same on the html blob path, where a whitespace-only <script> or <body> is skipped
    # and the real ld+json that follows it is still found
    html = '<html><script>\n</script><script type="application/ld+json">{"a":1}</script></html>'
    assert html_tools.extract_json_blob_from_html(html, None, "json:$.a") == "1"

    # A document that is nothing but blank markup has no JSON in it at all
    with pytest.raises(html_tools.JSONNotFound):
        html_tools.extract_json_blob_from_html('<html><body>   </body></html>', None, "json:$.a")


def test_lone_surrogate_escapes_do_not_break_filters():
    """A \\uD800-style escape in the watched JSON must not take the whole document down.

    jq re-serializes the document for its own C parser, which rejects a lone high surrogate,
    so every jq:/jqraw: filter on the page used to error even when pointing at an unrelated
    field. On the json: path the surrogate instead reached the caller intact and blew up later
    in checksums and history writes. Valid pairs must keep working untouched.
    See: https://github.com/dgtlmoon/changedetection.io/issues/4273
    """
    from .. import html_tools

    BS = chr(92)  # build the escapes at runtime so no quoting layer eats them
    lone_high = '{"title": "Example ' + BS + 'uD800", "other": "untouched"}'
    lone_low = '{"title": "Example ' + BS + 'uDC00", "other": "untouched"}'
    valid_pair = '{"title": "smile ' + BS + 'uD83D' + BS + 'uDE00", "other": "untouched"}'

    filters = ["json:$.%s", "jq:.%s", "jqraw:.%s"] if jq_support else ["json:$.%s"]

    for content in (lone_high, lone_low):
        for f in filters:
            # An unrelated field must still be reachable
            text = html_tools.extract_json_as_string(content, f % "other")
            assert "untouched" in text

            # And the offending field itself resolves, with the surrogate replaced
            text = html_tools.extract_json_as_string(content, f % "title")
            assert "Example" in text
            assert not any(0xD800 <= ord(c) <= 0xDFFF for c in text)
            # Whatever comes back has to survive the downstream checksum/history write
            text.encode('utf-8')

    # A well-formed surrogate pair is a normal emoji and must be preserved as-is
    for f in filters:
        text = html_tools.extract_json_as_string(valid_pair, f % "title")
        assert '\U0001F600' in text
        text.encode('utf-8')

    # The surrogate can just as easily land in an object *key*, which has to be sanitized too -
    # otherwise the whole-document re-serialization still trips over it
    lone_key = '{"ti' + BS + 'uD800tle": "value", "other": "untouched"}'
    for f in ("json:$.other", "jq:.other", "jqraw:.other") if jq_support else ("json:$.other",):
        text = html_tools.extract_json_as_string(lone_key, f)
        assert "untouched" in text
        text.encode('utf-8')

    # Dumping the whole document exercises the key path directly
    for f in ("json:$", "jq:.") if jq_support else ("json:$",):
        text = html_tools.extract_json_as_string(lone_key, f)
        assert "value" in text
        assert not any(0xD800 <= ord(c) <= 0xDFFF for c in text)
        text.encode('utf-8')

    # Nested arrays and non-string scalars have to be walked too. The scalars come first
    # deliberately: detection short-circuits on the first surrogate it finds, so anything after
    # the offending value would never be visited.
    nested = ('{"count": 3, "flag": null, "ok": true, '
              '"items": ["clean", "bad' + BS + 'uD800", [1, "deep' + BS + 'uDC00"]]}')
    for f in ("json:$", "jq:.") if jq_support else ("json:$",):
        text = html_tools.extract_json_as_string(nested, f)
        assert "clean" in text and "deep" in text
        assert not any(0xD800 <= ord(c) <= 0xDFFF for c in text)
        text.encode('utf-8')

    # ...and the scalars must survive the round-trip unmangled
    assert html_tools.extract_json_as_string(nested, "json:$.count").strip() == "3"


def test_unittest_inline_extract_body():
    content = """
    <html>
        <head></head>
        <body>
            <pre style="word-wrap: break-word; white-space: pre-wrap;">
                {"testKey": 42}
            </pre>
        </body>
    </html>
    """
    from .. import html_tools

    # See that we can find the second <script> one, which is not broken, and matches our filter
    text = html_tools.extract_json_as_string(content, "json:$.testKey")
    assert text == '42'

def set_original_ext_response(datastore_path):
    data = """
        [
        {
            "isPriceLowered": false,
            "status": "ForSale",
            "statusOrig": "for sale"
        },
        {
            "_id": "5e7b3e1fb3262d306323ff1e",
            "listingsType": "consumer",
            "status": "ForSale",
            "statusOrig": "for sale"
        }
    ]
        """

    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
        f.write(data)
    return None

def set_modified_ext_response(datastore_path):
    # This should get reformatted
    data = """ [ { "isPriceLowered": false,  "status": "Sold",  "statusOrig": "sold" }, {
        "_id": "5e7b3e1fb3262d306323ff1e",
        "listingsType": "consumer",
        "isPriceLowered": false,
        "status": "Sold"
    }
]
    """

    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
        f.write(data)
    return None

def set_original_response(datastore_path):
    test_return_data = """
    {
      "employees": [
        {
          "id": 1,
          "name": "Pankaj",
          "salary": "10000"
        },
        {
          "name": "David",
          "salary": "5000",
          "id": 2
        }
      ],
      "boss": {
        "name": "Fat guy"
      },
      "available": true
    }
    """
    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
        f.write(test_return_data)
    return None


def set_json_response_with_html(datastore_path):
    test_return_data = """
    {
      "test": [
        {
          "html": "<b>"
        }
      ]
    }
    """
    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
        f.write(test_return_data)
    return None

def set_modified_response(datastore_path):
    test_return_data = """
    {
      "employees": [
        {
          "id": 1,
          "name": "Pankaj",
          "salary": "10000"
        },
        {
          "name": "David",
          "salary": "5000",
          "id": 2
        }
      ],
      "boss": {
        "name": "Örnsköldsvik"
      },
      "available": false
    }
        """

    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
        f.write(test_return_data)

    return None

def test_check_json_without_filter(client, live_server, measure_memory_usage, datastore_path):
    # Request a JSON document from a application/json source containing HTML
    # and be sure it doesn't get chewed up by instriptis
    set_json_response_with_html(datastore_path=datastore_path)

    # Add our URL to the import page
    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
    client.post(url_for("ui.form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    wait_for_all_checks(client)

    res = client.get(
        url_for("ui.ui_preview.preview_page", uuid="first"),
        follow_redirects=True
    )

    # Should still see '"html": "<b>"'
    assert b'&#34;html&#34;: &#34;&lt;b&gt;&#34;' in res.data
    assert res.data.count(b'{') >= 2

    delete_all_watches(client)

def check_json_filter(json_filter, client, live_server, datastore_path):
    set_original_response(datastore_path=datastore_path)


    delete_all_watches(client)
    # Add our URL to the import page
    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url, extras={"include_filters": json_filter.splitlines()})
    client.post(url_for("ui.form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    wait_for_all_checks(client)

    # Check it saved
    res = client.get(
        url_for("ui.ui_edit.edit_page", uuid=uuid),
    )
    assert bytes(escape(json_filter).encode('utf-8')) in res.data

    # Give the thread time to pick it up
    wait_for_all_checks(client)
    #  Make a change
    set_modified_response(datastore_path=datastore_path)

    # Trigger a check
    client.post(url_for("ui.form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    wait_for_all_checks(client)

    # It should have 'has-unread-changes' still
    res = client.get(url_for("watchlist.index"))
    assert b'has-unread-changes' in res.data

    # Should not see this, because its not in the JSONPath we entered
    res = client.get(url_for("ui.ui_diff.diff_history_page", uuid=uuid))

    # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
    # And #462 - check we see the proper utf-8 string there
    assert "Örnsköldsvik".encode('utf-8') in res.data

    delete_all_watches(client)

def test_check_jsonpath_filter(client, live_server, measure_memory_usage, datastore_path):
    check_json_filter('json:boss.name', client, live_server, datastore_path=datastore_path)

def test_check_jq_filter(client, live_server, measure_memory_usage, datastore_path):
    if jq_support:
        check_json_filter('jq:.boss.name', client, live_server, datastore_path=datastore_path)

def test_check_jqraw_filter(client, live_server, measure_memory_usage, datastore_path):
    if jq_support:
        check_json_filter('jqraw:.boss.name', client, live_server, datastore_path=datastore_path)

def check_json_filter_bool_val(json_filter, client, live_server, datastore_path):
    set_original_response(datastore_path=datastore_path)

    test_url = url_for('test_endpoint', content_type="application/json", _external=True)

    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url, extras={"include_filters": [json_filter]})
    client.post(url_for("ui.form_watch_checknow"), follow_redirects=True)
    wait_for_all_checks(client)

    #  Make a change
    set_modified_response(datastore_path=datastore_path)

    # Trigger a check
    client.post(url_for("ui.form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    wait_for_all_checks(client)

    res = client.get(url_for("ui.ui_diff.diff_history_page", uuid="first"))
    # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
    assert b'false' in res.data

    delete_all_watches(client)

def test_check_jsonpath_filter_bool_val(client, live_server, measure_memory_usage, datastore_path):
    check_json_filter_bool_val("json:$['available']", client, live_server, datastore_path=datastore_path)
    delete_all_watches(client)

def test_check_jq_filter_bool_val(client, live_server, measure_memory_usage, datastore_path):
    if jq_support:
        check_json_filter_bool_val("jq:.available", client, live_server, datastore_path=datastore_path)
    delete_all_watches(client)

def test_check_jqraw_filter_bool_val(client, live_server, measure_memory_usage, datastore_path):
    if jq_support:
        check_json_filter_bool_val("jq:.available", client, live_server, datastore_path=datastore_path)
    delete_all_watches(client)

# Re #265 - Extended JSON selector test
# Stuff to consider here
# - Selector should be allowed to return empty when it doesnt match (people might wait for some condition)
# - The 'diff' tab could show the old and new content
# - Form should let us enter a selector that doesnt (yet) match anything
def check_json_ext_filter(json_filter, client, live_server, datastore_path):
    set_original_ext_response(datastore_path=datastore_path)

    # Add our URL to the import page
    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
    client.post(url_for("ui.form_watch_checknow"), follow_redirects=True)
    wait_for_all_checks(client)

    # Goto the edit page, add our ignore text
    # Add our URL to the import page
    res = client.post(
        url_for("ui.ui_edit.edit_page", uuid=uuid),
        data={"include_filters": json_filter,
              "url": test_url,
              "tags": "",
              "headers": "",
              "fetch_backend": "html_requests",
              "time_between_check_use_default": "y"
              },
        follow_redirects=True
    )
    assert b"Updated watch." in res.data

    # Check it saved
    res = client.get(
        url_for("ui.ui_edit.edit_page", uuid=uuid),
    )
    assert bytes(escape(json_filter).encode('utf-8')) in res.data

    # Give the thread time to pick it up
    wait_for_all_checks(client)
    #  Make a change
    set_modified_ext_response(datastore_path=datastore_path)

    # Trigger a check
    client.post(url_for("ui.form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    wait_for_all_checks(client)

    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
    dates = list(watch.history.keys())
    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])

    assert snapshot_contents[0] == '['

    # It should have 'has-unread-changes'
    res = client.get(url_for("watchlist.index"))
    assert b'has-unread-changes' in res.data

    res = client.get(url_for("ui.ui_preview.preview_page", uuid="first"))

    # We should never see 'ForSale' because we are selecting on 'Sold' in the rule,
    # But we should know it triggered ('has-unread-changes' assert above)
    assert b'ForSale' not in res.data
    assert b'Sold' in res.data


    # And the difference should have both?

    res = client.get(url_for("ui.ui_diff.diff_history_page", uuid="first"))
    assert b'ForSale' in res.data
    assert b'Sold' in res.data

    delete_all_watches(client)

def test_ignore_json_order(client, live_server, measure_memory_usage, datastore_path):
    # A change in order shouldn't trigger a notification

    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
        f.write('{"hello" : 123, "world": 123}')


    # Add our URL to the import page
    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
    client.post(url_for("ui.form_watch_checknow"), follow_redirects=True)

    wait_for_all_checks(client)

    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
        f.write('{"world" : 123, "hello": 123}')

    # Trigger a check
    client.post(url_for("ui.form_watch_checknow"), follow_redirects=True)
    wait_for_all_checks(client)

    res = client.get(url_for("watchlist.index"))
    assert b'has-unread-changes' not in res.data

    # Just to be sure it still works
    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
        f.write('{"world" : 123, "hello": 124}')

    # Trigger a check
    client.post(url_for("ui.form_watch_checknow"), follow_redirects=True)
    wait_for_all_checks(client)

    res = client.get(url_for("watchlist.index"))
    assert b'has-unread-changes' in res.data

    delete_all_watches(client)

def test_correct_header_detect(client, live_server, measure_memory_usage, datastore_path):
    # Like in https://github.com/dgtlmoon/changedetection.io/pull/1593
    # Specify extra html that JSON is sometimes wrapped in - when using SockpuppetBrowser / Puppeteer / Playwrightetc
    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
        f.write('<html><body>{ "world": 123, "hello" : 123}')

    # Add our URL to the import page
    # Check weird casing is cleaned up and detected also
    test_url = url_for('test_endpoint', content_type="aPPlication/JSon", uppercase_headers=True, _external=True)
    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
    client.post(url_for("ui.form_watch_checknow"), follow_redirects=True)
    wait_for_all_checks(client)
    res = client.get(url_for("watchlist.index"))

    # Fixed in #1593
    assert b'No parsable JSON found in this document' not in res.data

    res = client.get(
        url_for("ui.ui_preview.preview_page", uuid="first"),
        follow_redirects=True
    )


    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
    dates = list(watch.history.keys())
    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])

    assert b'&#34;hello&#34;: 123,' in res.data # properly html escaped in the front end
    import json
    data = json.loads(snapshot_contents)
    keys = list(data.keys())
    # Should be correctly formatted and sorted,  ("world" goes to end)
    assert keys == ["hello", "world"]

    delete_all_watches(client)


def test_content_type_json_with_unparsable_body(client, live_server, measure_memory_usage, datastore_path):
    # Re https://github.com/dgtlmoon/changedetection.io/issues/3827
    # Some servers send "Content-Type: application/json" but the body is NOT actually JSON
    # (for example GWT-RPC responses which start with "//OK[...]").
    # Previously this raised "No parsable JSON found in this document" and errored the whole
    # watch, so nothing could be viewed or diffed. Instead the raw content should be kept.
    gwt_rpc_body = '//OK[3,1,["com.example.User/123","Alice Smith","alice@example.com"],0,7]'
    with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
        f.write(gwt_rpc_body)

    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
    client.post(url_for("ui.form_watch_checknow"), follow_redirects=True)
    wait_for_all_checks(client)

    # The watch should not be left in an error state complaining about JSON parsing
    res = client.get(url_for("watchlist.index"))
    assert b'No parsable JSON found in this document' not in res.data

    # A snapshot should have been stored (not skipped due to an exception), and it should
    # contain the raw, unparsable "JSON" content so it can still be viewed/diffed.
    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]
    dates = list(watch.history.keys())
    assert len(dates) >= 1
    snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
    assert 'com.example.User' in snapshot_contents

    delete_all_watches(client)


def test_check_jsonpath_ext_filter(client, live_server, measure_memory_usage, datastore_path):
    check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server, datastore_path=datastore_path)
    delete_all_watches(client)

def test_check_jq_ext_filter(client, live_server, measure_memory_usage, datastore_path):
    if jq_support:
        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server, datastore_path=datastore_path)
    delete_all_watches(client)

def test_check_jqraw_ext_filter(client, live_server, measure_memory_usage, datastore_path):
    if jq_support:
        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server, datastore_path=datastore_path)
    delete_all_watches(client)

def test_jsonpath_BOM_utf8(client, live_server, measure_memory_usage, datastore_path):
    from .. import html_tools

    # JSON string with BOM and correct double-quoted keys
    json_str = '\ufeff{"name": "José", "emoji": "😊", "language": "中文", "greeting": "Привет"}'

    # See that we can find the second <script> one, which is not broken, and matches our filter
    text = html_tools.extract_json_as_string(json_str, "json:$.name")
    assert text == '"José"'
    delete_all_watches(client)