From e2304b2ce0376d471f774eb3693d188d45acda9a Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sun, 25 Jul 2021 07:02:19 +0200 Subject: [PATCH] Re #154 Ldjson extract parse (#158) * Use parsable JSON hiding in +``` + +`json:$.price` would give `23.50`, or you can extract the whole structure + ### Proxy A proxy for ChangeDetection.io can be configured by setting environment the diff --git a/backend/fetch_site_status.py b/backend/fetch_site_status.py index 69e977fad..870a6515a 100644 --- a/backend/fetch_site_status.py +++ b/backend/fetch_site_status.py @@ -92,27 +92,8 @@ class perform_site_check(): css_filter_rule = self.datastore.data['watching'][uuid]['css_filter'] if css_filter_rule and len(css_filter_rule.strip()): if 'json:' in css_filter_rule: - # POC hack, @todo rename vars, see how it fits in with the javascript version - import json - from jsonpath_ng import jsonpath, parse - - json_data = json.loads(html) - jsonpath_expression = parse(css_filter_rule.replace('json:', '')) - match = jsonpath_expression.find(json_data) - s = [] - - # More than one result, we will return it as a JSON list. - if len(match) > 1: - for i in match: - s.append(i.value) - - # Single value, use just the value, as it could be later used in a token in notifications. - if len(match) == 1: - s = match[0].value - - stripped_text_from_html = json.dumps(s, indent=4) + stripped_text_from_html = html_tools.extract_json_as_string(html, css_filter_rule) is_html = False - else: # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text html = html_tools.css_filter(css_filter=css_filter_rule, html_content=r.content) diff --git a/backend/html_tools.py b/backend/html_tools.py index d71c4fbdb..8a0ff3c55 100644 --- a/backend/html_tools.py +++ b/backend/html_tools.py @@ -1,6 +1,12 @@ +import json from bs4 import BeautifulSoup +from jsonpath_ng import parse +class JSONNotFound(ValueError): + def __init__(self, msg): + ValueError.__init__(self, msg) + # Given a CSS Rule, and a blob of HTML, return the blob of HTML that matches def css_filter(css_filter, html_content): soup = BeautifulSoup(html_content, "html.parser") @@ -24,3 +30,54 @@ def extract_element(find='title', html_content=''): return element_text +# +def _parse_json(json_data, jsonpath_filter): + s=[] + jsonpath_expression = parse(jsonpath_filter.replace('json:', '')) + match = jsonpath_expression.find(json_data) + + # More than one result, we will return it as a JSON list. + if len(match) > 1: + for i in match: + s.append(i.value) + + # Single value, use just the value, as it could be later used in a token in notifications. + if len(match) == 1: + s = match[0].value + + if not s: + raise JSONNotFound("No Matching JSON could be found for the rule {}".format(jsonpath_filter.replace('json:', ''))) + + stripped_text_from_html = json.dumps(s, indent=4) + + return stripped_text_from_html + +def extract_json_as_string(content, jsonpath_filter): + + stripped_text_from_html = False + + # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded blob.. just return the first that matches jsonpath_filter + s = [] + soup = BeautifulSoup(content, 'html.parser') + bs_result = soup.findAll('script') + + if not bs_result: + raise JSONNotFound("No parsable JSON found in this document") + + for result in bs_result: + try: + json_data = json.loads(result.string) + except json.JSONDecodeError: + # Just skip it + continue + else: + stripped_text_from_html = _parse_json(json_data, jsonpath_filter) + if stripped_text_from_html: + break + + return stripped_text_from_html diff --git a/backend/tests/test_jsonpath_selector.py b/backend/tests/test_jsonpath_selector.py index e955dc287..4db54b594 100644 --- a/backend/tests/test_jsonpath_selector.py +++ b/backend/tests/test_jsonpath_selector.py @@ -4,6 +4,42 @@ import time from flask import url_for from . util import live_server_setup +def test_unittest_inline_html_extract(): + # So lets pretend that the JSON we want is inside some HTML + content=""" + + + food and stuff and more + + + + +and it can also be repeated + +

ok

+ + + + """ + from .. import html_tools + + # See that we can find the second