From 446622159c8f56c9a2b20c98108b0d74e2da5445 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Fri, 30 Aug 2024 16:06:04 +0200 Subject: [PATCH] WIP - adding more scrape data and some dev tweaks --- .../res/xpath_element_scraper.js | 67 +++++++++++++------ changedetectionio/flask_app.py | 6 +- changedetectionio/model/Watch.py | 2 +- .../processors/restock_diff/processor.py | 9 +++ 4 files changed, 60 insertions(+), 24 deletions(-) diff --git a/changedetectionio/content_fetchers/res/xpath_element_scraper.js b/changedetectionio/content_fetchers/res/xpath_element_scraper.js index 17ac125ba..c7a92378b 100644 --- a/changedetectionio/content_fetchers/res/xpath_element_scraper.js +++ b/changedetectionio/content_fetchers/res/xpath_element_scraper.js @@ -15,6 +15,7 @@ try { console.log(e); } +const percentageNumerical = str => Math.round((str.match(/\d/g) || []).length / str.length * 100); // Include the getXpath script directly, easier than fetching function getxpath(e) { @@ -146,8 +147,10 @@ const visibleElementsArray = []; // Call collectVisibleElements with the starting parent element collectVisibleElements(document.body, visibleElementsArray); +// Append any custom selectors to the visibleElementsArray -visibleElementsArray.forEach(function (element) { + +function get_element_metadata(element) { bbox = element.getBoundingClientRect(); @@ -190,14 +193,21 @@ visibleElementsArray.forEach(function (element) { let label = "none" // A placeholder, the actual labels for training are done by hand for now - let text = element.textContent.trim().slice(0, 30).trim(); - while (/\n{2,}|\t{2,}/.test(text)) { - text = text.replace(/\n{2,}/g, '\n').replace(/\t{2,}/g, '\t') - } + // Check if the element was found and get its text , not including any child element + let text = Array.from(element.childNodes) + .filter(node => node.nodeType === Node.TEXT_NODE) + .map(node => node.textContent) + .join(''); + + // Remove any gaps in sequences of newlines and tabs inside the string + text = text.trim().replace(/[\s\t\n\r]{2,}/g, ' ').trim(); // Try to identify any possible currency amounts "Sale: 4000" or "Sale now 3000 Kc", can help with the training. // @todo could be instead of USD/AUD etc [A-Z]{2,3} ? + //const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6)) ) && /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|RM|,–)/.test(text) ; const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6)) ) && /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|RM|,–)/.test(text) ; + const hasDigit = /[0-9]/.test(text) ; + // Sizing of the actual text inside the element can be very different from the elements size const { textWidth, textHeight } = getTextWidthAndHeightinPx(element); @@ -211,8 +221,7 @@ visibleElementsArray.forEach(function (element) { // Assign default values if text is empty [red, green, blue] = [0, 0, 0]; } - - size_pos.push({ + return { xpath: xpath_result, width: Math.round(bbox['width']), height: Math.round(bbox['height']), @@ -223,18 +232,27 @@ visibleElementsArray.forEach(function (element) { // tagtype used by Browser Steps tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '', isClickable: window.getComputedStyle(element).cursor === "pointer", - // Used by the keras trainer + // Used by the keras/pytorch trainer fontSize: window.getComputedStyle(element).getPropertyValue('font-size'), fontWeight: window.getComputedStyle(element).getPropertyValue('font-weight'), + pcNumerical: text.length && percentageNumerical(text), + hasDigit: hasDigit, hasDigitCurrency: hasDigitCurrency, textWidth: textWidth, textHeight: textHeight, + textLength: text.length, t_r: red, t_g: green, t_b: blue, label: label, - }); + }; +} +visibleElementsArray.forEach(function (element) { + let metadata = get_element_metadata(element); + if(metadata) { + size_pos.push(metadata); + } }); @@ -243,7 +261,19 @@ visibleElementsArray.forEach(function (element) { if (include_filters.length) { let results; // Foreach filter, go and find it on the page and add it to the results so we can visualise it again + outerLoop: for (const f of include_filters) { + // Quick check so we dont end up with duplicates in the training data + for (let index = 0; index < size_pos.length; index++) { + let item = size_pos[index]; + if (item.xpath === f) { + item.highlight_as_custom_filter = true; + item.found_as_duplicate = true; + item.label = "price"; + continue outerLoop; + } + } + bbox = false; q = false; @@ -264,7 +294,6 @@ if (include_filters.length) { } } else { console.log("[css] Scanning for included filter " + f) - console.log("[css] Scanning for included filter " + f); results = document.querySelectorAll(f); } } catch (e) { @@ -301,17 +330,15 @@ if (include_filters.length) { console.log("xpath_element_scraper: error looking up q.ownerElement") } } - - if (bbox && bbox['width'] > 0 && bbox['height'] > 0) { - size_pos.push({ - xpath: f, - width: parseInt(bbox['width']), - height: parseInt(bbox['height']), - left: parseInt(bbox['left']), - top: parseInt(bbox['top']) + scroll_y, - highlight_as_custom_filter: true - }); + element_info = get_element_metadata(node); + if(element_info) { + // Be sure we use exactly what was written + element_info['xpath'] = f; + element_info['highlight_as_custom_filter'] = true; + element_info['label'] = "price"; + size_pos.push(element_info); } + }); } } diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index e6e7c6944..97ceee53d 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -792,9 +792,9 @@ def changedetection_app(config=None, datastore_o=None): # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds # But in the case something is added we should save straight away datastore.needs_write_urgent = True - - # Queue the watch for immediate recheck, with a higher priority - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False})) + if not datastore.data['watching'][uuid].get('paused'): + # Queue the watch for immediate recheck, with a higher priority + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False})) # Diff page [edit] link should go back to diff page if request.args.get("next") and request.args.get("next") == 'diff': diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index d3167bf90..bde9a158b 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -518,7 +518,7 @@ class model(watch_base): self.ensure_data_dir_exists() with open(target_path, 'w') as f: - f.write(json.dumps(data)) + f.write(json.dumps(data, indent=2)) f.close() # Save as PNG, PNG is larger but better for doing visual diff in the future diff --git a/changedetectionio/processors/restock_diff/processor.py b/changedetectionio/processors/restock_diff/processor.py index 52f4d11b0..6b7ef5dbf 100644 --- a/changedetectionio/processors/restock_diff/processor.py +++ b/changedetectionio/processors/restock_diff/processor.py @@ -37,6 +37,7 @@ def get_itemprop_availability(html_content) -> Restock: Kind of funny/cool way to find price/availability in one many different possibilities. Use 'extruct' to find any possible RDFa/microdata/json-ld data, make a JSON string from the output then search it. """ + from jsonpath_ng import parse now = time.time() @@ -54,6 +55,7 @@ def get_itemprop_availability(html_content) -> Restock: # First phase, dead simple scanning of anything that looks useful value = Restock() + return value if data: logger.debug(f"Using jsonpath to find price/availability/etc") price_parse = parse('$..(price|Price)') @@ -136,10 +138,15 @@ class perform_site_check(difference_detection_processor): logger.debug(f"ML Price scraper: response - {response_json}'") if isinstance(response_json, dict) and 'idx' in response_json.keys(): suggested_xpath_idx = response_json.get('idx') + if response_json.get('score') <0.80 or response_json.get('score') > 1.0: + logger.warning(f"Predict score was outside normal range, aborting ML/AI price check, needs better training data in this case?") + return None # Use the path provided to extra the price text from price_parser import Price scrape_element = self.fetcher.xpath_data.get('size_pos', {})[suggested_xpath_idx] + logger.debug(f"Predicted selector with price information is {scrape_element['xpath']}") + result_s = None if scrape_element['xpath'][0] == '/' or scrape_element['xpath'].startswith('xpath'): result_s = html_tools.xpath_filter(xpath_filter=scrape_element['xpath'], @@ -151,6 +158,7 @@ class perform_site_check(difference_detection_processor): if result_s: text = html_to_text(result_s) + logger.debug(f"Guessed the text '{text}' as the price information") if text: price_info = Price.fromstring(text) else: @@ -158,6 +166,7 @@ class perform_site_check(difference_detection_processor): else: print(f"ML Price scraper: Request failed with status code: {response.status_code}") +#@TODO THROW HELPFUL MESSAGE WITH LINK TO TUTORIAL IF IT CANT CONNECT! return price_info def run_changedetection(self, watch, skip_when_checksum_same=True):