WIP - adding more scrape data and some dev tweaks

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -15,6 +15,7 @@ try {
     console.log(e);
 }
 
+const percentageNumerical = str => Math.round((str.match(/\d/g) || []).length / str.length * 100);
 
 // Include the getXpath script directly, easier than fetching
 function getxpath(e) {
@@ -146,8 +147,10 @@ const visibleElementsArray = [];
 // Call collectVisibleElements with the starting parent element
 collectVisibleElements(document.body, visibleElementsArray);
 
+// Append any custom selectors to the visibleElementsArray
 
-visibleElementsArray.forEach(function (element) {
+
+function get_element_metadata(element) {
 
     bbox = element.getBoundingClientRect();
 
@@ -190,14 +193,21 @@ visibleElementsArray.forEach(function (element) {
 
     let label = "none" // A placeholder, the actual labels for training are done by hand for now
 
-    let text = element.textContent.trim().slice(0, 30).trim();
-    while (/\n{2,}|\t{2,}/.test(text)) {
-        text = text.replace(/\n{2,}/g, '\n').replace(/\t{2,}/g, '\t')
-    }
+    // Check if the element was found and get its text , not including any child element
+    let text = Array.from(element.childNodes)
+        .filter(node => node.nodeType === Node.TEXT_NODE)
+        .map(node => node.textContent)
+        .join('');
+
+    // Remove any gaps in sequences of newlines and tabs inside the string
+    text = text.trim().replace(/[\s\t\n\r]{2,}/g, ' ').trim();
 
     // Try to identify any possible currency amounts "Sale: 4000" or "Sale now 3000 Kc", can help with the training.
     // @todo could be instead of USD/AUD etc [A-Z]{2,3} ?
+    //const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6)) ) &&  /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|RM|,–)/.test(text) ;
     const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6)) ) &&  /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|RM|,–)/.test(text) ;
+    const hasDigit = /[0-9]/.test(text) ;
+
     // Sizing of the actual text inside the element can be very different from the elements size
     const { textWidth, textHeight } = getTextWidthAndHeightinPx(element);
 
@@ -211,8 +221,7 @@ visibleElementsArray.forEach(function (element) {
         // Assign default values if text is empty
         [red, green, blue] = [0, 0, 0];
     }
-
-    size_pos.push({
+    return {
         xpath: xpath_result,
         width: Math.round(bbox['width']),
         height: Math.round(bbox['height']),
@@ -223,18 +232,27 @@ visibleElementsArray.forEach(function (element) {
         // tagtype used by Browser Steps
         tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '',
         isClickable: window.getComputedStyle(element).cursor === "pointer",
-        // Used by the keras trainer
+        // Used by the keras/pytorch trainer
         fontSize: window.getComputedStyle(element).getPropertyValue('font-size'),
         fontWeight: window.getComputedStyle(element).getPropertyValue('font-weight'),
+        pcNumerical: text.length && percentageNumerical(text),
+        hasDigit: hasDigit,
         hasDigitCurrency: hasDigitCurrency,
         textWidth: textWidth,
         textHeight: textHeight,
+        textLength: text.length,
         t_r: red,
         t_g: green,
         t_b: blue,
         label: label,
-    });
+    };
+}
 
+visibleElementsArray.forEach(function (element) {
+    let metadata = get_element_metadata(element);
+    if(metadata) {
+        size_pos.push(metadata);
+    }
 });
 
 
@@ -243,7 +261,19 @@ visibleElementsArray.forEach(function (element) {
 if (include_filters.length) {
     let results;
     // Foreach filter, go and find it on the page and add it to the results so we can visualise it again
+    outerLoop:
     for (const f of include_filters) {
+        // Quick check so we dont end up with duplicates in the training data
+        for (let index = 0; index < size_pos.length; index++) {
+            let item = size_pos[index];
+            if (item.xpath === f) {
+                item.highlight_as_custom_filter = true;
+                item.found_as_duplicate = true;
+                item.label = "price";
+                continue outerLoop;
+            }
+        }
+
         bbox = false;
         q = false;
 
@@ -264,7 +294,6 @@ if (include_filters.length) {
                 }
             } else {
                 console.log("[css] Scanning for included filter " + f)
-                console.log("[css] Scanning for included filter " + f);
                 results = document.querySelectorAll(f);
             }
         } catch (e) {
@@ -301,17 +330,15 @@ if (include_filters.length) {
                         console.log("xpath_element_scraper: error looking up q.ownerElement")
                     }
                 }
-
-                if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
-                    size_pos.push({
-                        xpath: f,
-                        width: parseInt(bbox['width']),
-                        height: parseInt(bbox['height']),
-                        left: parseInt(bbox['left']),
-                        top: parseInt(bbox['top']) + scroll_y,
-                        highlight_as_custom_filter: true
-                    });
+                element_info = get_element_metadata(node);
+                if(element_info) {
+                    // Be sure we use exactly what was written
+                    element_info['xpath'] = f;
+                    element_info['highlight_as_custom_filter'] = true;
+                    element_info['label'] = "price";
+                    size_pos.push(element_info);
                 }
+
             });
         }
     }

changedetectionio/flask_app.py

@@ -792,9 +792,9 @@ def changedetection_app(config=None, datastore_o=None):
             # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
             # But in the case something is added we should save straight away
             datastore.needs_write_urgent = True
-
-            # Queue the watch for immediate recheck, with a higher priority
-            update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
+            if not datastore.data['watching'][uuid].get('paused'):
+                # Queue the watch for immediate recheck, with a higher priority
+                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
 
             # Diff page [edit] link should go back to diff page
             if request.args.get("next") and request.args.get("next") == 'diff':

changedetectionio/model/Watch.py

@@ -518,7 +518,7 @@ class model(watch_base):
         self.ensure_data_dir_exists()
 
         with open(target_path, 'w') as f:
-            f.write(json.dumps(data))
+            f.write(json.dumps(data, indent=2))
             f.close()
 
     # Save as PNG, PNG is larger but better for doing visual diff in the future

changedetectionio/processors/restock_diff/processor.py

@@ -37,6 +37,7 @@ def get_itemprop_availability(html_content) -> Restock:
     Kind of funny/cool way to find price/availability in one many different possibilities.
     Use 'extruct' to find any possible RDFa/microdata/json-ld data, make a JSON string from the output then search it.
     """
+
     from jsonpath_ng import parse
 
     now = time.time()
@@ -54,6 +55,7 @@ def get_itemprop_availability(html_content) -> Restock:
 
     # First phase, dead simple scanning of anything that looks useful
     value = Restock()
+    return value
     if data:
         logger.debug(f"Using jsonpath to find price/availability/etc")
         price_parse = parse('$..(price|Price)')
@@ -136,10 +138,15 @@ class perform_site_check(difference_detection_processor):
             logger.debug(f"ML Price scraper: response - {response_json}'")
             if isinstance(response_json, dict) and 'idx' in response_json.keys():
                 suggested_xpath_idx = response_json.get('idx')
+                if response_json.get('score') <0.80 or response_json.get('score') > 1.0:
+                    logger.warning(f"Predict score was outside normal range, aborting ML/AI price check, needs better training data in this case?")
+                    return None
 
                 # Use the path provided to extra the price text
                 from price_parser import Price
                 scrape_element = self.fetcher.xpath_data.get('size_pos', {})[suggested_xpath_idx]
+                logger.debug(f"Predicted selector with price information is {scrape_element['xpath']}")
+
                 result_s = None
                 if scrape_element['xpath'][0] == '/' or scrape_element['xpath'].startswith('xpath'):
                     result_s = html_tools.xpath_filter(xpath_filter=scrape_element['xpath'],
@@ -151,6 +158,7 @@ class perform_site_check(difference_detection_processor):
 
                 if result_s:
                     text = html_to_text(result_s)
+                    logger.debug(f"Guessed the text '{text}' as the price information")
                     if text:
                         price_info = Price.fromstring(text)
             else:
@@ -158,6 +166,7 @@ class perform_site_check(difference_detection_processor):
         else:
             print(f"ML Price scraper: Request failed with status code: {response.status_code}")
 
+#@TODO THROW HELPFUL MESSAGE WITH LINK TO TUTORIAL IF IT CANT CONNECT!
         return price_info
 
     def run_changedetection(self, watch, skip_when_checksum_same=True):