Merge branch 'master' into price-scraper-ML-integration

.github/workflows/containers.yml

@@ -95,7 +95,7 @@ jobs:
           push: true
           tags: |
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
@@ -116,7 +116,7 @@ jobs:
             ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
             ghcr.io/dgtlmoon/changedetection.io:latest
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
 # Looks like this was disabled

.github/workflows/test-container-build.yml

@@ -64,7 +64,7 @@ jobs:
           with:
             context: ./
             file: ./Dockerfile
-            platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
+            platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
             cache-from: type=local,src=/tmp/.buildx-cache
             cache-to: type=local,dest=/tmp/.buildx-cache
 

.github/workflows/test-stack-reusable-workflow.yml

@@ -52,6 +52,10 @@ jobs:
           docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest                    
           docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url  -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest
 
+          # CDIO AI Element scraper for prices
+          # Run CDIO with PRICE_SCRAPER_ML_ENDPOINT=http://cdio-ai-price-element:5005/price-element
+          docker run --network changedet-network -d -e "LOG_LEVEL=TRACE" --name cdio-ai-price-element --hostname cdio-ai-price-element -p 5005:5005 --rm dgtlmoon/changedetection.io-ai:latest
+
       - name: Spin up ancillary SMTP+Echo message test server
         run: |
           # Debug SMTP server/echo message back server
@@ -95,6 +99,11 @@ jobs:
           # Settings headers playwright tests - Call back in from Sockpuppetbrowser, check headers
           docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'find .; cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py; pwd;find .'
 
+# PLAYWRIGHT/NODE-> CDP
+      - name: ML/AI Price element scraper via Playwright+dgtlmoon/changedetection.io-ai
+        run: |
+          docker run --rm -e "FLASK_SERVER_NAME=cdio" -e "PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000" -e "PRICE_SCRAPER_ML_ENDPOINT=ws://cdio-ai-price-element:5005/price-element" --network changedet-network --hostname=cdio test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/ml_price_scraper/test_scrape_price_element.py'
+
       - name: Playwright and SocketPuppetBrowser - Restock detection
         run: |                            
           # restock detection via playwright - added name=changedet here so that playwright and sockpuppetbrowser can connect to it

README.md

@@ -41,6 +41,20 @@ Using the **Browser Steps** configuration, add basic steps before performing cha
 After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
 Requires Playwright to be enabled.
 
+### Awesome restock and price change notifications
+
+Enable the _"Re-stock & Price detection for single product pages"_ option to activate the best way to monitor product pricing, this will extract any meta-data in the HTML page and give you many options to follow the pricing of the product.
+
+Easily organise and monitor prices for products from the dashboard, get alerts and notifications when the price of a product changes or comes back in stock again!
+
+[<img src="docs/restock-overview.png" style="max-width:100%;" alt="Easily keep an eye on product price changes directly from the UI"  title="Easily keep an eye on product price changes directly from the UI" />](https://changedetection.io?src=github)
+
+Set price change notification parameters, upper and lower price, price change percentage and more.
+Always know when a product for sale drops in price.
+
+[<img src="docs/restock-settings.png" style="max-width:100%;" alt="Set upper lower and percentage price change notification values"  title="Set upper lower and percentage price change notification values" />](https://changedetection.io?src=github)
+
+
 
 ### Example use cases
 

changedetection.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # Only exists for direct CLI usage
 

changedetectionio/__init__.py

@@ -1,8 +1,8 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.45.26'
+__version__ = '0.46.04'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/blueprint/browser_steps/__init__.py

@@ -85,7 +85,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
             playwright_browser=browsersteps_start_session['browser'],
             proxy=proxy,
-            start_url=datastore.data['watching'][watch_uuid].get('url')
+            start_url=datastore.data['watching'][watch_uuid].get('url'),
+            headers=datastore.data['watching'][watch_uuid].get('headers')
         )
 
         # For test

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import os
 import time

changedetectionio/blueprint/tags/__init__.py

@@ -1,4 +1,6 @@
-from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
+from flask import Blueprint, request, render_template, flash, url_for, redirect
+
+
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.flask_app import login_optionally_required
 
@@ -96,22 +98,55 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
     @login_optionally_required
     def form_tag_edit(uuid):
-        from changedetectionio import forms
-
+        from changedetectionio.blueprint.tags.form import group_restock_settings_form
         if uuid == 'first':
             uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
 
         default = datastore.data['settings']['application']['tags'].get(uuid)
 
-        form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
-                               data=default,
-                               )
-        form.datastore=datastore # needed?
+        form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
+                                       data=default,
+                                       extra_notification_tokens=datastore.get_unique_notification_tokens_available()
+                                       )
+
+        template_args = {
+            'data': default,
+            'form': form,
+            'watch': default,
+            'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
+        }
+
+        included_content = {}
+        if form.extra_form_content():
+            # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/
+            # And then render the code from the module
+            from jinja2 import Environment, FileSystemLoader
+            import importlib.resources
+            templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates'))
+            env = Environment(loader=FileSystemLoader(templates_dir))
+            template_str = """{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
+        <script>        
+            $(document).ready(function () {
+                toggleOpacity('#overrides_watch', '#restock-fieldset-price-group', true);
+            });
+        </script>            
+                <fieldset>
+                    <div class="pure-control-group">
+                        <fieldset class="pure-group">
+                        {{ render_checkbox_field(form.overrides_watch) }}
+                        <span class="pure-form-message-inline">Used for watches in "Restock & Price detection" mode</span>
+                        </fieldset>
+                </fieldset>
+                """
+            template_str += form.extra_form_content()
+            template = env.from_string(template_str)
+            included_content = template.render(**template_args)
 
         output = render_template("edit-tag.html",
-                                 data=default,
-                                 form=form,
                                  settings_application=datastore.data['settings']['application'],
+                                 extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,
+                                 extra_form_content=included_content,
+                                 **template_args
                                  )
 
         return output
@@ -120,14 +155,15 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
     @login_optionally_required
     def form_tag_edit_submit(uuid):
-        from changedetectionio import forms
+        from changedetectionio.blueprint.tags.form import group_restock_settings_form
         if uuid == 'first':
             uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
 
         default = datastore.data['settings']['application']['tags'].get(uuid)
 
-        form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
+        form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
                                data=default,
+                               extra_notification_tokens=datastore.get_unique_notification_tokens_available()
                                )
         # @todo subclass form so validation works
         #if not form.validate():
@@ -136,6 +172,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 #           return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
 
         datastore.data['settings']['application']['tags'][uuid].update(form.data)
+        datastore.data['settings']['application']['tags'][uuid]['processor'] = 'restock_diff'
         datastore.needs_write_urgent = True
         flash("Updated")
 

changedetectionio/blueprint/tags/form.py

@@ -1,16 +1,15 @@
 from wtforms import (
-    BooleanField,
     Form,
-    IntegerField,
-    RadioField,
-    SelectField,
     StringField,
     SubmitField,
-    TextAreaField,
     validators,
 )
+from wtforms.fields.simple import BooleanField
 
+from changedetectionio.processors.restock_diff.forms import processor_settings_form as restock_settings_form
 
+class group_restock_settings_form(restock_settings_form):
+    overrides_watch = BooleanField('Activate for individual watches in this tag/group?', default=False)
 
 class SingleTag(Form):
 

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -26,6 +26,9 @@
         <ul>
             <li class="tab" id=""><a href="#general">General</a></li>
             <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
+            {% if extra_tab_content %}
+            <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
+            {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
         </ul>
     </div>
@@ -97,6 +100,12 @@ nav
 
             </div>
 
+        {# rendered sub Template #}
+        {% if extra_form_content %}
+            <div class="tab-pane-inner" id="extras_tab">
+            {{ extra_form_content|safe }}
+            </div>
+        {% endif %}
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div  class="pure-control-group inline-radio">
@@ -119,7 +128,7 @@ nav
                         {% endif %}
                         <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
 
-                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
+                        {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
                     </div>
                 </fieldset>
             </div>

changedetectionio/content_fetchers/__init__.py

@@ -4,7 +4,7 @@ from loguru import logger
 from changedetectionio.content_fetchers.exceptions import BrowserStepsStepException
 import os
 
-visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4,header,footer,section,article,aside,details,main,nav,section,summary'
+visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4,header,footer,section,article,aside,details,main,nav,section,summary,bdi,strong'
 
 # available_fetchers() will scan this implementation looking for anything starting with html_
 # this information is used in the form selections

changedetectionio/content_fetchers/base.py

@@ -65,8 +65,8 @@ class Fetcher():
 
     def __init__(self):
         import importlib.resources
-        self.xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text()
-        self.instock_data_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('stock-not-in-stock.js').read_text()
+        self.xpath_element_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('xpath_element_scraper.js').read_text(encoding='utf-8')
+        self.instock_data_js = importlib.resources.files("changedetectionio.content_fetchers.res").joinpath('stock-not-in-stock.js').read_text(encoding='utf-8')
 
     @abstractmethod
     def get_error(self):
@@ -81,7 +81,8 @@ class Fetcher():
             request_method,
             ignore_status_codes=False,
             current_include_filters=None,
-            is_binary=False):
+            is_binary=False,
+            empty_pages_are_a_change=False):
         # Should set self.error, self.status_code and self.content
         pass
 

changedetectionio/content_fetchers/playwright.py

@@ -83,7 +83,8 @@ class fetcher(Fetcher):
             request_method,
             ignore_status_codes=False,
             current_include_filters=None,
-            is_binary=False):
+            is_binary=False,
+            empty_pages_are_a_change=False):
 
         from playwright.sync_api import sync_playwright
         import playwright._impl._errors
@@ -130,7 +131,7 @@ class fetcher(Fetcher):
             if response is None:
                 context.close()
                 browser.close()
-                logger.debug("Content Fetcher > Response object was none")
+                logger.debug("Content Fetcher > Response object from the browser communication was none")
                 raise EmptyReply(url=url, status_code=None)
 
             try:
@@ -166,10 +167,10 @@ class fetcher(Fetcher):
 
                 raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
 
-            if len(self.page.content().strip()) == 0:
+            if not empty_pages_are_a_change and len(self.page.content().strip()) == 0:
+                logger.debug("Content Fetcher > Content was empty, empty_pages_are_a_change = False")
                 context.close()
                 browser.close()
-                logger.debug("Content Fetcher > Content was empty")
                 raise EmptyReply(url=url, status_code=response.status)
 
             # Run Browser Steps here

changedetectionio/content_fetchers/puppeteer.py

@@ -75,7 +75,8 @@ class fetcher(Fetcher):
                          request_method,
                          ignore_status_codes,
                          current_include_filters,
-                         is_binary
+                         is_binary,
+                         empty_pages_are_a_change
                          ):
 
         from changedetectionio.content_fetchers import visualselector_xpath_selectors
@@ -153,7 +154,7 @@ class fetcher(Fetcher):
         if response is None:
             await self.page.close()
             await browser.close()
-            logger.warning("Content Fetcher > Response object was none")
+            logger.warning("Content Fetcher > Response object was none (as in, the response from the browser was empty, not just the content)")
             raise EmptyReply(url=url, status_code=None)
 
         self.headers = response.headers
@@ -186,10 +187,11 @@ class fetcher(Fetcher):
 
             raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, screenshot=screenshot)
         content = await self.page.content
-        if len(content.strip()) == 0:
+
+        if not empty_pages_are_a_change and len(content.strip()) == 0:
+            logger.error("Content Fetcher > Content was empty (empty_pages_are_a_change is False), closing browsers")
             await self.page.close()
             await browser.close()
-            logger.error("Content Fetcher > Content was empty")
             raise EmptyReply(url=url, status_code=response.status)
 
         # Run Browser Steps here
@@ -247,7 +249,7 @@ class fetcher(Fetcher):
         await self.fetch_page(**kwargs)
 
     def run(self, url, timeout, request_headers, request_body, request_method, ignore_status_codes=False,
-            current_include_filters=None, is_binary=False):
+            current_include_filters=None, is_binary=False, empty_pages_are_a_change=False):
 
         #@todo make update_worker async which could run any of these content_fetchers within memory and time constraints
         max_time = os.getenv('PUPPETEER_MAX_PROCESSING_TIMEOUT_SECONDS', 180)
@@ -262,7 +264,8 @@ class fetcher(Fetcher):
                 request_method=request_method,
                 ignore_status_codes=ignore_status_codes,
                 current_include_filters=current_include_filters,
-                is_binary=is_binary
+                is_binary=is_binary,
+                empty_pages_are_a_change=empty_pages_are_a_change
             ), timeout=max_time))
         except asyncio.TimeoutError:
             raise(BrowserFetchTimedOut(msg=f"Browser connected but was unable to process the page in {max_time} seconds."))

changedetectionio/content_fetchers/requests.py

@@ -1,9 +1,9 @@
+from loguru import logger
+import chardet
 import hashlib
 import os
-
-import chardet
 import requests
-
+from changedetectionio import strtobool
 from changedetectionio.content_fetchers.exceptions import BrowserStepsInUnsupportedFetcher, EmptyReply, Non200ErrorCodeReceived
 from changedetectionio.content_fetchers.base import Fetcher
 
@@ -25,7 +25,8 @@ class fetcher(Fetcher):
             request_method,
             ignore_status_codes=False,
             current_include_filters=None,
-            is_binary=False):
+            is_binary=False,
+            empty_pages_are_a_change=False):
 
         if self.browser_steps_get_valid_steps():
             raise BrowserStepsInUnsupportedFetcher(url=url)
@@ -45,13 +46,19 @@ class fetcher(Fetcher):
             if self.system_https_proxy:
                 proxies['https'] = self.system_https_proxy
 
-        r = requests.request(method=request_method,
-                             data=request_body,
-                             url=url,
-                             headers=request_headers,
-                             timeout=timeout,
-                             proxies=proxies,
-                             verify=False)
+        session = requests.Session()
+
+        if strtobool(os.getenv('ALLOW_FILE_URI', 'false')) and url.startswith('file://'):
+            from requests_file import FileAdapter
+            session.mount('file://', FileAdapter())
+
+        r = session.request(method=request_method,
+                            data=request_body.encode('utf-8') if type(request_body) is str else request_body,
+                            url=url,
+                            headers=request_headers,
+                            timeout=timeout,
+                            proxies=proxies,
+                            verify=False)
 
         # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
         # For example - some sites don't tell us it's utf-8, but return utf-8 content
@@ -67,7 +74,10 @@ class fetcher(Fetcher):
         self.headers = r.headers
 
         if not r.content or not len(r.content):
-            raise EmptyReply(url=url, status_code=r.status_code)
+            if not empty_pages_are_a_change:
+                raise EmptyReply(url=url, status_code=r.status_code)
+            else:
+                logger.debug(f"URL {url} gave zero byte content reply with Status Code {r.status_code}, but empty_pages_are_a_change = True")
 
         # @todo test this
         # @todo maybe you really want to test zero-byte return pages?

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -75,6 +75,7 @@ function isItemInStock() {
         'vergriffen',
         'vorbestellen',
         'vorbestellung ist bald möglich',
+        'we don\'t currently have any',
         'we couldn\'t find any products that match',
         'we do not currently have an estimate of when this product will be back in stock.',
         'we don\'t know when or if this item will be back in stock.',
@@ -173,7 +174,8 @@ function isItemInStock() {
         const element = elementsToScan[i];
         // outside the 'fold' or some weird text in the heading area
         // .getBoundingClientRect() was causing a crash in chrome 119, can only be run on contentVisibility != hidden
-        if (element.getBoundingClientRect().top + window.scrollY >= vh + 150 || element.getBoundingClientRect().top + window.scrollY <= 100) {
+        // Note: theres also an automated test that places the 'out of stock' text fairly low down
+        if (element.getBoundingClientRect().top + window.scrollY >= vh + 250 || element.getBoundingClientRect().top + window.scrollY <= 100) {
             continue
         }
         elementText = "";
@@ -187,7 +189,7 @@ function isItemInStock() {
             // and these mean its out of stock
             for (const outOfStockText of outOfStockTexts) {
                 if (elementText.includes(outOfStockText)) {
-                    console.log(`Selected 'Out of Stock' - found text "${outOfStockText}" - "${elementText}"`)
+                    console.log(`Selected 'Out of Stock' - found text "${outOfStockText}" - "${elementText}" - offset top ${element.getBoundingClientRect().top}, page height is ${vh}`)
                     return outOfStockText; // item is out of stock
                 }
             }

changedetectionio/content_fetchers/res/xpath_element_scraper.js

@@ -15,6 +15,7 @@ try {
     console.log(e);
 }
 
+const percentageNumerical = str => Math.round((str.match(/\d/g) || []).length / str.length * 100);
 
 // Include the getXpath script directly, easier than fetching
 function getxpath(e) {
@@ -77,6 +78,30 @@ const findUpTag = (el) => {
     }
     return null;
 }
+// Text width scraper for ML training/detection
+// Create a single canvas and get its 2D context
+const canvas = document.createElement("canvas");
+const context = canvas.getContext("2d");
+
+// Function to get the width and height of the text inside an element and round them to the nearest integer
+function getTextWidthAndHeightinPx(element) {
+    // Set the font to match the style of the text in the element
+    context.font = window.getComputedStyle(element).font;
+
+    // Get the text inside the element
+    const text = element.textContent || element.innerText;
+
+    // Measure the text width
+    const metrics = context.measureText(text);
+    const width = Math.round(metrics.width);
+
+    // Get the font size from the computed style
+    const fontSize = parseFloat(window.getComputedStyle(element).fontSize);
+    const height = Math.round(fontSize); // Using font size as an approximation of height
+
+    // Return both width and height as an object
+    return { textWidth: width, textHeight: height };
+}
 
 
 // @todo - if it's SVG or IMG, go into image diff mode
@@ -122,8 +147,10 @@ const visibleElementsArray = [];
 // Call collectVisibleElements with the starting parent element
 collectVisibleElements(document.body, visibleElementsArray);
 
+// Append any custom selectors to the visibleElementsArray
 
-visibleElementsArray.forEach(function (element) {
+
+function get_element_metadata(element) {
 
     bbox = element.getBoundingClientRect();
 
@@ -164,18 +191,68 @@ visibleElementsArray.forEach(function (element) {
         }
     }
 
+    let label = "none" // A placeholder, the actual labels for training are done by hand for now
 
-    size_pos.push({
+    // Check if the element was found and get its text , not including any child element
+    let text = Array.from(element.childNodes)
+        .filter(node => node.nodeType === Node.TEXT_NODE)
+        .map(node => node.textContent)
+        .join('');
+
+    // Remove any gaps in sequences of newlines and tabs inside the string
+    text = text.trim().replace(/[\s\t\n\r]{2,}/g, ' ').trim();
+
+    // Try to identify any possible currency amounts "Sale: 4000" or "Sale now 3000 Kc", can help with the training.
+    // @todo could be instead of USD/AUD etc [A-Z]{2,3} ?
+    //const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6)) ) &&  /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|RM|,–)/.test(text) ;
+    const hasDigitCurrency = (/\d/.test(text.slice(0, 6)) || /\d/.test(text.slice(-6)) ) &&  /([€£$¥₩₹]|USD|AUD|EUR|Kč|kr|SEK|RM|,–)/.test(text) ;
+    const hasDigit = /[0-9]/.test(text) ;
+
+    // Sizing of the actual text inside the element can be very different from the elements size
+    const { textWidth, textHeight } = getTextWidthAndHeightinPx(element);
+
+    const computedStyle = window.getComputedStyle(element);
+    let red, green, blue;
+
+    if (text.length) {
+        // Extract the RGB values from the color string (format: rgb(r, g, b))
+        [red, green, blue] = computedStyle.color.match(/\d+/g).map(Number);
+    } else {
+        // Assign default values if text is empty
+        [red, green, blue] = [0, 0, 0];
+    }
+    return {
         xpath: xpath_result,
         width: Math.round(bbox['width']),
         height: Math.round(bbox['height']),
         left: Math.floor(bbox['left']),
         top: Math.floor(bbox['top']) + scroll_y,
+        // tagName used by Browser Steps
         tagName: (element.tagName) ? element.tagName.toLowerCase() : '',
+        // tagtype used by Browser Steps
         tagtype: (element.tagName.toLowerCase() === 'input' && element.type) ? element.type.toLowerCase() : '',
-        isClickable: window.getComputedStyle(element).cursor == "pointer"
-    });
+        isClickable: window.getComputedStyle(element).cursor === "pointer",
+        // Used by the keras/pytorch trainer
+        fontSize: window.getComputedStyle(element).getPropertyValue('font-size'),
+        fontWeight: window.getComputedStyle(element).getPropertyValue('font-weight'),
+        pcNumerical: text.length && percentageNumerical(text),
+        hasDigit: hasDigit,
+        hasDigitCurrency: hasDigitCurrency,
+        textWidth: textWidth,
+        textHeight: textHeight,
+        textLength: text.length,
+        t_r: red,
+        t_g: green,
+        t_b: blue,
+        label: label,
+    };
+}
 
+visibleElementsArray.forEach(function (element) {
+    let metadata = get_element_metadata(element);
+    if(metadata) {
+        size_pos.push(metadata);
+    }
 });
 
 
@@ -184,7 +261,19 @@ visibleElementsArray.forEach(function (element) {
 if (include_filters.length) {
     let results;
     // Foreach filter, go and find it on the page and add it to the results so we can visualise it again
+    outerLoop:
     for (const f of include_filters) {
+        // Quick check so we dont end up with duplicates in the training data
+        for (let index = 0; index < size_pos.length; index++) {
+            let item = size_pos[index];
+            if (item.xpath === f) {
+                item.highlight_as_custom_filter = true;
+                item.found_as_duplicate = true;
+                item.label = "price";
+                continue outerLoop;
+            }
+        }
+
         bbox = false;
         q = false;
 
@@ -205,7 +294,6 @@ if (include_filters.length) {
                 }
             } else {
                 console.log("[css] Scanning for included filter " + f)
-                console.log("[css] Scanning for included filter " + f);
                 results = document.querySelectorAll(f);
             }
         } catch (e) {
@@ -214,7 +302,7 @@ if (include_filters.length) {
             console.log(e);
         }
 
-        if (results.length) {
+        if (results != null && results.length) {
 
             // Iterate over the results
             results.forEach(node => {
@@ -242,17 +330,15 @@ if (include_filters.length) {
                         console.log("xpath_element_scraper: error looking up q.ownerElement")
                     }
                 }
-
-                if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
-                    size_pos.push({
-                        xpath: f,
-                        width: parseInt(bbox['width']),
-                        height: parseInt(bbox['height']),
-                        left: parseInt(bbox['left']),
-                        top: parseInt(bbox['top']) + scroll_y,
-                        highlight_as_custom_filter: true
-                    });
+                element_info = get_element_metadata(node);
+                if(element_info) {
+                    // Be sure we use exactly what was written
+                    element_info['xpath'] = f;
+                    element_info['highlight_as_custom_filter'] = true;
+                    element_info['label'] = "price";
+                    size_pos.push(element_info);
                 }
+
             });
         }
     }

changedetectionio/content_fetchers/webdriver_selenium.py

@@ -56,7 +56,8 @@ class fetcher(Fetcher):
             request_method,
             ignore_status_codes=False,
             current_include_filters=None,
-            is_binary=False):
+            is_binary=False,
+            empty_pages_are_a_change=False):
 
         from selenium import webdriver
         from selenium.webdriver.chrome.options import Options as ChromeOptions

changedetectionio/flask_app.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import datetime
 import flask_login
@@ -532,12 +532,21 @@ def changedetection_app(config=None, datastore_o=None):
     @login_optionally_required
     def ajax_callback_send_notification_test(watch_uuid=None):
 
-        # Watch_uuid could be unsuet in the case its used in tag editor, global setings
+        # Watch_uuid could be unset in the case its used in tag editor, global setings
         import apprise
+        import random
         from .apprise_asset import asset
         apobj = apprise.Apprise(asset=asset)
 
-        watch = datastore.data['watching'].get(watch_uuid) if watch_uuid else None
+        is_global_settings_form = request.args.get('mode', '') == 'global-settings'
+        is_group_settings_form = request.args.get('mode', '') == 'group-settings'
+
+        # Use an existing random one on the global/main settings form
+        if not watch_uuid and (is_global_settings_form or is_group_settings_form):
+            logger.debug(f"Send test notification - Choosing random Watch {watch_uuid}")
+            watch_uuid = random.choice(list(datastore.data['watching'].keys()))
+
+        watch = datastore.data['watching'].get(watch_uuid)
 
         notification_urls = request.form['notification_urls'].strip().splitlines()
 
@@ -549,8 +558,6 @@ def changedetection_app(config=None, datastore_o=None):
                     tag = datastore.tag_exists_by_name(k.strip())
                     notification_urls = tag.get('notifications_urls') if tag and tag.get('notifications_urls') else None
 
-        is_global_settings_form = request.args.get('mode', '') == 'global-settings'
-        is_group_settings_form = request.args.get('mode', '') == 'group-settings'
         if not notification_urls and not is_global_settings_form and not is_group_settings_form:
             # In the global settings, use only what is typed currently in the text box
             logger.debug("Test notification - Trying by global system settings notifications")
@@ -569,7 +576,7 @@ def changedetection_app(config=None, datastore_o=None):
         try:
             # use the same as when it is triggered, but then override it with the form test values
             n_object = {
-                'watch_url': request.form['window_url'],
+                'watch_url': request.form.get('window_url', "https://changedetection.io"),
                 'notification_urls': notification_urls
             }
 
@@ -696,11 +703,16 @@ def changedetection_app(config=None, datastore_o=None):
             form_class = forms.processor_text_json_diff_form
 
         form = form_class(formdata=request.form if request.method == 'POST' else None,
-                               data=default
-                               )
+                          data=default,
+                          extra_notification_tokens=default.extra_notification_token_values()
+                          )
 
-        # For the form widget tag uuid lookup
-        form.tags.datastore = datastore # in _value
+        # For the form widget tag UUID back to "string name" for the field
+        form.tags.datastore = datastore
+
+        # Used by some forms that need to dig deeper
+        form.datastore = datastore
+        form.watch = default
 
         for p in datastore.extra_browsers:
             form.fetch_backend.choices.append(p)
@@ -766,6 +778,11 @@ def changedetection_app(config=None, datastore_o=None):
             datastore.data['watching'][uuid].update(form.data)
             datastore.data['watching'][uuid].update(extra_update_obj)
 
+            if not datastore.data['watching'][uuid].get('tags'):
+                # Force it to be a list, because form.data['tags'] will be string if nothing found
+                # And del(form.data['tags'] ) wont work either for some reason
+                datastore.data['watching'][uuid]['tags'] = []
+
             # Recast it if need be to right data Watch handler
             watch_class = get_custom_watch_obj_for_processor(form.data.get('processor'))
             datastore.data['watching'][uuid] = watch_class(datastore_path=datastore_o.datastore_path, default=datastore.data['watching'][uuid])
@@ -775,9 +792,9 @@ def changedetection_app(config=None, datastore_o=None):
             # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
             # But in the case something is added we should save straight away
             datastore.needs_write_urgent = True
-
-            # Queue the watch for immediate recheck, with a higher priority
-            update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
+            if not datastore.data['watching'][uuid].get('paused'):
+                # Queue the watch for immediate recheck, with a higher priority
+                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
 
             # Diff page [edit] link should go back to diff page
             if request.args.get("next") and request.args.get("next") == 'diff':
@@ -815,6 +832,7 @@ def changedetection_app(config=None, datastore_o=None):
                 'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
                 'extra_title': f" - Edit - {watch.label}",
                 'extra_processor_config': form.extra_tab_content(),
+                'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
                 'form': form,
                 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,
                 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
@@ -869,7 +887,8 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status
         form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,
-                                        data=default
+                                        data=default,
+                                        extra_notification_tokens=datastore.get_unique_notification_tokens_available()
                                         )
 
         # Remove the last option 'System default'
@@ -921,6 +940,7 @@ def changedetection_app(config=None, datastore_o=None):
         output = render_template("settings.html",
                                  api_key=datastore.data['settings']['application'].get('api_access_token'),
                                  emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
+                                 extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(),
                                  form=form,
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
                                  min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
@@ -1349,6 +1369,32 @@ def changedetection_app(config=None, datastore_o=None):
         except FileNotFoundError:
             abort(404)
 
+    @app.route("/edit/<string:uuid>/get-html", methods=['GET'])
+    @login_optionally_required
+    def watch_get_latest_html(uuid):
+        from io import BytesIO
+        from flask import send_file
+        import brotli
+
+        watch = datastore.data['watching'].get(uuid)
+        if watch and watch.history.keys() and os.path.isdir(watch.watch_data_dir):
+            latest_filename = list(watch.history.keys())[-1]
+            html_fname = os.path.join(watch.watch_data_dir, f"{latest_filename}.html.br")
+            with open(html_fname, 'rb') as f:
+                if html_fname.endswith('.br'):
+                    # Read and decompress the Brotli file
+                    decompressed_data = brotli.decompress(f.read())
+                else:
+                    decompressed_data = f.read()
+
+            buffer = BytesIO(decompressed_data)
+
+            return send_file(buffer, as_attachment=True, download_name=f"{latest_filename}.html", mimetype='text/html')
+
+
+        # Return a 500 error
+        abort(500)
+
     @app.route("/form/add/quickwatch", methods=['POST'])
     @login_optionally_required
     def form_quick_watch_add():
@@ -1548,9 +1594,22 @@ def changedetection_app(config=None, datastore_o=None):
                     for uuid in uuids:
                         uuid = uuid.strip()
                         if datastore.data['watching'].get(uuid):
+                            # Bug in old versions caused by bad edit page/tag handler
+                            if isinstance(datastore.data['watching'][uuid]['tags'], str):
+                                datastore.data['watching'][uuid]['tags'] = []
+
                             datastore.data['watching'][uuid]['tags'].append(tag_uuid)
 
-            flash("{} watches assigned tag".format(len(uuids)))
+            flash(f"{len(uuids)} watches were tagged")
+
+        elif op.startswith('mode:'):
+            mode = op.replace('mode:','')
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    datastore.data['watching'][uuid]['processor'] = mode
+            flash(f"{len(uuids)} watches changed modes")
+
 
         return redirect(url_for('index'))
 

changedetectionio/forms.py

@@ -231,9 +231,6 @@ class ValidateJinja2Template(object):
     """
     Validates that a {token} is from a valid set
     """
-    def __init__(self, message=None):
-        self.message = message
-
     def __call__(self, form, field):
         from changedetectionio import notification
 
@@ -248,6 +245,10 @@ class ValidateJinja2Template(object):
         try:
             jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader)
             jinja2_env.globals.update(notification.valid_tokens)
+            # Extra validation tokens provided on the form_class(... extra_tokens={}) setup
+            if hasattr(field, 'extra_notification_tokens'):
+                jinja2_env.globals.update(field.extra_notification_tokens)
+
             jinja2_env.from_string(joined_data).render()
         except TemplateSyntaxError as e:
             raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
@@ -422,6 +423,12 @@ class quickWatchForm(Form):
 class commonSettingsForm(Form):
     from . import processors
 
+    def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs):
+        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
+        self.notification_body.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
+        self.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
+        self.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
+
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
     fetch_backend = RadioField(u'Fetch Method', choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
@@ -429,8 +436,8 @@ class commonSettingsForm(Form):
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
     processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff")
-    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1,
-                                                                                                                                    message="Should contain one or more seconds")])
+    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")])
+
 
 class importForm(Form):
     from . import processors
@@ -590,6 +597,11 @@ class globalSettingsForm(Form):
     # Define these as FormFields/"sub forms", this way it matches the JSON storage
     # datastore.data['settings']['application']..
     # datastore.data['settings']['requests']..
+    def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs):
+        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
+        self.application.notification_body.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
+        self.application.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
+        self.application.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
 
     requests = FormField(globalSettingsRequestForm)
     application = FormField(globalSettingsApplicationForm)

changedetectionio/model/Tag.py

@@ -1,14 +1,14 @@
 
 from changedetectionio.model import watch_base
 
+
 class model(watch_base):
 
     def __init__(self, *arg, **kw):
-
         super(model, self).__init__(*arg, **kw)
 
+        self['overrides_watch'] = kw.get('default', {}).get('overrides_watch')
+
         if kw.get('default'):
             self.update(kw['default'])
             del kw['default']
-
-

changedetectionio/model/Watch.py

@@ -432,6 +432,17 @@ class model(watch_base):
     def toggle_mute(self):
         self['notification_muted'] ^= True
 
+    def extra_notification_token_values(self):
+        # Used for providing extra tokens
+        # return {'widget': 555}
+        return {}
+
+    def extra_notification_token_placeholder_info(self):
+        # Used for providing extra tokens
+        # return [('widget', "Get widget amounts")]
+        return []
+
+
     def extract_regex_from_all_history(self, regex):
         import csv
         import re
@@ -507,7 +518,7 @@ class model(watch_base):
         self.ensure_data_dir_exists()
 
         with open(target_path, 'w') as f:
-            f.write(json.dumps(data))
+            f.write(json.dumps(data, indent=2))
             f.close()
 
     # Save as PNG, PNG is larger but better for doing visual diff in the future

changedetectionio/notification.py

@@ -107,7 +107,7 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
 
     r(results.get('url'),
       auth=auth,
-      data=body,
+      data=body.encode('utf-8') if type(body) is str else body,
       headers=headers,
       params=params
       )
@@ -157,7 +157,7 @@ def process_notification(n_object, datastore):
                 logger.warning(f"Process Notification: skipping empty notification URL.")
                 continue
 
-            logger.info(">> Process Notification: AppRise notifying {}".format(url))
+            logger.info(f">> Process Notification: AppRise notifying {url}")
             url = jinja_render(template_str=url, **notification_parameters)
 
             # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
@@ -230,6 +230,7 @@ def process_notification(n_object, datastore):
         log_value = logs.getvalue()
 
         if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
+            logger.critical(log_value)
             raise Exception(log_value)
 
     # Return what was sent for better logging - after the for loop
@@ -272,19 +273,18 @@ def create_notification_parameters(n_object, datastore):
     tokens.update(
         {
             'base_url': base_url,
-            'current_snapshot': n_object.get('current_snapshot', ''),
-            'diff': n_object.get('diff', ''),  # Null default in the case we use a test
-            'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test
-            'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
-            'diff_patch': n_object.get('diff_patch', ''),  # Null default in the case we use a test
-            'diff_removed': n_object.get('diff_removed', ''),  # Null default in the case we use a test
             'diff_url': diff_url,
             'preview_url': preview_url,
-            'triggered_text': n_object.get('triggered_text', ''),
             'watch_tag': watch_tag if watch_tag is not None else '',
             'watch_title': watch_title if watch_title is not None else '',
             'watch_url': watch_url,
             'watch_uuid': uuid,
         })
 
+    # n_object will contain diff, diff_added etc etc
+    tokens.update(n_object)
+
+    if uuid:
+        tokens.update(datastore.data['watching'].get(uuid).extra_notification_token_values())
+
     return tokens

changedetectionio/processors/__init__.py

@@ -26,6 +26,8 @@ class difference_detection_processor():
 
     def call_browser(self):
         from requests.structures import CaseInsensitiveDict
+        from changedetectionio.content_fetchers.exceptions import EmptyReply
+
         # Protect against file:// access
         if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE):
             if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
@@ -133,8 +135,18 @@ class difference_detection_processor():
         is_binary = self.watch.is_pdf
 
         # And here we go! call the right browser with browser-specific settings
-        self.fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, self.watch.get('include_filters'),
-                    is_binary=is_binary)
+        empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
+
+        self.fetcher.run(url=url,
+                         timeout=timeout,
+                         request_headers=request_headers,
+                         request_body=request_body,
+                         request_method=request_method,
+                         ignore_status_codes=ignore_status_codes,
+                         current_include_filters=self.watch.get('include_filters'),
+                         is_binary=is_binary,
+                         empty_pages_are_a_change=empty_pages_are_a_change
+                         )
 
         #@todo .quit here could go on close object, so we can run JS if change-detected
         self.fetcher.quit()

changedetectionio/processors/restock_diff/__init__.py

@@ -1,11 +1,12 @@
 
-from changedetectionio.model.Watch import model as BaseWatch
-import re
 from babel.numbers import parse_decimal
+from changedetectionio.model.Watch import model as BaseWatch
+from typing import Union
+import re
 
 class Restock(dict):
 
-    def parse_currency(self, raw_value: str) -> float:
+    def parse_currency(self, raw_value: str) -> Union[float, None]:
         # Clean and standardize the value (ie 1,400.00 should be 1400.00), even better would be store the whole thing as an integer.
         standardized_value = raw_value
 
@@ -21,8 +22,11 @@ class Restock(dict):
         # Remove any non-numeric characters except for the decimal point
         standardized_value = re.sub(r'[^\d.-]', '', standardized_value)
 
-        # Convert to float
-        return float(parse_decimal(standardized_value, locale='en'))
+        if standardized_value:
+            # Convert to float
+            return float(parse_decimal(standardized_value, locale='en'))
+
+        return None
 
     def __init__(self, *args, **kwargs):
         # Define default values
@@ -45,13 +49,10 @@ class Restock(dict):
 
     def __setitem__(self, key, value):
         # Custom logic to handle setting price and original_price
-        if key == 'price':
+        if key == 'price' or key == 'original_price':
             if isinstance(value, str):
                 value = self.parse_currency(raw_value=value)
 
-            if value and not self.get('original_price'):
-                self['original_price'] = value
-
         super().__setitem__(key, value)
 
 class Watch(BaseWatch):
@@ -59,7 +60,25 @@ class Watch(BaseWatch):
         super().__init__(*arg, **kw)
         self['restock'] = Restock(kw['default']['restock']) if kw.get('default') and kw['default'].get('restock') else Restock()
 
+        self['restock_settings'] = kw['default']['restock_settings'] if kw.get('default',{}).get('restock_settings') else {
+            'follow_price_changes': True,
+            'in_stock_processing' : 'in_stock_only'
+        } #@todo update
+
     def clear_watch(self):
         super().clear_watch()
         self.update({'restock': Restock()})
 
+    def extra_notification_token_values(self):
+        values = super().extra_notification_token_values()
+        values['restock'] = self.get('restock', {})
+        return values
+
+    def extra_notification_token_placeholder_info(self):
+        values = super().extra_notification_token_placeholder_info()
+
+        values.append(('restock.price', "Price detected"))
+        values.append(('restock.original_price', "Original price at first check"))
+
+        return values
+

changedetectionio/processors/restock_diff/forms.py

@@ -1,17 +1,25 @@
-
 from wtforms import (
     BooleanField,
     validators,
     FloatField
 )
+from wtforms.fields.choices import RadioField
+from wtforms.fields.form import FormField
+from wtforms.form import Form
 
 from changedetectionio.forms import processor_text_json_diff_form
 
-class processor_settings_form(processor_text_json_diff_form):
-    in_stock_only = BooleanField('Only trigger when product goes BACK to in-stock', default=True)
-    price_change_min = FloatField('Minimum amount to trigger notification', [validators.Optional()],
+
+class RestockSettingsForm(Form):
+    in_stock_processing = RadioField(label='Re-stock detection', choices=[
+        ('in_stock_only', "In Stock only (Out Of Stock -> In Stock only)"),
+        ('all_changes', "Any availability changes"),
+        ('off', "Off, don't follow availability/restock"),
+    ], default="in_stock_only")
+
+    price_change_min = FloatField('Below price to trigger notification', [validators.Optional()],
                                   render_kw={"placeholder": "No limit", "size": "10"})
-    price_change_max = FloatField('Maximum amount to trigger notification', [validators.Optional()],
+    price_change_max = FloatField('Above price to trigger notification', [validators.Optional()],
                                   render_kw={"placeholder": "No limit", "size": "10"})
     price_change_threshold_percent = FloatField('Threshold in % for price changes since the original price', validators=[
 
@@ -19,45 +27,55 @@ class processor_settings_form(processor_text_json_diff_form):
         validators.NumberRange(min=0, max=100, message="Should be between 0 and 100"),
     ], render_kw={"placeholder": "0%", "size": "5"})
 
-    follow_price_changes = BooleanField('Follow price changes', default=False)
+    follow_price_changes = BooleanField('Follow price changes', default=True)
+
+class processor_settings_form(processor_text_json_diff_form):
+    restock_settings = FormField(RestockSettingsForm)
 
     def extra_tab_content(self):
         return 'Restock & Price Detection'
 
     def extra_form_content(self):
-        return """
+        output = ""
+
+        if getattr(self, 'watch', None) and getattr(self, 'datastore'):
+            for tag_uuid in self.watch.get('tags'):
+                tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {})
+                if tag.get('overrides_watch'):
+                    # @todo - Quick and dirty, cant access 'url_for' here because its out of scope somehow
+                    output = f"""<p><strong>Note! A Group tag overrides the restock and price detection here.</strong></p><style>#restock-fieldset-price-group {{ opacity: 0.6; }}</style>"""
+
+        output += """
         {% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
         <script>        
             $(document).ready(function () {
-                toggleOpacity('#follow_price_changes', '.price-change-minmax', true);
+                toggleOpacity('#restock_settings-follow_price_changes', '.price-change-minmax', true);
             });
         </script>
 
-
-        <fieldset>
+        <fieldset id="restock-fieldset-price-group">
             <div class="pure-control-group">
-                <fieldset class="pure-group">
-                    {{ render_checkbox_field(form.in_stock_only) }}
-                    <span class="pure-form-message-inline">Only trigger re-stock notification when page changes from <strong>out of stock</strong> to <strong>back in stock</strong></span>
+                <fieldset class="pure-group inline-radio">
+                    {{ render_field(form.restock_settings.in_stock_processing) }}
                 </fieldset>
                 <fieldset class="pure-group">
-                    {{ render_checkbox_field(form.follow_price_changes) }}
+                    {{ render_checkbox_field(form.restock_settings.follow_price_changes) }}
                     <span class="pure-form-message-inline">Changes in price should trigger a notification</span>
-                    <br>
-                    <span class="pure-form-message-inline">When OFF - Only care about restock detection</span>                    
                 </fieldset>
                 <fieldset class="pure-group price-change-minmax">               
-                    {{ render_field(form.price_change_min, placeholder=watch['restock']['price']) }}
-                    <span class="pure-form-message-inline">Minimum amount, only trigger a change when the price is less than this amount.</span>
+                    {{ render_field(form.restock_settings.price_change_min, placeholder=watch.get('restock', {}).get('price')) }}
+                    <span class="pure-form-message-inline">Minimum amount, Trigger a change/notification when the price drops <i>below</i> this value.</span>
                 </fieldset>
                 <fieldset class="pure-group price-change-minmax">
-                    {{ render_field(form.price_change_max, placeholder=watch['restock']['price']) }}
-                    <span class="pure-form-message-inline">Maximum amount, only trigger a change when the price is more than this amount.</span>
+                    {{ render_field(form.restock_settings.price_change_max, placeholder=watch.get('restock', {}).get('price')) }}
+                    <span class="pure-form-message-inline">Maximum amount, Trigger a change/notification when the price rises <i>above</i> this value.</span>
                 </fieldset>
                 <fieldset class="pure-group price-change-minmax">
-                    {{ render_field(form.price_change_threshold_percent) }}
-                    <span class="pure-form-message-inline">Price must change more than this % to trigger a change.</span><br>
+                    {{ render_field(form.restock_settings.price_change_threshold_percent) }}
+                    <span class="pure-form-message-inline">Price must change more than this % to trigger a change since the first check.</span><br>
                     <span class="pure-form-message-inline">For example, If the product is $1,000 USD originally, <strong>2%</strong> would mean it has to change more than $20 since the first check.</span><br>
                 </fieldset>                
             </div>
-        </fieldset>"""
+        </fieldset>
+        """
+        return output

changedetectionio/processors/restock_diff/processor.py

@@ -3,10 +3,13 @@ from ..exceptions import ProcessorException
 from . import Restock
 from loguru import logger
 import hashlib
+import os
 import re
 import urllib3
 import time
 
+from ...html_tools import html_to_text
+
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 name = 'Re-stock & Price detection for single product pages'
 description = 'Detects if the product goes back to in-stock'
@@ -34,23 +37,28 @@ def get_itemprop_availability(html_content) -> Restock:
     Kind of funny/cool way to find price/availability in one many different possibilities.
     Use 'extruct' to find any possible RDFa/microdata/json-ld data, make a JSON string from the output then search it.
     """
+
     from jsonpath_ng import parse
 
     now = time.time()
     import extruct
     logger.trace(f"Imported extruct module in {time.time() - now:.3f}s")
 
-    value = {}
     now = time.time()
+
     # Extruct is very slow, I'm wondering if some ML is going to be faster (800ms on my i7), 'rdfa' seems to be the heaviest.
-
     syntaxes = ['dublincore', 'json-ld', 'microdata', 'microformat', 'opengraph']
+    try:
+        data = extruct.extract(html_content, syntaxes=syntaxes)
+    except Exception as e:
+        logger.warning(f"Unable to extract data, document parsing with extruct failed with {type(e).__name__} - {str(e)}")
+        return Restock()
 
-    data = extruct.extract(html_content, syntaxes=syntaxes)
     logger.trace(f"Extruct basic extract of all metadata done in {time.time() - now:.3f}s")
 
     # First phase, dead simple scanning of anything that looks useful
     value = Restock()
+    return value
     if data:
         logger.debug(f"Using jsonpath to find price/availability/etc")
         price_parse = parse('$..(price|Price)')
@@ -118,6 +126,52 @@ class perform_site_check(difference_detection_processor):
     screenshot = None
     xpath_data = None
 
+    def ML_scrape_for_price_data(self, ML_price_scraper_url):
+        import requests
+        from changedetectionio import html_tools
+
+        price_info = None
+
+        # Perform the POST request
+        response = requests.post(ML_price_scraper_url, json=self.fetcher.xpath_data)
+        logger.debug(f"ML Price scraper - {ML_price_scraper_url} Response OK? - '{response.ok}'")
+        # Check if the response contains a dict
+        if response.ok:  # This checks if the request was successful (status code 200-299)
+            response_json = response.json()
+            logger.debug(f"ML Price scraper: response - {response_json}'")
+            if isinstance(response_json, dict) and 'idx' in response_json.keys():
+                suggested_xpath_idx = response_json.get('idx')
+                if response_json.get('score') <0.80 or response_json.get('score') > 1.0:
+                    logger.warning(f"Predict score was outside normal range, aborting ML/AI price check, needs better training data in this case?")
+                    return None
+
+                # Use the path provided to extra the price text
+                from price_parser import Price
+                scrape_element = self.fetcher.xpath_data.get('size_pos', {})[suggested_xpath_idx]
+                logger.debug(f"Predicted selector with price information is {scrape_element['xpath']}")
+
+                result_s = None
+                if scrape_element['xpath'][0] == '/' or scrape_element['xpath'].startswith('xpath'):
+                    result_s = html_tools.xpath_filter(xpath_filter=scrape_element['xpath'],
+                                                       html_content=self.fetcher.content)
+                else:
+                    # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+                    result_s = html_tools.include_filters(include_filters=scrape_element['xpath'],
+                                                          html_content=self.fetcher.content)
+
+                if result_s:
+                    text = html_to_text(result_s)
+                    logger.debug(f"Guessed the text '{text}' as the price information")
+                    if text:
+                        price_info = Price.fromstring(text)
+            else:
+                logger.error(f"ML Price scraper: missing xpath index (IDX) in response?")
+        else:
+            print(f"ML Price scraper: Request failed with status code: {response.status_code}")
+
+#@TODO THROW HELPFUL MESSAGE WITH LINK TO TUTORIAL IF IT CANT CONNECT!
+        return price_info
+
     def run_changedetection(self, watch, skip_when_checksum_same=True):
         if not watch:
             raise Exception("Watch no longer exists.")
@@ -132,6 +186,18 @@ class perform_site_check(difference_detection_processor):
         update_obj['content_type'] = self.fetcher.headers.get('Content-Type', '')
         update_obj["last_check_status"] = self.fetcher.get_last_status_code()
 
+        # Which restock settings to compare against?
+        restock_settings = watch.get('restock_settings', {})
+
+        # See if any tags have 'activate for individual watches in this tag/group?' enabled and use the first we find
+        for tag_uuid in watch.get('tags'):
+            tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {})
+            if tag.get('overrides_watch'):
+                restock_settings = tag.get('restock_settings', {})
+                logger.info(f"Watch {watch.get('uuid')} - Tag '{tag.get('title')}' selected for restock settings override")
+                break
+
+
         itemprop_availability = {}
         try:
             itemprop_availability = get_itemprop_availability(html_content=self.fetcher.content)
@@ -162,9 +228,29 @@ class perform_site_check(difference_detection_processor):
                 else:
                     update_obj['restock']['in_stock'] = False
 
+        # Attempt to pass the elements off to the machine-learning endpoint if its enabled
+        # This might return a confident guess as to which element contains the price data
+        if not itemprop_availability.get('price'):
+            ML_price_scraper_url = os.getenv("PRICE_SCRAPER_ML_ENDPOINT")
+            if self.fetcher.xpath_data and ML_price_scraper_url:
+                price_info = self.ML_scrape_for_price_data(ML_price_scraper_url)
+                if price_info and price_info.amount:
+                    logger.success(f"ML Price scraper: Got price data {price_info}")
+                    itemprop_availability['price'] = f"{price_info.amount}"
+                    update_obj['restock']['price'] = f"{price_info.amount}"
+                if price_info and price_info.currency:
+                    itemprop_availability['currency'] = price_info.currency
+                    update_obj['restock']['currency'] = price_info.currency
+
+
         # Main detection method
         fetched_md5 = None
 
+        # store original price if not set
+        if itemprop_availability and itemprop_availability.get('price') and not itemprop_availability.get('original_price'):
+            itemprop_availability['original_price'] = itemprop_availability.get('price')
+            update_obj['restock']["original_price"] = itemprop_availability.get('price')
+
         if not self.fetcher.instock_data and not itemprop_availability.get('availability'):
             raise ProcessorException(
                 message=f"Unable to extract restock data for this page unfortunately. (Got code {self.fetcher.get_last_status_code()} from server), no embedded stock information was found and nothing interesting in the text, try using this watch with Chrome.",
@@ -183,7 +269,7 @@ class perform_site_check(difference_detection_processor):
 
         # What we store in the snapshot
         price = update_obj.get('restock').get('price') if update_obj.get('restock').get('price') else ""
-        snapshot_content = f"{update_obj.get('restock').get('in_stock')} - {price}"
+        snapshot_content = f"In Stock: {update_obj.get('restock').get('in_stock')} - Price: {price}"
 
         # Main detection method
         fetched_md5 = hashlib.md5(snapshot_content.encode('utf-8')).hexdigest()
@@ -195,14 +281,14 @@ class perform_site_check(difference_detection_processor):
         # out of stock -> back in stock only?
         if watch.get('restock') and watch['restock'].get('in_stock') != update_obj['restock'].get('in_stock'):
             # Yes if we only care about it going to instock, AND we are in stock
-            if watch.get('in_stock_only') and update_obj['restock']['in_stock']:
+            if restock_settings.get('in_stock_processing') == 'in_stock_only' and update_obj['restock']['in_stock']:
                 changed_detected = True
 
-            if not watch.get('in_stock_only'):
+            if restock_settings.get('in_stock_processing') == 'all_changes':
                 # All cases
                 changed_detected = True
 
-        if watch.get('follow_price_changes') and watch.get('restock') and update_obj.get('restock') and update_obj['restock'].get('price'):
+        if restock_settings.get('follow_price_changes') and watch.get('restock') and update_obj.get('restock') and update_obj['restock'].get('price'):
             price = float(update_obj['restock'].get('price'))
             # Default to current price if no previous price found
             if watch['restock'].get('original_price'):
@@ -214,26 +300,25 @@ class perform_site_check(difference_detection_processor):
             # Minimum/maximum price limit
             if update_obj.get('restock') and update_obj['restock'].get('price'):
                 logger.debug(
-                    f"{watch.get('uuid')} - Change was detected, 'price_change_max' is '{watch.get('price_change_max', '')}' 'price_change_min' is '{watch.get('price_change_min', '')}', price from website is '{update_obj['restock'].get('price', '')}'.")
+                    f"{watch.get('uuid')} - Change was detected, 'price_change_max' is '{restock_settings.get('price_change_max', '')}' 'price_change_min' is '{restock_settings.get('price_change_min', '')}', price from website is '{update_obj['restock'].get('price', '')}'.")
                 if update_obj['restock'].get('price'):
-                    min_limit = float(watch.get('price_change_min')) if watch.get('price_change_min') else None
-                    max_limit = float(watch.get('price_change_max')) if watch.get('price_change_max') else None
+                    min_limit = float(restock_settings.get('price_change_min')) if restock_settings.get('price_change_min') else None
+                    max_limit = float(restock_settings.get('price_change_max')) if restock_settings.get('price_change_max') else None
 
                     price = float(update_obj['restock'].get('price'))
                     logger.debug(f"{watch.get('uuid')} after float conversion - Min limit: '{min_limit}' Max limit: '{max_limit}' Price: '{price}'")
                     if min_limit or max_limit:
                         if is_between(number=price, lower=min_limit, upper=max_limit):
-                            logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}")
-                            if changed_detected:
-                                logger.debug(f"{watch.get('uuid')} Override change-detected to FALSE because price was inside threshold")
-                                changed_detected = False
+                            # Price was between min/max limit, so there was nothing todo in any case
+                            logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}, nothing to check, forcing changed_detected = False (was {changed_detected})")
+                            changed_detected = False
                         else:
-                            logger.trace(f"{watch.get('uuid')} {price} is NOT between {min_limit} and {max_limit}")
+                            logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}, continuing normal comparison")
 
                     # Price comparison by %
-                    if watch['restock'].get('original_price') and changed_detected and watch.get('price_change_threshold_percent'):
+                    if watch['restock'].get('original_price') and changed_detected and restock_settings.get('price_change_threshold_percent'):
                         previous_price = float(watch['restock'].get('original_price'))
-                        pc = float(watch.get('price_change_threshold_percent'))
+                        pc = float(restock_settings.get('price_change_threshold_percent'))
                         change = abs((price - previous_price) / previous_price * 100)
                         if change and change <= pc:
                             logger.debug(f"{watch.get('uuid')} Override change-detected to FALSE because % threshold ({pc}%) was {change:.3f}%")

changedetectionio/run_basic_tests.sh

@@ -35,4 +35,8 @@ pytest tests/test_access_control.py
 pytest tests/test_notification.py
 pytest tests/test_backend.py
 pytest tests/test_rss.py
-pytest tests/test_unique_lines.py
+pytest tests/test_unique_lines.py
+
+# Check file:// will pickup a file when enabled
+echo "Hello world" > /tmp/test-file.txt
+ALLOW_FILE_URI=yes pytest tests/test_security.py

changedetectionio/static/js/watch-settings.js

@@ -27,6 +27,5 @@ $(document).ready(function () {
 
     toggleOpacity('#time_between_check_use_default', '#time_between_check', false);
 
-
 });
 

changedetectionio/store.py

@@ -83,16 +83,14 @@ class ChangeDetectionStore:
                         self.__data['settings']['application'].update(from_disk['settings']['application'])
 
                 # Convert each existing watch back to the Watch.model object
-
                 for uuid, watch in self.__data['watching'].items():
-                    watch['uuid'] = uuid
-                    watch_class = get_custom_watch_obj_for_processor(watch.get('processor'))
-                    if watch.get('uuid') != 'text_json_diff':
-                        logger.trace(f"Loading Watch object '{watch_class.__module__}.{watch_class.__name__}' for UUID {uuid}")
+                    self.__data['watching'][uuid] = self.rehydrate_entity(uuid, watch)
+                    logger.info(f"Watching: {uuid} {watch['url']}")
 
-                    self.__data['watching'][uuid] = watch_class(datastore_path=self.datastore_path, default=watch)
-
-                    logger.info(f"Watching: {uuid} {self.__data['watching'][uuid]['url']}")
+                # And for Tags also, should be Restock type because it has extra settings
+                for uuid, tag in self.__data['settings']['application']['tags'].items():
+                    self.__data['settings']['application']['tags'][uuid] = self.rehydrate_entity(uuid, tag, processor_override='restock_diff')
+                    logger.info(f"Tag: {uuid} {tag['title']}")
 
         # First time ran, Create the datastore.
         except (FileNotFoundError):
@@ -147,6 +145,22 @@ class ChangeDetectionStore:
         # Finally start the thread that will manage periodic data saves to JSON
         save_data_thread = threading.Thread(target=self.save_datastore).start()
 
+    def rehydrate_entity(self, uuid, entity, processor_override=None):
+        """Set the dict back to the dict Watch object"""
+        entity['uuid'] = uuid
+
+        if processor_override:
+            watch_class = get_custom_watch_obj_for_processor(processor_override)
+            entity['processor']=processor_override
+        else:
+            watch_class = get_custom_watch_obj_for_processor(entity.get('processor'))
+
+        if entity.get('uuid') != 'text_json_diff':
+            logger.trace(f"Loading Watch object '{watch_class.__module__}.{watch_class.__name__}' for UUID {uuid}")
+
+        entity = watch_class(datastore_path=self.datastore_path, default=entity)
+        return entity
+
     def set_last_viewed(self, uuid, timestamp):
         logger.debug(f"Setting watch UUID: {uuid} last viewed to {int(timestamp)}")
         self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
@@ -185,6 +199,9 @@ class ChangeDetectionStore:
 
     @property
     def has_unviewed(self):
+        if not self.__data.get('watching'):
+            return None
+
         for uuid, watch in self.__data['watching'].items():
             if watch.history_n >= 2 and watch.viewed == False:
                 return True
@@ -614,6 +631,33 @@ class ChangeDetectionStore:
                 return True
         return False
 
+    def get_unique_notification_tokens_available(self):
+        # Ask each type of watch if they have any extra notification token to add to the validation
+        extra_notification_tokens = {}
+        watch_processors_checked = set()
+
+        for watch_uuid, watch in self.__data['watching'].items():
+            processor = watch.get('processor')
+            if processor not in watch_processors_checked:
+                extra_notification_tokens.update(watch.extra_notification_token_values())
+                watch_processors_checked.add(processor)
+
+        return extra_notification_tokens
+
+    def get_unique_notification_token_placeholders_available(self):
+        # The actual description of the tokens, could be combined with get_unique_notification_tokens_available instead of doing this twice
+        extra_notification_tokens = []
+        watch_processors_checked = set()
+
+        for watch_uuid, watch in self.__data['watching'].items():
+            processor = watch.get('processor')
+            if processor not in watch_processors_checked:
+                extra_notification_tokens+=watch.extra_notification_token_placeholder_info()
+                watch_processors_checked.add(processor)
+
+        return extra_notification_tokens
+
+
     def get_updates_available(self):
         import inspect
         updates_available = []
@@ -850,4 +894,17 @@ class ChangeDetectionStore:
                 watch['restock'] = Restock({'in_stock': watch.get('in_stock')})
                 del watch['in_stock']
 
+    # Migrate old restock settings
+    def update_18(self):
+        for uuid, watch in self.data['watching'].items():
+            if not watch.get('restock_settings'):
+                # So we enable price following by default
+                self.data['watching'][uuid]['restock_settings'] = {'follow_price_changes': True}
+
+            # Migrate and cleanoff old value
+            self.data['watching'][uuid]['restock_settings']['in_stock_processing'] = 'in_stock_only' if watch.get(
+                'in_stock_only') else 'all_changes'
+
+            if self.data['watching'][uuid].get('in_stock_only'):
+                del (self.data['watching'][uuid]['in_stock_only'])
 

changedetectionio/templates/_common_fields.html

@@ -1,7 +1,7 @@
 
 {% from '_helpers.html' import render_field %}
 
-{% macro render_common_settings_form(form, emailprefix, settings_application) %}
+{% macro render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) %}
                         <div class="pure-control-group">
                             {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
     Gitter - gitter://token/room
@@ -107,7 +107,15 @@
                                     <tr>
                                         <td><code>{{ '{{triggered_text}}' }}</code></td>
                                         <td>Text that tripped the trigger from filters</td>
-                                    </tr>
+
+                                        {% if extra_notification_token_placeholder_info %}
+                                            {% for token in extra_notification_token_placeholder_info %}
+                                                <tr>
+                                                    <td><code>{{ '{{' }}{{ token[0] }}{{ '}}' }}</code></td>
+                                                    <td>{{ token[1] }}</td>
+                                                </tr>
+                                            {% endfor %}
+                                        {% endif %}
                                     </tbody>
                                 </table>
                                 <div class="pure-form-message-inline">

changedetectionio/templates/edit.html

@@ -246,7 +246,7 @@ User-Agent: wonderbra 1.0") }}
                         {% endif %}
                         <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
 
-                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
+                        {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
                     </div>
                 </fieldset>
             </div>
@@ -479,6 +479,12 @@ Unavailable") }}
                         </tr>
                         </tbody>
                     </table>
+                    {% if watch.history_n %}
+                        <p>
+                             <a href="{{url_for('watch_get_latest_html', uuid=uuid)}}" class="pure-button button-small">Download latest HTML snapshot</a>
+                        </p>
+                    {% endif %}
+
                 </div>
             </div>
             <div id="actions">

changedetectionio/templates/settings.html

@@ -76,7 +76,7 @@
                     </div>
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
-                        <span class="pure-form-message-inline">When a page contains HTML, but no renderable text appears (empty page), is this considered a change?</span>
+                        <span class="pure-form-message-inline">When a request returns no content, or the HTML does not contain any text, is this considered a change?</span>
                     </div>
                 {% if form.requests.proxy %}
                     <div class="pure-control-group inline-radio">
@@ -92,7 +92,7 @@
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div class="field-group">
-                        {{ render_common_settings_form(form.application.form, emailprefix, settings_application) }}
+                        {{ render_common_settings_form(form.application.form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
                     </div>
                 </fieldset>
                 <div class="pure-control-group" id="notification-base-url">

changedetectionio/templates/watch-overview.html

@@ -37,6 +37,8 @@
         <button class="pure-button button-secondary button-xsmall" name="op" value="assign-tag" id="checkbox-assign-tag">Tag</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="mark-viewed">Mark viewed</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="notification-default">Use default notification</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="mode:text_json_diff">Mode: Page changes</button>
+        <button class="pure-button button-secondary button-xsmall" name="op" value="mode:restock_diff">Mode: Price/Restock</button>
         <button class="pure-button button-secondary button-xsmall" name="op" value="clear-errors">Clear errors</button>
         <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="clear-history">Clear/reset history</button>
         <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="delete">Delete</button>

changedetectionio/tests/conftest.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 import resource
 import time
 from threading import Thread
@@ -8,7 +8,7 @@ from changedetectionio import changedetection_app
 from changedetectionio import store
 import os
 import sys
-from loguru import logger
+
 
 # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
 # Much better boilerplate than the docs

changedetectionio/tests/custom_browser_url/test_custom_browser_url.py

@@ -1,4 +1,4 @@
-# !/usr/bin/python3
+#!/usr/bin/env python3
 import os
 
 from flask import url_for

changedetectionio/tests/fetchers/conftest.py

@@ -1,3 +1,3 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from .. import conftest

changedetectionio/tests/fetchers/test_content.py

@@ -1,6 +1,5 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
-import time
 from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks
 import logging

changedetectionio/tests/ml_price_scraper/test_scrape_price_element.py

@@ -0,0 +1,127 @@
+import os
+
+from flask import url_for
+from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
+    wait_for_all_checks, \
+    set_longer_modified_response
+
+import time
+
+# No semantic data just some text, we should be able to find the product price.
+def set_response(price="121.95"):
+    html_content = f"""
+    <!DOCTYPE html>
+    <html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title>Ajax Widget</title>
+        <style>
+            body {{
+                font-family: Arial, sans-serif;
+                margin: 0;
+                padding: 0;
+                display: flex;
+                justify-content: center;
+                align-items: center;
+                height: 100vh;
+                background-color: #f4f4f4;
+            }}
+            .container {{
+                display: flex;
+                flex-direction: row;
+                background-color: #fff;
+                border: 1px solid #ddd;
+                padding: 20px;
+                border-radius: 5px;
+                box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
+                width: 80%;
+                max-width: 800px;
+            }}
+            .description {{
+                flex: 2;
+                margin-right: 20px;
+            }}
+            .description h1 {{
+                margin-top: 0;
+            }}
+            .price {{
+                flex: 1;
+                text-align: right;
+                font-size: 24px;
+                color: #333;
+            }}
+            .price span {{
+                font-size: 32px;
+                font-weight: bold;
+            }}
+            .buy-button {{
+                display: inline-block;
+                margin-top: 20px;
+                padding: 10px 20px;
+                background-color: #28a745;
+                color: #fff;
+                text-decoration: none;
+                border-radius: 5px;
+                font-size: 16px;
+            }}
+            .buy-button:hover {{
+                background-color: #218838;
+            }}
+        </style>
+    </head>
+    <body>
+        <div class="container">
+            <div class="description">
+                <h1>Ajax Widget</h1>
+                <p>The Ajax Widget is the ultimate solution for all your widget needs. Crafted with precision and using the latest technology, this widget offers unmatched performance and durability. Whether you're using it for personal or professional purposes, the Ajax Widget will not disappoint. It's easy to use, reliable, and comes with a sleek design that complements any setup. Don't settle for less; get the best with the Ajax Widget today!</p>
+            </div>
+            <div class="price">
+                <span>${price}</span>
+                <br>
+                <a href="#" class="buy-button">Buy Now</a><br>
+                IN STOCK
+            </div>
+        </div>
+    </body>
+    </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(html_content)
+    time.sleep(1)
+    return None
+
+
+
+
+def test_restock_itemprop_basic(client, live_server):
+
+    # needs to be set and something like 'ws://127.0.0.1:3000'
+    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
+    assert os.getenv('PRICE_SCRAPER_ML_ENDPOINT'), "Needs PRICE_SCRAPER_ML_ENDPOINT set for this test"
+
+
+    live_server_setup(live_server)
+
+    set_response(price="123.99")
+
+    # because it needs to access itself from within the sockpuppetbrowser
+    test_url = url_for('test_endpoint', _external=True)
+    test_url = test_url.replace('localhost.localdomain', 'cdio')
+    test_url = test_url.replace('localhost', 'cdio')
+
+    client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
+        follow_redirects=True
+    )
+    wait_for_all_checks(client)
+    res = client.get(url_for("index"))
+
+    assert b'123.99' in res.data
+    assert b' in-stock' in res.data
+    assert b' not-in-stock' not in res.data
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetectionio/tests/proxy_list/conftest.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from .. import conftest
 

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import os
 from flask import url_for

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/proxy_list/test_proxy.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 import os
 import time
 from flask import url_for

changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 import os
 import time
 from flask import url_for

changedetectionio/tests/restock/conftest.py

@@ -1,3 +1,3 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from .. import conftest

changedetectionio/tests/restock/test_restock.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 import os
 import time
 from flask import url_for

changedetectionio/tests/smtp/smtp-test-server.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 import asyncio
 from aiosmtpd.controller import Controller
 from aiosmtpd.smtp import SMTP

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 import os.path
 import time
 from flask import url_for
@@ -112,7 +112,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     res = client.post(
         url_for("settings_page"),
         data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
-              "application-notification_body": 'triggered text was -{{triggered_text}}-',
+              "application-notification_body": 'triggered text was -{{triggered_text}}- 网站监测 内容更新了',
               # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
               "application-notification_urls": test_notification_url,
               "application-minutes_between_check": 180,
@@ -167,9 +167,10 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa
     # Takes a moment for apprise to fire
     time.sleep(3)
     assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file"
-    with open("test-datastore/notification.txt", 'r') as f:
-        response= f.read()
-        assert '-Oh yes please-' in response
+    with open("test-datastore/notification.txt", 'rb') as f:
+        response = f.read()
+        assert b'-Oh yes please-' in response
+        assert '网站监测 内容更新了'.encode('utf-8') in response
 
 
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_api.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_auth.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_backend.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for
@@ -69,6 +69,12 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     wait_for_all_checks(client)
 
+    uuid = extract_UUID_from_client(client)
+
+    # Check the 'get latest snapshot works'
+    res = client.get(url_for("watch_get_latest_html", uuid=uuid))
+    assert b'which has this one new line' in res.data
+
     # Now something should be ready, indicated by having a 'unviewed' class
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
@@ -86,7 +92,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     assert expected_url.encode('utf-8') in res.data
 
     # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
-    res = client.get(url_for("diff_history_page", uuid="first"))
+    res = client.get(url_for("diff_history_page", uuid=uuid))
     assert b'selected=""' in res.data, "Confirm diff history page loaded"
 
     # Check the [preview] pulls the right one
@@ -143,7 +149,6 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     assert b'unviewed' not in res.data
 
     # #2458 "clear history" should make the Watch object update its status correctly when the first snapshot lands again
-    uuid = extract_UUID_from_client(client)
     client.get(url_for("clear_watch_history", uuid=uuid))
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)

changedetectionio/tests/test_backup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from .util import set_original_response, live_server_setup, wait_for_all_checks
 from flask import url_for

changedetectionio/tests/test_block_while_text_present.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_clone.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_css_selector.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_element_removal.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 

changedetectionio/tests/test_encoding.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 # coding=utf-8
 
 import time

changedetectionio/tests/test_errorhandling.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 

changedetectionio/tests/test_extract_csv.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_extract_regex.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_filter_exist_changes.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # https://www.reddit.com/r/selfhosted/comments/wa89kp/comment/ii3a4g7/?context=3
 import os

changedetectionio/tests/test_group.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_history_consistency.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 import os

changedetectionio/tests/test_html_to_text.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 """Test suite for the method to extract text from an html string"""
 from ..html_tools import html_to_text
 

changedetectionio/tests/test_ignore_regex_text.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from . util import live_server_setup
 from changedetectionio import html_tools

changedetectionio/tests/test_ignore_text.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_ignorehighlighter.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_ignorehyperlinks.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 """Test suite for the render/not render anchor tag content functionality"""
 
 import time

changedetectionio/tests/test_ignorestatuscode.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_ignorewhitespace.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_import.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 import io
 import os
 import time

changedetectionio/tests/test_jinja2.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 # coding=utf-8
 
 import time

changedetectionio/tests/test_nonrenderable_pages.py

@@ -1,12 +1,8 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
-import time
 from flask import url_for
-from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup
-
-sleep_time_for_fetch_thread = 3
-
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+import time
 
 def set_nonrenderable_response():
     test_return_data = """<html>
@@ -22,6 +18,13 @@ def set_nonrenderable_response():
 
     return None
 
+def set_zero_byte_response():
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("")
+
+    return None
+
 def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage):
     set_original_response()
     live_server_setup(live_server)
@@ -35,18 +38,11 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
 
     assert b"1 Imported" in res.data
 
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
-    # Do this a few times.. ensures we dont accidently set the status
-    for n in range(3):
-        client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
-        # Give the thread time to pick it up
-        time.sleep(sleep_time_for_fetch_thread)
-
-        # It should report nothing found (no new 'unviewed' class)
-        res = client.get(url_for("index"))
-        assert b'unviewed' not in res.data
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
 
 
     #####################
@@ -64,7 +60,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -86,14 +82,21 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
+    client.get(url_for("mark_all_viewed"), follow_redirects=True)
 
-
-
+    # A totally zero byte (#2528) response should also not trigger an error
+    set_zero_byte_response()
+    time.sleep(2)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data # A change should have registered because empty_pages_are_a_change is ON
+    assert b'fetch-error' not in res.data
 
     #
     # Cleanup everything

changedetectionio/tests/test_notification.py

@@ -3,6 +3,8 @@ import os
 import time
 import re
 from flask import url_for
+from loguru import logger
+
 from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
     set_longer_modified_response
 from . util import  extract_UUID_from_client
@@ -289,11 +291,11 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
         data={
               "application-fetch_backend": "html_requests",
               "application-minutes_between_check": 180,
-              "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }',
+              "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444, "somebug": "网站监测 内容更新了" }',
               "application-notification_format": default_notification_format,
               "application-notification_urls": test_notification_url,
               # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-              "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
+              "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }} ",
               },
         follow_redirects=True
     )
@@ -322,6 +324,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
         j = json.loads(x)
         assert j['url'].startswith('http://localhost')
         assert j['secret'] == 444
+        assert j['somebug'] == '网站监测 内容更新了'
 
     # URL check, this will always be converted to lowercase
     assert os.path.isfile("test-datastore/notification-url.txt")
@@ -347,3 +350,82 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
         url_for("form_delete", uuid="all"),
         follow_redirects=True
     )
+
+
+#2510
+def test_global_send_test_notification(client, live_server, measure_memory_usage):
+
+    #live_server_setup(live_server)
+    set_original_response()
+    if os.path.isfile("test-datastore/notification.txt"):
+        os.unlink("test-datastore/notification.txt")
+
+    # otherwise other settings would have already existed from previous tests in this file
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "application-fetch_backend": "html_requests",
+            "application-minutes_between_check": 180,
+            #1995 UTF-8 content should be encoded
+            "application-notification_body": 'change detection is cool 网站监测 内容更新了',
+            "application-notification_format": default_notification_format,
+            "application-notification_urls": "",
+            "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
+        },
+        follow_redirects=True
+    )
+    assert b'Settings updated' in res.data
+
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": 'nice one'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
+
+    ######### Test global/system settings
+    res = client.post(
+        url_for("ajax_callback_send_notification_test")+"?mode=global-settings",
+        data={"notification_urls": test_notification_url},
+        follow_redirects=True
+    )
+
+    assert res.status_code != 400
+    assert res.status_code != 500
+
+    # Give apprise time to fire
+    time.sleep(4)
+
+    with open("test-datastore/notification.txt", 'r') as f:
+        x = f.read()
+        assert 'change detection is cool 网站监测 内容更新了' in x
+
+    os.unlink("test-datastore/notification.txt")
+
+    ######### Test group/tag settings
+    res = client.post(
+        url_for("ajax_callback_send_notification_test")+"?mode=group-settings",
+        data={"notification_urls": test_notification_url},
+        follow_redirects=True
+    )
+
+    assert res.status_code != 400
+    assert res.status_code != 500
+
+    # Give apprise time to fire
+    time.sleep(4)
+
+    with open("test-datastore/notification.txt", 'r') as f:
+        x = f.read()
+        # Should come from notification.py default handler when there is no notification body to pull from
+        assert 'change detection is cool 网站监测 内容更新了' in x
+
+    client.get(
+        url_for("form_delete", uuid="all"),
+        follow_redirects=True
+    )
+

changedetectionio/tests/test_obfuscations.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_pdf.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_restock_itemprop.py

@@ -1,8 +1,10 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
+import os
 import time
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
+from ..notification import default_notification_format
 
 instock_props = [
     # LD+JSON with non-standard list of 'type' https://github.com/dgtlmoon/changedetection.io/issues/1833
@@ -52,6 +54,8 @@ def test_restock_itemprop_basic(client, live_server):
 
     test_url = url_for('test_endpoint', _external=True)
 
+    # By default it should enable ('in_stock_processing') == 'all_changes'
+
     for p in instock_props:
         set_original_response(props_markup=p)
         client.post(
@@ -87,6 +91,7 @@ def test_restock_itemprop_basic(client, live_server):
 def test_itemprop_price_change(client, live_server):
     #live_server_setup(live_server)
 
+    # Out of the box 'Follow price changes' should be ON
     test_url = url_for('test_endpoint', _external=True)
 
     set_original_response(props_markup=instock_props[0], price="190.95")
@@ -114,7 +119,7 @@ def test_itemprop_price_change(client, live_server):
     set_original_response(props_markup=instock_props[0], price='120.45')
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"restock_settings-follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -128,8 +133,7 @@ def test_itemprop_price_change(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_itemprop_price_minmax_limit(client, live_server):
-    #live_server_setup(live_server)
+def _run_test_minmax_limit(client, extra_watch_edit_form):
 
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
@@ -146,17 +150,16 @@ def test_itemprop_price_minmax_limit(client, live_server):
     # A change in price, should trigger a change by default
     wait_for_all_checks(client)
 
-
+    data = {
+        "tags": "",
+        "url": test_url,
+        "headers": "",
+        'fetch_backend': "html_requests"
+    }
+    data.update(extra_watch_edit_form)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"follow_price_changes": "y",
-              "price_change_min": 900.0,
-              "price_change_max": 1100.10,
-              "url": test_url,
-              "tags": "",
-              "headers": "",
-              'fetch_backend': "html_requests"
-              },
+        data=data,
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -164,7 +167,7 @@ def test_itemprop_price_minmax_limit(client, live_server):
 
     client.get(url_for("mark_all_viewed"))
 
-    # price changed to something greater than min (900), and less than max (1100).. should be no change
+    # price changed to something greater than min (900), BUT less than max (1100).. should be no change
     set_original_response(props_markup=instock_props[0], price='1000.45')
     client.get(url_for("form_watch_checknow"))
     wait_for_all_checks(client)
@@ -201,6 +204,44 @@ def test_itemprop_price_minmax_limit(client, live_server):
     assert b'Deleted' in res.data
 
 
+def test_restock_itemprop_minmax(client, live_server):
+#    live_server_setup(live_server)
+    extras = {
+        "restock_settings-follow_price_changes": "y",
+        "restock_settings-price_change_min": 900.0,
+        "restock_settings-price_change_max": 1100.10
+    }
+    _run_test_minmax_limit(client, extra_watch_edit_form=extras)
+
+def test_restock_itemprop_with_tag(client, live_server):
+    #live_server_setup(live_server)
+
+    res = client.post(
+        url_for("tags.form_tag_add"),
+        data={"name": "test-tag"},
+        follow_redirects=True
+    )
+    assert b"Tag added" in res.data
+
+    res = client.post(
+        url_for("tags.form_tag_edit_submit", uuid="first"),
+        data={"name": "test-tag",
+              "restock_settings-follow_price_changes": "y",
+              "restock_settings-price_change_min": 900.0,
+              "restock_settings-price_change_max": 1100.10,
+              "overrides_watch": "y", #overrides_watch should be restock_overrides_watch
+              },
+        follow_redirects=True
+    )
+
+    extras = {
+        "tags": "test-tag"
+    }
+
+    _run_test_minmax_limit(client, extra_watch_edit_form=extras)
+
+
+
 def test_itemprop_percent_threshold(client, live_server):
     #live_server_setup(live_server)
 
@@ -221,8 +262,8 @@ def test_itemprop_percent_threshold(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"follow_price_changes": "y",
-              "price_change_threshold_percent": 5.0,
+        data={"restock_settings-follow_price_changes": "y",
+              "restock_settings-price_change_threshold_percent": 5.0,
               "url": test_url,
               "tags": "",
               "headers": "",
@@ -266,6 +307,70 @@ def test_itemprop_percent_threshold(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
+
+
+def test_change_with_notification_values(client, live_server):
+    #live_server_setup(live_server)
+
+    if os.path.isfile("test-datastore/notification.txt"):
+        os.unlink("test-datastore/notification.txt")
+
+    test_url = url_for('test_endpoint', _external=True)
+    set_original_response(props_markup=instock_props[0], price='960.45')
+
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
+
+    ######################
+    # You must add a type of 'restock_diff' for its tokens to register as valid in the global settings
+    client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
+        follow_redirects=True
+    )
+
+    # A change in price, should trigger a change by default
+    wait_for_all_checks(client)
+
+    # Should see new tokens register
+    res = client.get(url_for("settings_page"))
+    assert b'{{restock.original_price}}' in res.data
+    assert b'Original price at first check' in res.data
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "title new price {{restock.price}}",
+              "application-notification_body": "new price {{restock.price}}",
+              "application-notification_format": default_notification_format,
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    # check tag accepts without error
+
+    # Check the watches in these modes add the tokens for validating
+    assert b"A variable or function is not defined" not in res.data
+
+    assert b"Settings updated." in res.data
+
+
+    set_original_response(props_markup=instock_props[0], price='960.45')
+    # A change in price, should trigger a change by default
+    set_original_response(props_markup=instock_props[0], price='1950.45')
+    client.get(url_for("form_watch_checknow"))
+    wait_for_all_checks(client)
+    time.sleep(3)
+    assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
+    with open("test-datastore/notification.txt", 'r') as f:
+        notification = f.read()
+        assert "new price 1950.45" in notification
+        assert "title new price 1950.45" in notification
+
+
+
 def test_data_sanity(client, live_server):
     #live_server_setup(live_server)
 

changedetectionio/tests/test_rss.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_security.py

@@ -1,7 +1,12 @@
+import os
+
 from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 import time
 
+from .. import strtobool
+
+
 def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
@@ -55,17 +60,33 @@ def test_bad_access(client, live_server, measure_memory_usage):
 
     assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
 
-    # file:// is permitted by default, but it will be caught by ALLOW_FILE_URI
 
+def test_file_access(client, live_server, measure_memory_usage):
+    #live_server_setup(live_server)
+
+    test_file_path = "/tmp/test-file.txt"
+
+    # file:// is permitted by default, but it will be caught by ALLOW_FILE_URI
     client.post(
         url_for("form_quick_watch_add"),
-        data={"url": 'file:///tasty/disk/drive', "tags": ''},
+        data={"url": f"file://{test_file_path}", "tags": ''},
         follow_redirects=True
     )
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
 
-    assert b'file:// type access is denied for security reasons.' in res.data
+    # If it is enabled at test time
+    if strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
+        res = client.get(
+            url_for("preview_page", uuid="first"),
+            follow_redirects=True
+        )
+
+        # Should see something (this file added by run_basic_tests.sh)
+        assert b"Hello world" in res.data
+    else:
+        # Default should be here
+        assert b'file:// type access is denied for security reasons.' in res.data
 
 def test_xss(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)

changedetectionio/tests/test_share_watch.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_source.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_trigger.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_trigger_regex.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_trigger_regex_with_filter.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/test_unique_lines.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import time
 from flask import url_for

changedetectionio/tests/unit/test_jinja2_security.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # run from dir above changedetectionio/ dir
 # python3 -m unittest changedetectionio.tests.unit.test_jinja2_security

changedetectionio/tests/unit/test_notification_diff.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # run from dir above changedetectionio/ dir
 # python3 -m unittest changedetectionio.tests.unit.test_notification_diff

changedetectionio/tests/unit/test_restock_logic.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # run from dir above changedetectionio/ dir
 # python3 -m unittest changedetectionio.tests.unit.test_restock_logic

changedetectionio/tests/unit/test_watch_model.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # run from dir above changedetectionio/ dir
 # python3 -m unittest changedetectionio.tests.unit.test_notification_diff

changedetectionio/tests/util.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from flask import make_response, request
 from flask import url_for

changedetectionio/tests/visualselector/conftest.py

@@ -1,3 +1,3 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 from .. import conftest

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import os
 from flask import url_for

changedetectionio/update_worker.py

@@ -1,6 +1,5 @@
 from .processors.exceptions import ProcessorException
-from . import content_fetchers
-
+import changedetectionio.content_fetchers.exceptions as content_fetchers_exceptions
 from changedetectionio.processors.text_json_diff.processor import FilterNotFoundInResponse
 from changedetectionio import html_tools
 
@@ -81,6 +80,9 @@ class update_worker(threading.Thread):
             'uuid': watch.get('uuid') if watch else None,
             'watch_url': watch.get('url') if watch else None,
         })
+
+        n_object.update(watch.extra_notification_token_values())
+
         logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
         logger.debug("Queued notification for sending")
         notification_q.put(n_object)
@@ -298,7 +300,7 @@ class update_worker(threading.Thread):
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': e.message})
                         process_changedetection_results = False
 
-                    except content_fetchers.exceptions.ReplyWithContentButNoText as e:
+                    except content_fetchers_exceptions.ReplyWithContentButNoText as e:
                         # Totally fine, it's by choice - just continue on, nothing more to care about
                         # Page had elements/content but no renderable text
                         # Backend (not filters) gave zero output
@@ -324,7 +326,7 @@ class update_worker(threading.Thread):
                             
                         process_changedetection_results = False
 
-                    except content_fetchers.exceptions.Non200ErrorCodeReceived as e:
+                    except content_fetchers_exceptions.Non200ErrorCodeReceived as e:
                         if e.status_code == 403:
                             err_text = "Error - 403 (Access denied) received"
                         elif e.status_code == 404:
@@ -377,23 +379,23 @@ class update_worker(threading.Thread):
 
                         process_changedetection_results = False
 
-                    except content_fetchers.exceptions.checksumFromPreviousCheckWasTheSame as e:
+                    except content_fetchers_exceptions.checksumFromPreviousCheckWasTheSame as e:
                         # Yes fine, so nothing todo, don't continue to process.
                         process_changedetection_results = False
                         changed_detected = False
-                    except content_fetchers.exceptions.BrowserConnectError as e:
+                    except content_fetchers_exceptions.BrowserConnectError as e:
                         self.datastore.update_watch(uuid=uuid,
                                                     update_obj={'last_error': e.msg
                                                                 }
                                                     )
                         process_changedetection_results = False
-                    except content_fetchers.exceptions.BrowserFetchTimedOut as e:
+                    except content_fetchers_exceptions.BrowserFetchTimedOut as e:
                         self.datastore.update_watch(uuid=uuid,
                                                     update_obj={'last_error': e.msg
                                                                 }
                                                     )
                         process_changedetection_results = False
-                    except content_fetchers.exceptions.BrowserStepsStepException as e:
+                    except content_fetchers_exceptions.BrowserStepsStepException as e:
 
                         if not self.datastore.data['watching'].get(uuid):
                             continue
@@ -435,25 +437,25 @@ class update_worker(threading.Thread):
 
                         process_changedetection_results = False
 
-                    except content_fetchers.exceptions.EmptyReply as e:
+                    except content_fetchers_exceptions.EmptyReply as e:
                         # Some kind of custom to-str handler in the exception handler that does this?
                         err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
                         process_changedetection_results = False
-                    except content_fetchers.exceptions.ScreenshotUnavailable as e:
+                    except content_fetchers_exceptions.ScreenshotUnavailable as e:
                         err_text = "Screenshot unavailable, page did not render fully in the expected time or page was too long - try increasing 'Wait seconds before extracting text'"
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
                         process_changedetection_results = False
-                    except content_fetchers.exceptions.JSActionExceptions as e:
+                    except content_fetchers_exceptions.JSActionExceptions as e:
                         err_text = "Error running JS Actions - Page request - "+e.message
                         if e.screenshot:
                             watch.save_screenshot(screenshot=e.screenshot, as_error=True)
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            'last_check_status': e.status_code})
                         process_changedetection_results = False
-                    except content_fetchers.exceptions.PageUnloadable as e:
+                    except content_fetchers_exceptions.PageUnloadable as e:
                         err_text = "Page request from server didnt respond correctly"
                         if e.message:
                             err_text = "{} - {}".format(err_text, e.message)
@@ -465,7 +467,7 @@ class update_worker(threading.Thread):
                                                                            'last_check_status': e.status_code,
                                                                            'has_ldjson_price_data': None})
                         process_changedetection_results = False
-                    except content_fetchers.exceptions.BrowserStepsInUnsupportedFetcher as e:
+                    except content_fetchers_exceptions.BrowserStepsInUnsupportedFetcher as e:
                         err_text = "This watch has Browser Steps configured and so it cannot run with the 'Basic fast Plaintext/HTTP Client', either remove the Browser Steps or select a Chrome fetcher."
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
                         process_changedetection_results = False

docker-compose.yml

@@ -18,7 +18,7 @@ services:
   #
   #        Log levels are in descending order. (TRACE is the most detailed one)
   #        Log output levels: TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL
-  #      - LOGGER_LEVEL=DEBUG
+  #      - LOGGER_LEVEL=TRACE
   #
   #       Alternative WebDriver/selenium URL, do not use "'s or 's!
   #      - WEBDRIVER_URL=http://browser-chrome:4444/wd/hub
@@ -29,8 +29,9 @@ services:
   #
   #             https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.proxy
   #
-  #       Alternative Playwright URL, do not use "'s or 's!
-  #      - PLAYWRIGHT_DRIVER_URL=ws://playwright-chrome:3000
+  #       Alternative target "Chrome" Playwright URL, do not use "'s or 's!
+  #       "Playwright" is a driver/librarythat allows changedetection to talk to a Chrome or similar browser.
+  #      - PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000
   #
   #       Playwright proxy settings playwright_proxy_server, playwright_proxy_bypass, playwright_proxy_username, playwright_proxy_password
   #
@@ -57,6 +58,10 @@ services:
   #
   #        Absolute minimum seconds to recheck, overrides any watch minimum, change to 0 to disable
   #      - MINIMUM_SECONDS_RECHECK_TIME=3
+  #
+  #        Scrape prices from web pages automatically where the page has no embedded price information (see below also)
+  #      - PRICE_SCRAPER_ML_ENDPOINT=http://cdio-price-scraper:5005
+
       # Comment out ports: when using behind a reverse proxy , enable networks: etc.
       ports:
         - 5000:5000
@@ -73,10 +78,10 @@ services:
 #              condition: service_started
 
 
-     # Used for fetching pages via Playwright+Chrome where you need Javascript support.
+     # Sockpuppetbrowser is basically chrome wrapped in an API for allowing fast fetching of web-pages.
      # RECOMMENDED FOR FETCHING PAGES WITH CHROME
-#    playwright-chrome:
-#        hostname: playwright-chrome
+#    sockpuppetbrowser:
+#        hostname: sockpuppetbrowser
 #        image: dgtlmoon/sockpuppetbrowser:latest
 #        cap_add:
 #            - SYS_ADMIN
@@ -103,6 +108,13 @@ services:
 #            # Workaround to avoid the browser crashing inside a docker container
 #            # See https://github.com/SeleniumHQ/docker-selenium#quick-start
 #            - /dev/shm:/dev/shm
+#        restart: unless-stopped
+
+     # Machine Learning/AI - Use "Visual Selector" elements data to scrape price data
+
+#    cdio-keras-price-scraper:
+#        hostname: cdio-price-scraper
+#        image: dgtlmoon/changedetection-AI-pricescraper
 #        restart: unless-stopped
 
 volumes: