Merge branch 'master' into fix-file-access

.github/workflows/containers.yml

@@ -95,7 +95,7 @@ jobs:
           push: true
           tags: |
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
@@ -116,7 +116,7 @@ jobs:
             ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
             ghcr.io/dgtlmoon/changedetection.io:latest
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
 # Looks like this was disabled

.github/workflows/test-container-build.yml

@@ -64,7 +64,7 @@ jobs:
           with:
             context: ./
             file: ./Dockerfile
-            platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
+            platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
             cache-from: type=local,src=/tmp/.buildx-cache
             cache-to: type=local,dest=/tmp/.buildx-cache
 

README.md

@@ -41,6 +41,20 @@ Using the **Browser Steps** configuration, add basic steps before performing cha
 After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
 Requires Playwright to be enabled.
 
+### Awesome restock and price change notifications
+
+Enable the _"Re-stock & Price detection for single product pages"_ option to activate the best way to monitor product pricing.
+
+Easily organise and monitor prices for products from the dashboard, get alerts and notifications when the price of a product changes or comes back in stock again!
+
+[<img src="docs/restock-overview.png" style="max-width:100%;" alt="Easily keep an eye on product price changes directly from the UI"  title="Easily keep an eye on product price changes directly from the UI" />](https://changedetection.io?src=github)
+
+Set price change notification parameters, upper and lower price, price change percentage and more.
+Always know when a product for sale drops in price.
+
+[<img src="docs/restock-settings.png" style="max-width:100%;" alt="Set upper lower and percentage price change notification values"  title="Set upper lower and percentage price change notification values" />](https://changedetection.io?src=github)
+
+
 
 ### Example use cases
 

changedetectionio/blueprint/tags/__init__.py

@@ -1,4 +1,6 @@
-from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
+from flask import Blueprint, request, render_template, flash, url_for, redirect
+
+
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.flask_app import login_optionally_required
 
@@ -96,22 +98,55 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
     @login_optionally_required
     def form_tag_edit(uuid):
-        from changedetectionio import forms
-
+        from changedetectionio.blueprint.tags.form import group_restock_settings_form
         if uuid == 'first':
             uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
 
         default = datastore.data['settings']['application']['tags'].get(uuid)
 
-        form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
-                               data=default,
-                               )
-        form.datastore=datastore # needed?
+        form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
+                                       data=default,
+                                       extra_notification_tokens=datastore.get_unique_notification_tokens_available()
+                                       )
+
+        template_args = {
+            'data': default,
+            'form': form,
+            'watch': default,
+            'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
+        }
+
+        included_content = {}
+        if form.extra_form_content():
+            # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/
+            # And then render the code from the module
+            from jinja2 import Environment, FileSystemLoader
+            import importlib.resources
+            templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates'))
+            env = Environment(loader=FileSystemLoader(templates_dir))
+            template_str = """{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
+        <script>        
+            $(document).ready(function () {
+                toggleOpacity('#overrides_watch', '#restock-fieldset-price-group', true);
+            });
+        </script>            
+                <fieldset>
+                    <div class="pure-control-group">
+                        <fieldset class="pure-group">
+                        {{ render_checkbox_field(form.overrides_watch) }}
+                        <span class="pure-form-message-inline">Used for watches in "Restock & Price detection" mode</span>
+                        </fieldset>
+                </fieldset>
+                """
+            template_str += form.extra_form_content()
+            template = env.from_string(template_str)
+            included_content = template.render(**template_args)
 
         output = render_template("edit-tag.html",
-                                 data=default,
-                                 form=form,
                                  settings_application=datastore.data['settings']['application'],
+                                 extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,
+                                 extra_form_content=included_content,
+                                 **template_args
                                  )
 
         return output
@@ -120,14 +155,15 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
     @login_optionally_required
     def form_tag_edit_submit(uuid):
-        from changedetectionio import forms
+        from changedetectionio.blueprint.tags.form import group_restock_settings_form
         if uuid == 'first':
             uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
 
         default = datastore.data['settings']['application']['tags'].get(uuid)
 
-        form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
+        form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
                                data=default,
+                               extra_notification_tokens=datastore.get_unique_notification_tokens_available()
                                )
         # @todo subclass form so validation works
         #if not form.validate():
@@ -136,6 +172,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 #           return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
 
         datastore.data['settings']['application']['tags'][uuid].update(form.data)
+        datastore.data['settings']['application']['tags'][uuid]['processor'] = 'restock_diff'
         datastore.needs_write_urgent = True
         flash("Updated")
 

changedetectionio/blueprint/tags/form.py

@@ -1,16 +1,15 @@
 from wtforms import (
-    BooleanField,
     Form,
-    IntegerField,
-    RadioField,
-    SelectField,
     StringField,
     SubmitField,
-    TextAreaField,
     validators,
 )
+from wtforms.fields.simple import BooleanField
 
+from changedetectionio.processors.restock_diff.forms import processor_settings_form as restock_settings_form
 
+class group_restock_settings_form(restock_settings_form):
+    overrides_watch = BooleanField('Activate for individual watches in this tag/group?', default=False)
 
 class SingleTag(Form):
 

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -26,6 +26,9 @@
         <ul>
             <li class="tab" id=""><a href="#general">General</a></li>
             <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
+            {% if extra_tab_content %}
+            <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
+            {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
         </ul>
     </div>
@@ -97,6 +100,12 @@ nav
 
             </div>
 
+        {# rendered sub Template #}
+        {% if extra_form_content %}
+            <div class="tab-pane-inner" id="extras_tab">
+            {{ extra_form_content|safe }}
+            </div>
+        {% endif %}
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div  class="pure-control-group inline-radio">
@@ -119,7 +128,7 @@ nav
                         {% endif %}
                         <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
 
-                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
+                        {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
                     </div>
                 </fieldset>
             </div>

changedetectionio/content_fetchers/requests.py

@@ -4,6 +4,7 @@ import os
 import chardet
 import requests
 
+from changedetectionio import strtobool
 from changedetectionio.content_fetchers.exceptions import BrowserStepsInUnsupportedFetcher, EmptyReply, Non200ErrorCodeReceived
 from changedetectionio.content_fetchers.base import Fetcher
 
@@ -45,13 +46,19 @@ class fetcher(Fetcher):
             if self.system_https_proxy:
                 proxies['https'] = self.system_https_proxy
 
-        r = requests.request(method=request_method,
-                             data=request_body,
-                             url=url,
-                             headers=request_headers,
-                             timeout=timeout,
-                             proxies=proxies,
-                             verify=False)
+        session = requests.Session()
+
+        if strtobool(os.getenv('ALLOW_FILE_URI', 'false')) and url.startswith('file://'):
+            from requests_file import FileAdapter
+            session.mount('file://', FileAdapter())
+
+        r = session.request(method=request_method,
+                            data=request_body,
+                            url=url,
+                            headers=request_headers,
+                            timeout=timeout,
+                            proxies=proxies,
+                            verify=False)
 
         # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
         # For example - some sites don't tell us it's utf-8, but return utf-8 content

changedetectionio/flask_app.py

@@ -696,11 +696,16 @@ def changedetection_app(config=None, datastore_o=None):
             form_class = forms.processor_text_json_diff_form
 
         form = form_class(formdata=request.form if request.method == 'POST' else None,
-                               data=default
-                               )
+                          data=default,
+                          extra_notification_tokens=default.extra_notification_token_values()
+                          )
 
-        # For the form widget tag uuid lookup
-        form.tags.datastore = datastore # in _value
+        # For the form widget tag UUID back to "string name" for the field
+        form.tags.datastore = datastore
+
+        # Used by some forms that need to dig deeper
+        form.datastore = datastore
+        form.watch = default
 
         for p in datastore.extra_browsers:
             form.fetch_backend.choices.append(p)
@@ -766,6 +771,11 @@ def changedetection_app(config=None, datastore_o=None):
             datastore.data['watching'][uuid].update(form.data)
             datastore.data['watching'][uuid].update(extra_update_obj)
 
+            if not datastore.data['watching'][uuid].get('tags'):
+                # Force it to be a list, because form.data['tags'] will be string if nothing found
+                # And del(form.data['tags'] ) wont work either for some reason
+                datastore.data['watching'][uuid]['tags'] = []
+
             # Recast it if need be to right data Watch handler
             watch_class = get_custom_watch_obj_for_processor(form.data.get('processor'))
             datastore.data['watching'][uuid] = watch_class(datastore_path=datastore_o.datastore_path, default=datastore.data['watching'][uuid])
@@ -815,6 +825,7 @@ def changedetection_app(config=None, datastore_o=None):
                 'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
                 'extra_title': f" - Edit - {watch.label}",
                 'extra_processor_config': form.extra_tab_content(),
+                'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
                 'form': form,
                 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,
                 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
@@ -869,7 +880,8 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status
         form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,
-                                        data=default
+                                        data=default,
+                                        extra_notification_tokens=datastore.get_unique_notification_tokens_available()
                                         )
 
         # Remove the last option 'System default'
@@ -921,6 +933,7 @@ def changedetection_app(config=None, datastore_o=None):
         output = render_template("settings.html",
                                  api_key=datastore.data['settings']['application'].get('api_access_token'),
                                  emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
+                                 extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(),
                                  form=form,
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
                                  min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
@@ -1548,9 +1561,13 @@ def changedetection_app(config=None, datastore_o=None):
                     for uuid in uuids:
                         uuid = uuid.strip()
                         if datastore.data['watching'].get(uuid):
+                            # Bug in old versions caused by bad edit page/tag handler
+                            if isinstance(datastore.data['watching'][uuid]['tags'], str):
+                                datastore.data['watching'][uuid]['tags'] = []
+
                             datastore.data['watching'][uuid]['tags'].append(tag_uuid)
 
-            flash("{} watches assigned tag".format(len(uuids)))
+            flash(f"{len(uuids)} watches were tagged")
 
         return redirect(url_for('index'))
 

changedetectionio/forms.py

@@ -231,9 +231,6 @@ class ValidateJinja2Template(object):
     """
     Validates that a {token} is from a valid set
     """
-    def __init__(self, message=None):
-        self.message = message
-
     def __call__(self, form, field):
         from changedetectionio import notification
 
@@ -248,6 +245,10 @@ class ValidateJinja2Template(object):
         try:
             jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader)
             jinja2_env.globals.update(notification.valid_tokens)
+            # Extra validation tokens provided on the form_class(... extra_tokens={}) setup
+            if hasattr(field, 'extra_notification_tokens'):
+                jinja2_env.globals.update(field.extra_notification_tokens)
+
             jinja2_env.from_string(joined_data).render()
         except TemplateSyntaxError as e:
             raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
@@ -422,6 +423,12 @@ class quickWatchForm(Form):
 class commonSettingsForm(Form):
     from . import processors
 
+    def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs):
+        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
+        self.notification_body.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
+        self.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
+        self.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
+
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
     fetch_backend = RadioField(u'Fetch Method', choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
@@ -429,8 +436,8 @@ class commonSettingsForm(Form):
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
     processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff")
-    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1,
-                                                                                                                                    message="Should contain one or more seconds")])
+    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")])
+
 
 class importForm(Form):
     from . import processors
@@ -590,6 +597,11 @@ class globalSettingsForm(Form):
     # Define these as FormFields/"sub forms", this way it matches the JSON storage
     # datastore.data['settings']['application']..
     # datastore.data['settings']['requests']..
+    def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs):
+        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
+        self.application.notification_body.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
+        self.application.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
+        self.application.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
 
     requests = FormField(globalSettingsRequestForm)
     application = FormField(globalSettingsApplicationForm)

changedetectionio/model/Tag.py

@@ -1,14 +1,14 @@
 
 from changedetectionio.model import watch_base
 
+
 class model(watch_base):
 
     def __init__(self, *arg, **kw):
-
         super(model, self).__init__(*arg, **kw)
 
+        self['overrides_watch'] = kw.get('default', {}).get('overrides_watch')
+
         if kw.get('default'):
             self.update(kw['default'])
             del kw['default']
-
-

changedetectionio/model/Watch.py

@@ -432,6 +432,17 @@ class model(watch_base):
     def toggle_mute(self):
         self['notification_muted'] ^= True
 
+    def extra_notification_token_values(self):
+        # Used for providing extra tokens
+        # return {'widget': 555}
+        return {}
+
+    def extra_notification_token_placeholder_info(self):
+        # Used for providing extra tokens
+        # return [('widget', "Get widget amounts")]
+        return []
+
+
     def extract_regex_from_all_history(self, regex):
         import csv
         import re

changedetectionio/notification.py

@@ -272,19 +272,18 @@ def create_notification_parameters(n_object, datastore):
     tokens.update(
         {
             'base_url': base_url,
-            'current_snapshot': n_object.get('current_snapshot', ''),
-            'diff': n_object.get('diff', ''),  # Null default in the case we use a test
-            'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test
-            'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
-            'diff_patch': n_object.get('diff_patch', ''),  # Null default in the case we use a test
-            'diff_removed': n_object.get('diff_removed', ''),  # Null default in the case we use a test
             'diff_url': diff_url,
             'preview_url': preview_url,
-            'triggered_text': n_object.get('triggered_text', ''),
             'watch_tag': watch_tag if watch_tag is not None else '',
             'watch_title': watch_title if watch_title is not None else '',
             'watch_url': watch_url,
             'watch_uuid': uuid,
         })
 
+    # n_object will contain diff, diff_added etc etc
+    tokens.update(n_object)
+
+    if uuid:
+        tokens.update(datastore.data['watching'].get(uuid).extra_notification_token_values())
+
     return tokens

changedetectionio/processors/restock_diff/__init__.py

@@ -59,7 +59,25 @@ class Watch(BaseWatch):
         super().__init__(*arg, **kw)
         self['restock'] = Restock(kw['default']['restock']) if kw.get('default') and kw['default'].get('restock') else Restock()
 
+        self['restock_settings'] = kw['default']['restock_settings'] if kw.get('default',{}).get('restock_settings') else {
+            'follow_price_changes': True,
+            'in_stock_processing' : 'in_stock_only'
+        } #@todo update
+
     def clear_watch(self):
         super().clear_watch()
         self.update({'restock': Restock()})
 
+    def extra_notification_token_values(self):
+        values = super().extra_notification_token_values()
+        values['restock'] = self.get('restock', {})
+        return values
+
+    def extra_notification_token_placeholder_info(self):
+        values = super().extra_notification_token_placeholder_info()
+
+        values.append(('restock.price', "Price detected"))
+        values.append(('restock.original_price', "Original price at first check"))
+
+        return values
+

changedetectionio/processors/restock_diff/forms.py

@@ -1,17 +1,25 @@
-
 from wtforms import (
     BooleanField,
     validators,
     FloatField
 )
+from wtforms.fields.choices import RadioField
+from wtforms.fields.form import FormField
+from wtforms.form import Form
 
 from changedetectionio.forms import processor_text_json_diff_form
 
-class processor_settings_form(processor_text_json_diff_form):
-    in_stock_only = BooleanField('Only trigger when product goes BACK to in-stock', default=True)
-    price_change_min = FloatField('Minimum amount to trigger notification', [validators.Optional()],
+
+class RestockSettingsForm(Form):
+    in_stock_processing = RadioField(label='Re-stock detection', choices=[
+        ('in_stock_only', "In Stock only (Out Of Stock -> In Stock only)"),
+        ('all_changes', "Any availability changes"),
+        ('off', "Off, don't follow availability/restock"),
+    ], default="in_stock_only")
+
+    price_change_min = FloatField('Below price to trigger notification', [validators.Optional()],
                                   render_kw={"placeholder": "No limit", "size": "10"})
-    price_change_max = FloatField('Maximum amount to trigger notification', [validators.Optional()],
+    price_change_max = FloatField('Above price to trigger notification', [validators.Optional()],
                                   render_kw={"placeholder": "No limit", "size": "10"})
     price_change_threshold_percent = FloatField('Threshold in % for price changes since the original price', validators=[
 
@@ -19,45 +27,55 @@ class processor_settings_form(processor_text_json_diff_form):
         validators.NumberRange(min=0, max=100, message="Should be between 0 and 100"),
     ], render_kw={"placeholder": "0%", "size": "5"})
 
-    follow_price_changes = BooleanField('Follow price changes', default=False)
+    follow_price_changes = BooleanField('Follow price changes', default=True)
+
+class processor_settings_form(processor_text_json_diff_form):
+    restock_settings = FormField(RestockSettingsForm)
 
     def extra_tab_content(self):
         return 'Restock & Price Detection'
 
     def extra_form_content(self):
-        return """
+        output = ""
+
+        if getattr(self, 'watch', None) and getattr(self, 'datastore'):
+            for tag_uuid in self.watch.get('tags'):
+                tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {})
+                if tag.get('overrides_watch'):
+                    # @todo - Quick and dirty, cant access 'url_for' here because its out of scope somehow
+                    output = f"""<p><strong>Note! A Group tag overrides the restock and price detection here.</strong></p><style>#restock-fieldset-price-group {{ opacity: 0.6; }}</style>"""
+
+        output += """
         {% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
         <script>        
             $(document).ready(function () {
-                toggleOpacity('#follow_price_changes', '.price-change-minmax', true);
+                toggleOpacity('#restock_settings-follow_price_changes', '.price-change-minmax', true);
             });
         </script>
 
-
-        <fieldset>
+        <fieldset id="restock-fieldset-price-group">
             <div class="pure-control-group">
-                <fieldset class="pure-group">
-                    {{ render_checkbox_field(form.in_stock_only) }}
-                    <span class="pure-form-message-inline">Only trigger re-stock notification when page changes from <strong>out of stock</strong> to <strong>back in stock</strong></span>
+                <fieldset class="pure-group inline-radio">
+                    {{ render_field(form.restock_settings.in_stock_processing) }}
                 </fieldset>
                 <fieldset class="pure-group">
-                    {{ render_checkbox_field(form.follow_price_changes) }}
+                    {{ render_checkbox_field(form.restock_settings.follow_price_changes) }}
                     <span class="pure-form-message-inline">Changes in price should trigger a notification</span>
-                    <br>
-                    <span class="pure-form-message-inline">When OFF - Only care about restock detection</span>                    
                 </fieldset>
                 <fieldset class="pure-group price-change-minmax">               
-                    {{ render_field(form.price_change_min, placeholder=watch['restock']['price']) }}
-                    <span class="pure-form-message-inline">Minimum amount, only trigger a change when the price is less than this amount.</span>
+                    {{ render_field(form.restock_settings.price_change_min, placeholder=watch.get('restock', {}).get('price')) }}
+                    <span class="pure-form-message-inline">Minimum amount, Trigger a change/notification when the price drops <i>below</i> this value.</span>
                 </fieldset>
                 <fieldset class="pure-group price-change-minmax">
-                    {{ render_field(form.price_change_max, placeholder=watch['restock']['price']) }}
-                    <span class="pure-form-message-inline">Maximum amount, only trigger a change when the price is more than this amount.</span>
+                    {{ render_field(form.restock_settings.price_change_max, placeholder=watch.get('restock', {}).get('price')) }}
+                    <span class="pure-form-message-inline">Maximum amount, Trigger a change/notification when the price rises <i>above</i> this value.</span>
                 </fieldset>
                 <fieldset class="pure-group price-change-minmax">
-                    {{ render_field(form.price_change_threshold_percent) }}
-                    <span class="pure-form-message-inline">Price must change more than this % to trigger a change.</span><br>
+                    {{ render_field(form.restock_settings.price_change_threshold_percent) }}
+                    <span class="pure-form-message-inline">Price must change more than this % to trigger a change since the first check.</span><br>
                     <span class="pure-form-message-inline">For example, If the product is $1,000 USD originally, <strong>2%</strong> would mean it has to change more than $20 since the first check.</span><br>
                 </fieldset>                
             </div>
-        </fieldset>"""
+        </fieldset>
+        """
+        return output

changedetectionio/processors/restock_diff/processor.py

@@ -132,6 +132,18 @@ class perform_site_check(difference_detection_processor):
         update_obj['content_type'] = self.fetcher.headers.get('Content-Type', '')
         update_obj["last_check_status"] = self.fetcher.get_last_status_code()
 
+        # Which restock settings to compare against?
+        restock_settings = watch.get('restock_settings', {})
+
+        # See if any tags have 'activate for individual watches in this tag/group?' enabled and use the first we find
+        for tag_uuid in watch.get('tags'):
+            tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {})
+            if tag.get('overrides_watch'):
+                restock_settings = tag.get('restock_settings', {})
+                logger.info(f"Watch {watch.get('uuid')} - Tag '{tag.get('title')}' selected for restock settings override")
+                break
+
+
         itemprop_availability = {}
         try:
             itemprop_availability = get_itemprop_availability(html_content=self.fetcher.content)
@@ -195,14 +207,14 @@ class perform_site_check(difference_detection_processor):
         # out of stock -> back in stock only?
         if watch.get('restock') and watch['restock'].get('in_stock') != update_obj['restock'].get('in_stock'):
             # Yes if we only care about it going to instock, AND we are in stock
-            if watch.get('in_stock_only') and update_obj['restock']['in_stock']:
+            if restock_settings.get('in_stock_processing') == 'in_stock_only' and update_obj['restock']['in_stock']:
                 changed_detected = True
 
-            if not watch.get('in_stock_only'):
+            if restock_settings.get('in_stock_processing') == 'all_changes':
                 # All cases
                 changed_detected = True
 
-        if watch.get('follow_price_changes') and watch.get('restock') and update_obj.get('restock') and update_obj['restock'].get('price'):
+        if restock_settings.get('follow_price_changes') and watch.get('restock') and update_obj.get('restock') and update_obj['restock'].get('price'):
             price = float(update_obj['restock'].get('price'))
             # Default to current price if no previous price found
             if watch['restock'].get('original_price'):
@@ -214,26 +226,25 @@ class perform_site_check(difference_detection_processor):
             # Minimum/maximum price limit
             if update_obj.get('restock') and update_obj['restock'].get('price'):
                 logger.debug(
-                    f"{watch.get('uuid')} - Change was detected, 'price_change_max' is '{watch.get('price_change_max', '')}' 'price_change_min' is '{watch.get('price_change_min', '')}', price from website is '{update_obj['restock'].get('price', '')}'.")
+                    f"{watch.get('uuid')} - Change was detected, 'price_change_max' is '{restock_settings.get('price_change_max', '')}' 'price_change_min' is '{restock_settings.get('price_change_min', '')}', price from website is '{update_obj['restock'].get('price', '')}'.")
                 if update_obj['restock'].get('price'):
-                    min_limit = float(watch.get('price_change_min')) if watch.get('price_change_min') else None
-                    max_limit = float(watch.get('price_change_max')) if watch.get('price_change_max') else None
+                    min_limit = float(restock_settings.get('price_change_min')) if restock_settings.get('price_change_min') else None
+                    max_limit = float(restock_settings.get('price_change_max')) if restock_settings.get('price_change_max') else None
 
                     price = float(update_obj['restock'].get('price'))
                     logger.debug(f"{watch.get('uuid')} after float conversion - Min limit: '{min_limit}' Max limit: '{max_limit}' Price: '{price}'")
                     if min_limit or max_limit:
                         if is_between(number=price, lower=min_limit, upper=max_limit):
-                            logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}")
-                            if changed_detected:
-                                logger.debug(f"{watch.get('uuid')} Override change-detected to FALSE because price was inside threshold")
-                                changed_detected = False
+                            # Price was between min/max limit, so there was nothing todo in any case
+                            logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}, nothing to check, forcing changed_detected = False (was {changed_detected})")
+                            changed_detected = False
                         else:
-                            logger.trace(f"{watch.get('uuid')} {price} is NOT between {min_limit} and {max_limit}")
+                            logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}, continuing normal comparison")
 
                     # Price comparison by %
-                    if watch['restock'].get('original_price') and changed_detected and watch.get('price_change_threshold_percent'):
+                    if watch['restock'].get('original_price') and changed_detected and restock_settings.get('price_change_threshold_percent'):
                         previous_price = float(watch['restock'].get('original_price'))
-                        pc = float(watch.get('price_change_threshold_percent'))
+                        pc = float(restock_settings.get('price_change_threshold_percent'))
                         change = abs((price - previous_price) / previous_price * 100)
                         if change and change <= pc:
                             logger.debug(f"{watch.get('uuid')} Override change-detected to FALSE because % threshold ({pc}%) was {change:.3f}%")

changedetectionio/run_basic_tests.sh

@@ -35,4 +35,8 @@ pytest tests/test_access_control.py
 pytest tests/test_notification.py
 pytest tests/test_backend.py
 pytest tests/test_rss.py
-pytest tests/test_unique_lines.py
+pytest tests/test_unique_lines.py
+
+# Check file:// will pickup a file when enabled
+echo "Hello world" > /tmp/test-file.txt
+ALLOW_FILE_URI=yes pytest tests/test_security.py

changedetectionio/static/js/watch-settings.js

@@ -27,6 +27,5 @@ $(document).ready(function () {
 
     toggleOpacity('#time_between_check_use_default', '#time_between_check', false);
 
-
 });
 

changedetectionio/store.py

@@ -83,16 +83,14 @@ class ChangeDetectionStore:
                         self.__data['settings']['application'].update(from_disk['settings']['application'])
 
                 # Convert each existing watch back to the Watch.model object
-
                 for uuid, watch in self.__data['watching'].items():
-                    watch['uuid'] = uuid
-                    watch_class = get_custom_watch_obj_for_processor(watch.get('processor'))
-                    if watch.get('uuid') != 'text_json_diff':
-                        logger.trace(f"Loading Watch object '{watch_class.__module__}.{watch_class.__name__}' for UUID {uuid}")
+                    self.__data['watching'][uuid] = self.rehydrate_entity(uuid, watch)
+                    logger.info(f"Watching: {uuid} {watch['url']}")
 
-                    self.__data['watching'][uuid] = watch_class(datastore_path=self.datastore_path, default=watch)
-
-                    logger.info(f"Watching: {uuid} {self.__data['watching'][uuid]['url']}")
+                # And for Tags also, should be Restock type because it has extra settings
+                for uuid, tag in self.__data['settings']['application']['tags'].items():
+                    self.__data['settings']['application']['tags'][uuid] = self.rehydrate_entity(uuid, tag, processor_override='restock_diff')
+                    logger.info(f"Tag: {uuid} {tag['title']}")
 
         # First time ran, Create the datastore.
         except (FileNotFoundError):
@@ -147,6 +145,22 @@ class ChangeDetectionStore:
         # Finally start the thread that will manage periodic data saves to JSON
         save_data_thread = threading.Thread(target=self.save_datastore).start()
 
+    def rehydrate_entity(self, uuid, entity, processor_override=None):
+        """Set the dict back to the dict Watch object"""
+        entity['uuid'] = uuid
+
+        if processor_override:
+            watch_class = get_custom_watch_obj_for_processor(processor_override)
+            entity['processor']=processor_override
+        else:
+            watch_class = get_custom_watch_obj_for_processor(entity.get('processor'))
+
+        if entity.get('uuid') != 'text_json_diff':
+            logger.trace(f"Loading Watch object '{watch_class.__module__}.{watch_class.__name__}' for UUID {uuid}")
+
+        entity = watch_class(datastore_path=self.datastore_path, default=entity)
+        return entity
+
     def set_last_viewed(self, uuid, timestamp):
         logger.debug(f"Setting watch UUID: {uuid} last viewed to {int(timestamp)}")
         self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
@@ -185,6 +199,9 @@ class ChangeDetectionStore:
 
     @property
     def has_unviewed(self):
+        if not self.__data.get('watching'):
+            return None
+
         for uuid, watch in self.__data['watching'].items():
             if watch.history_n >= 2 and watch.viewed == False:
                 return True
@@ -614,6 +631,33 @@ class ChangeDetectionStore:
                 return True
         return False
 
+    def get_unique_notification_tokens_available(self):
+        # Ask each type of watch if they have any extra notification token to add to the validation
+        extra_notification_tokens = {}
+        watch_processors_checked = set()
+
+        for watch_uuid, watch in self.__data['watching'].items():
+            processor = watch.get('processor')
+            if processor not in watch_processors_checked:
+                extra_notification_tokens.update(watch.extra_notification_token_values())
+                watch_processors_checked.add(processor)
+
+        return extra_notification_tokens
+
+    def get_unique_notification_token_placeholders_available(self):
+        # The actual description of the tokens, could be combined with get_unique_notification_tokens_available instead of doing this twice
+        extra_notification_tokens = []
+        watch_processors_checked = set()
+
+        for watch_uuid, watch in self.__data['watching'].items():
+            processor = watch.get('processor')
+            if processor not in watch_processors_checked:
+                extra_notification_tokens+=watch.extra_notification_token_placeholder_info()
+                watch_processors_checked.add(processor)
+
+        return extra_notification_tokens
+
+
     def get_updates_available(self):
         import inspect
         updates_available = []
@@ -850,4 +894,17 @@ class ChangeDetectionStore:
                 watch['restock'] = Restock({'in_stock': watch.get('in_stock')})
                 del watch['in_stock']
 
+    # Migrate old restock settings
+    def update_18(self):
+        for uuid, watch in self.data['watching'].items():
+            if not watch.get('restock_settings'):
+                # So we enable price following by default
+                self.data['watching'][uuid]['restock_settings'] = {'follow_price_changes': True}
+
+            # Migrate and cleanoff old value
+            self.data['watching'][uuid]['restock_settings']['in_stock_processing'] = 'in_stock_only' if watch.get(
+                'in_stock_only') else 'all_changes'
+
+            if self.data['watching'][uuid].get('in_stock_only'):
+                del (self.data['watching'][uuid]['in_stock_only'])
 

changedetectionio/templates/_common_fields.html

@@ -1,7 +1,7 @@
 
 {% from '_helpers.html' import render_field %}
 
-{% macro render_common_settings_form(form, emailprefix, settings_application) %}
+{% macro render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) %}
                         <div class="pure-control-group">
                             {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
     Gitter - gitter://token/room
@@ -107,7 +107,15 @@
                                     <tr>
                                         <td><code>{{ '{{triggered_text}}' }}</code></td>
                                         <td>Text that tripped the trigger from filters</td>
-                                    </tr>
+
+                                        {% if extra_notification_token_placeholder_info %}
+                                            {% for token in extra_notification_token_placeholder_info %}
+                                                <tr>
+                                                    <td><code>{{ '{{' }}{{ token[0] }}{{ '}}' }}</code></td>
+                                                    <td>{{ token[1] }}</td>
+                                                </tr>
+                                            {% endfor %}
+                                        {% endif %}
                                     </tbody>
                                 </table>
                                 <div class="pure-form-message-inline">

changedetectionio/templates/edit.html

@@ -246,7 +246,7 @@ User-Agent: wonderbra 1.0") }}
                         {% endif %}
                         <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
 
-                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
+                        {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
                     </div>
                 </fieldset>
             </div>

changedetectionio/templates/settings.html

@@ -92,7 +92,7 @@
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div class="field-group">
-                        {{ render_common_settings_form(form.application.form, emailprefix, settings_application) }}
+                        {{ render_common_settings_form(form.application.form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
                     </div>
                 </fieldset>
                 <div class="pure-control-group" id="notification-base-url">

changedetectionio/tests/test_restock_itemprop.py

@@ -1,8 +1,10 @@
 #!/usr/bin/python3
+import os
 import time
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
+from ..notification import default_notification_format
 
 instock_props = [
     # LD+JSON with non-standard list of 'type' https://github.com/dgtlmoon/changedetection.io/issues/1833
@@ -52,6 +54,8 @@ def test_restock_itemprop_basic(client, live_server):
 
     test_url = url_for('test_endpoint', _external=True)
 
+    # By default it should enable ('in_stock_processing') == 'all_changes'
+
     for p in instock_props:
         set_original_response(props_markup=p)
         client.post(
@@ -87,6 +91,7 @@ def test_restock_itemprop_basic(client, live_server):
 def test_itemprop_price_change(client, live_server):
     #live_server_setup(live_server)
 
+    # Out of the box 'Follow price changes' should be ON
     test_url = url_for('test_endpoint', _external=True)
 
     set_original_response(props_markup=instock_props[0], price="190.95")
@@ -114,7 +119,7 @@ def test_itemprop_price_change(client, live_server):
     set_original_response(props_markup=instock_props[0], price='120.45')
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"restock_settings-follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -128,8 +133,7 @@ def test_itemprop_price_change(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def test_itemprop_price_minmax_limit(client, live_server):
-    #live_server_setup(live_server)
+def _run_test_minmax_limit(client, extra_watch_edit_form):
 
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
@@ -146,17 +150,16 @@ def test_itemprop_price_minmax_limit(client, live_server):
     # A change in price, should trigger a change by default
     wait_for_all_checks(client)
 
-
+    data = {
+        "tags": "",
+        "url": test_url,
+        "headers": "",
+        'fetch_backend': "html_requests"
+    }
+    data.update(extra_watch_edit_form)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"follow_price_changes": "y",
-              "price_change_min": 900.0,
-              "price_change_max": 1100.10,
-              "url": test_url,
-              "tags": "",
-              "headers": "",
-              'fetch_backend': "html_requests"
-              },
+        data=data,
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -164,7 +167,7 @@ def test_itemprop_price_minmax_limit(client, live_server):
 
     client.get(url_for("mark_all_viewed"))
 
-    # price changed to something greater than min (900), and less than max (1100).. should be no change
+    # price changed to something greater than min (900), BUT less than max (1100).. should be no change
     set_original_response(props_markup=instock_props[0], price='1000.45')
     client.get(url_for("form_watch_checknow"))
     wait_for_all_checks(client)
@@ -201,6 +204,44 @@ def test_itemprop_price_minmax_limit(client, live_server):
     assert b'Deleted' in res.data
 
 
+def test_restock_itemprop_minmax(client, live_server):
+#    live_server_setup(live_server)
+    extras = {
+        "restock_settings-follow_price_changes": "y",
+        "restock_settings-price_change_min": 900.0,
+        "restock_settings-price_change_max": 1100.10
+    }
+    _run_test_minmax_limit(client, extra_watch_edit_form=extras)
+
+def test_restock_itemprop_with_tag(client, live_server):
+    #live_server_setup(live_server)
+
+    res = client.post(
+        url_for("tags.form_tag_add"),
+        data={"name": "test-tag"},
+        follow_redirects=True
+    )
+    assert b"Tag added" in res.data
+
+    res = client.post(
+        url_for("tags.form_tag_edit_submit", uuid="first"),
+        data={"name": "test-tag",
+              "restock_settings-follow_price_changes": "y",
+              "restock_settings-price_change_min": 900.0,
+              "restock_settings-price_change_max": 1100.10,
+              "overrides_watch": "y", #overrides_watch should be restock_overrides_watch
+              },
+        follow_redirects=True
+    )
+
+    extras = {
+        "tags": "test-tag"
+    }
+
+    _run_test_minmax_limit(client, extra_watch_edit_form=extras)
+
+
+
 def test_itemprop_percent_threshold(client, live_server):
     #live_server_setup(live_server)
 
@@ -221,8 +262,8 @@ def test_itemprop_percent_threshold(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"follow_price_changes": "y",
-              "price_change_threshold_percent": 5.0,
+        data={"restock_settings-follow_price_changes": "y",
+              "restock_settings-price_change_threshold_percent": 5.0,
               "url": test_url,
               "tags": "",
               "headers": "",
@@ -266,6 +307,70 @@ def test_itemprop_percent_threshold(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
+
+
+def test_change_with_notification_values(client, live_server):
+    #live_server_setup(live_server)
+
+    if os.path.isfile("test-datastore/notification.txt"):
+        os.unlink("test-datastore/notification.txt")
+
+    test_url = url_for('test_endpoint', _external=True)
+    set_original_response(props_markup=instock_props[0], price='960.45')
+
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
+
+    ######################
+    # You must add a type of 'restock_diff' for its tokens to register as valid in the global settings
+    client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
+        follow_redirects=True
+    )
+
+    # A change in price, should trigger a change by default
+    wait_for_all_checks(client)
+
+    # Should see new tokens register
+    res = client.get(url_for("settings_page"))
+    assert b'{{restock.original_price}}' in res.data
+    assert b'Original price at first check' in res.data
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "title new price {{restock.price}}",
+              "application-notification_body": "new price {{restock.price}}",
+              "application-notification_format": default_notification_format,
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    # check tag accepts without error
+
+    # Check the watches in these modes add the tokens for validating
+    assert b"A variable or function is not defined" not in res.data
+
+    assert b"Settings updated." in res.data
+
+
+    set_original_response(props_markup=instock_props[0], price='960.45')
+    # A change in price, should trigger a change by default
+    set_original_response(props_markup=instock_props[0], price='1950.45')
+    client.get(url_for("form_watch_checknow"))
+    wait_for_all_checks(client)
+    time.sleep(3)
+    assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
+    with open("test-datastore/notification.txt", 'r') as f:
+        notification = f.read()
+        assert "new price 1950.45" in notification
+        assert "title new price 1950.45" in notification
+
+
+
 def test_data_sanity(client, live_server):
     #live_server_setup(live_server)
 

changedetectionio/tests/test_security.py

@@ -1,7 +1,12 @@
+import os
+
 from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 import time
 
+from .. import strtobool
+
+
 def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
@@ -55,17 +60,33 @@ def test_bad_access(client, live_server, measure_memory_usage):
 
     assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
 
-    # file:// is permitted by default, but it will be caught by ALLOW_FILE_URI
 
+def test_file_access(client, live_server, measure_memory_usage):
+    #live_server_setup(live_server)
+
+    test_file_path = "/tmp/test-file.txt"
+
+    # file:// is permitted by default, but it will be caught by ALLOW_FILE_URI
     client.post(
         url_for("form_quick_watch_add"),
-        data={"url": 'file:///tasty/disk/drive', "tags": ''},
+        data={"url": f"file://{test_file_path}", "tags": ''},
         follow_redirects=True
     )
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
 
-    assert b'file:// type access is denied for security reasons.' in res.data
+    # If it is enabled at test time
+    if strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
+        res = client.get(
+            url_for("preview_page", uuid="first"),
+            follow_redirects=True
+        )
+
+        # Should see something (this file added by run_basic_tests.sh)
+        assert b"Hello world" in res.data
+    else:
+        # Default should be here
+        assert b'file:// type access is denied for security reasons.' in res.data
 
 def test_xss(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)

changedetectionio/update_worker.py

@@ -81,6 +81,9 @@ class update_worker(threading.Thread):
             'uuid': watch.get('uuid') if watch else None,
             'watch_url': watch.get('url') if watch else None,
         })
+
+        n_object.update(watch.extra_notification_token_values())
+
         logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
         logger.debug("Queued notification for sending")
         notification_q.put(n_object)

requirements.txt

@@ -22,6 +22,7 @@ validators~=0.21
 # >= 2.26 also adds Brotli support if brotli is installed
 brotli~=1.0
 requests[socks]
+requests-file
 
 urllib3==1.26.19
 chardet>2.3.0