Extract <title> as title should work on all processors

.github/workflows/containers.yml

@@ -95,7 +95,7 @@ jobs:
           push: true
           tags: |
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
@@ -116,7 +116,7 @@ jobs:
             ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
             ghcr.io/dgtlmoon/changedetection.io:latest
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
           cache-from: type=gha
           cache-to: type=gha,mode=max
 # Looks like this was disabled

.github/workflows/test-container-build.yml

@@ -64,7 +64,7 @@ jobs:
           with:
             context: ./
             file: ./Dockerfile
-            platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8,linux/arm64/v8
+            platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
             cache-from: type=local,src=/tmp/.buildx-cache
             cache-to: type=local,dest=/tmp/.buildx-cache
 

README.md

@@ -41,20 +41,6 @@ Using the **Browser Steps** configuration, add basic steps before performing cha
 After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
 Requires Playwright to be enabled.
 
-### Awesome restock and price change notifications
-
-Enable the _"Re-stock & Price detection for single product pages"_ option to activate the best way to monitor product pricing.
-
-Easily organise and monitor prices for products from the dashboard, get alerts and notifications when the price of a product changes or comes back in stock again!
-
-[<img src="docs/restock-overview.png" style="max-width:100%;" alt="Easily keep an eye on product price changes directly from the UI"  title="Easily keep an eye on product price changes directly from the UI" />](https://changedetection.io?src=github)
-
-Set price change notification parameters, upper and lower price, price change percentage and more.
-Always know when a product for sale drops in price.
-
-[<img src="docs/restock-settings.png" style="max-width:100%;" alt="Set upper lower and percentage price change notification values"  title="Set upper lower and percentage price change notification values" />](https://changedetection.io?src=github)
-
-
 
 ### Example use cases
 

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.46.01'
+__version__ = '0.45.26'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/blueprint/tags/__init__.py

@@ -1,6 +1,4 @@
-from flask import Blueprint, request, render_template, flash, url_for, redirect
-
-
+from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.flask_app import login_optionally_required
 
@@ -98,55 +96,22 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
     @login_optionally_required
     def form_tag_edit(uuid):
-        from changedetectionio.blueprint.tags.form import group_restock_settings_form
+        from changedetectionio import forms
+
         if uuid == 'first':
             uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
 
         default = datastore.data['settings']['application']['tags'].get(uuid)
 
-        form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
-                                       data=default,
-                                       extra_notification_tokens=datastore.get_unique_notification_tokens_available()
-                                       )
-
-        template_args = {
-            'data': default,
-            'form': form,
-            'watch': default,
-            'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
-        }
-
-        included_content = {}
-        if form.extra_form_content():
-            # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/
-            # And then render the code from the module
-            from jinja2 import Environment, FileSystemLoader
-            import importlib.resources
-            templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates'))
-            env = Environment(loader=FileSystemLoader(templates_dir))
-            template_str = """{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
-        <script>        
-            $(document).ready(function () {
-                toggleOpacity('#overrides_watch', '#restock-fieldset-price-group', true);
-            });
-        </script>            
-                <fieldset>
-                    <div class="pure-control-group">
-                        <fieldset class="pure-group">
-                        {{ render_checkbox_field(form.overrides_watch) }}
-                        <span class="pure-form-message-inline">Used for watches in "Restock & Price detection" mode</span>
-                        </fieldset>
-                </fieldset>
-                """
-            template_str += form.extra_form_content()
-            template = env.from_string(template_str)
-            included_content = template.render(**template_args)
+        form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
+                               data=default,
+                               )
+        form.datastore=datastore # needed?
 
         output = render_template("edit-tag.html",
+                                 data=default,
+                                 form=form,
                                  settings_application=datastore.data['settings']['application'],
-                                 extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,
-                                 extra_form_content=included_content,
-                                 **template_args
                                  )
 
         return output
@@ -155,15 +120,14 @@ def construct_blueprint(datastore: ChangeDetectionStore):
     @tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
     @login_optionally_required
     def form_tag_edit_submit(uuid):
-        from changedetectionio.blueprint.tags.form import group_restock_settings_form
+        from changedetectionio import forms
         if uuid == 'first':
             uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
 
         default = datastore.data['settings']['application']['tags'].get(uuid)
 
-        form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,
+        form = forms.processor_text_json_diff_form(formdata=request.form if request.method == 'POST' else None,
                                data=default,
-                               extra_notification_tokens=datastore.get_unique_notification_tokens_available()
                                )
         # @todo subclass form so validation works
         #if not form.validate():
@@ -172,7 +136,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 #           return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
 
         datastore.data['settings']['application']['tags'][uuid].update(form.data)
-        datastore.data['settings']['application']['tags'][uuid]['processor'] = 'restock_diff'
         datastore.needs_write_urgent = True
         flash("Updated")
 

changedetectionio/blueprint/tags/form.py

@@ -1,15 +1,16 @@
 from wtforms import (
+    BooleanField,
     Form,
+    IntegerField,
+    RadioField,
+    SelectField,
     StringField,
     SubmitField,
+    TextAreaField,
     validators,
 )
-from wtforms.fields.simple import BooleanField
 
-from changedetectionio.processors.restock_diff.forms import processor_settings_form as restock_settings_form
 
-class group_restock_settings_form(restock_settings_form):
-    overrides_watch = BooleanField('Activate for individual watches in this tag/group?', default=False)
 
 class SingleTag(Form):
 

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -26,9 +26,6 @@
         <ul>
             <li class="tab" id=""><a href="#general">General</a></li>
             <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
-            {% if extra_tab_content %}
-            <li class="tab"><a href="#extras_tab">{{ extra_tab_content }}</a></li>
-            {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
         </ul>
     </div>
@@ -100,12 +97,6 @@ nav
 
             </div>
 
-        {# rendered sub Template #}
-        {% if extra_form_content %}
-            <div class="tab-pane-inner" id="extras_tab">
-            {{ extra_form_content|safe }}
-            </div>
-        {% endif %}
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div  class="pure-control-group inline-radio">
@@ -128,7 +119,7 @@ nav
                         {% endif %}
                         <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
 
-                        {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
+                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
                     </div>
                 </fieldset>
             </div>

changedetectionio/content_fetchers/requests.py

@@ -4,7 +4,6 @@ import os
 import chardet
 import requests
 
-from changedetectionio import strtobool
 from changedetectionio.content_fetchers.exceptions import BrowserStepsInUnsupportedFetcher, EmptyReply, Non200ErrorCodeReceived
 from changedetectionio.content_fetchers.base import Fetcher
 
@@ -46,19 +45,13 @@ class fetcher(Fetcher):
             if self.system_https_proxy:
                 proxies['https'] = self.system_https_proxy
 
-        session = requests.Session()
-
-        if strtobool(os.getenv('ALLOW_FILE_URI', 'false')) and url.startswith('file://'):
-            from requests_file import FileAdapter
-            session.mount('file://', FileAdapter())
-
-        r = session.request(method=request_method,
-                            data=request_body,
-                            url=url,
-                            headers=request_headers,
-                            timeout=timeout,
-                            proxies=proxies,
-                            verify=False)
+        r = requests.request(method=request_method,
+                             data=request_body,
+                             url=url,
+                             headers=request_headers,
+                             timeout=timeout,
+                             proxies=proxies,
+                             verify=False)
 
         # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
         # For example - some sites don't tell us it's utf-8, but return utf-8 content

changedetectionio/flask_app.py

@@ -532,21 +532,12 @@ def changedetection_app(config=None, datastore_o=None):
     @login_optionally_required
     def ajax_callback_send_notification_test(watch_uuid=None):
 
-        # Watch_uuid could be unset in the case its used in tag editor, global setings
+        # Watch_uuid could be unsuet in the case its used in tag editor, global setings
         import apprise
-        import random
         from .apprise_asset import asset
         apobj = apprise.Apprise(asset=asset)
 
-        is_global_settings_form = request.args.get('mode', '') == 'global-settings'
-        is_group_settings_form = request.args.get('mode', '') == 'group-settings'
-
-        # Use an existing random one on the global/main settings form
-        if not watch_uuid and (is_global_settings_form or is_group_settings_form):
-            logger.debug(f"Send test notification - Choosing random Watch {watch_uuid}")
-            watch_uuid = random.choice(list(datastore.data['watching'].keys()))
-
-        watch = datastore.data['watching'].get(watch_uuid)
+        watch = datastore.data['watching'].get(watch_uuid) if watch_uuid else None
 
         notification_urls = request.form['notification_urls'].strip().splitlines()
 
@@ -558,6 +549,8 @@ def changedetection_app(config=None, datastore_o=None):
                     tag = datastore.tag_exists_by_name(k.strip())
                     notification_urls = tag.get('notifications_urls') if tag and tag.get('notifications_urls') else None
 
+        is_global_settings_form = request.args.get('mode', '') == 'global-settings'
+        is_group_settings_form = request.args.get('mode', '') == 'group-settings'
         if not notification_urls and not is_global_settings_form and not is_group_settings_form:
             # In the global settings, use only what is typed currently in the text box
             logger.debug("Test notification - Trying by global system settings notifications")
@@ -576,7 +569,7 @@ def changedetection_app(config=None, datastore_o=None):
         try:
             # use the same as when it is triggered, but then override it with the form test values
             n_object = {
-                'watch_url': request.form.get('window_url', "https://changedetection.io"),
+                'watch_url': request.form['window_url'],
                 'notification_urls': notification_urls
             }
 
@@ -703,16 +696,11 @@ def changedetection_app(config=None, datastore_o=None):
             form_class = forms.processor_text_json_diff_form
 
         form = form_class(formdata=request.form if request.method == 'POST' else None,
-                          data=default,
-                          extra_notification_tokens=default.extra_notification_token_values()
-                          )
+                               data=default
+                               )
 
-        # For the form widget tag UUID back to "string name" for the field
-        form.tags.datastore = datastore
-
-        # Used by some forms that need to dig deeper
-        form.datastore = datastore
-        form.watch = default
+        # For the form widget tag uuid lookup
+        form.tags.datastore = datastore # in _value
 
         for p in datastore.extra_browsers:
             form.fetch_backend.choices.append(p)
@@ -778,11 +766,6 @@ def changedetection_app(config=None, datastore_o=None):
             datastore.data['watching'][uuid].update(form.data)
             datastore.data['watching'][uuid].update(extra_update_obj)
 
-            if not datastore.data['watching'][uuid].get('tags'):
-                # Force it to be a list, because form.data['tags'] will be string if nothing found
-                # And del(form.data['tags'] ) wont work either for some reason
-                datastore.data['watching'][uuid]['tags'] = []
-
             # Recast it if need be to right data Watch handler
             watch_class = get_custom_watch_obj_for_processor(form.data.get('processor'))
             datastore.data['watching'][uuid] = watch_class(datastore_path=datastore_o.datastore_path, default=datastore.data['watching'][uuid])
@@ -832,7 +815,6 @@ def changedetection_app(config=None, datastore_o=None):
                 'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
                 'extra_title': f" - Edit - {watch.label}",
                 'extra_processor_config': form.extra_tab_content(),
-                'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
                 'form': form,
                 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,
                 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
@@ -887,8 +869,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status
         form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,
-                                        data=default,
-                                        extra_notification_tokens=datastore.get_unique_notification_tokens_available()
+                                        data=default
                                         )
 
         # Remove the last option 'System default'
@@ -940,7 +921,6 @@ def changedetection_app(config=None, datastore_o=None):
         output = render_template("settings.html",
                                  api_key=datastore.data['settings']['application'].get('api_access_token'),
                                  emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
-                                 extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(),
                                  form=form,
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
                                  min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
@@ -1568,13 +1548,9 @@ def changedetection_app(config=None, datastore_o=None):
                     for uuid in uuids:
                         uuid = uuid.strip()
                         if datastore.data['watching'].get(uuid):
-                            # Bug in old versions caused by bad edit page/tag handler
-                            if isinstance(datastore.data['watching'][uuid]['tags'], str):
-                                datastore.data['watching'][uuid]['tags'] = []
-
                             datastore.data['watching'][uuid]['tags'].append(tag_uuid)
 
-            flash(f"{len(uuids)} watches were tagged")
+            flash("{} watches assigned tag".format(len(uuids)))
 
         return redirect(url_for('index'))
 

changedetectionio/forms.py

@@ -231,6 +231,9 @@ class ValidateJinja2Template(object):
     """
     Validates that a {token} is from a valid set
     """
+    def __init__(self, message=None):
+        self.message = message
+
     def __call__(self, form, field):
         from changedetectionio import notification
 
@@ -245,10 +248,6 @@ class ValidateJinja2Template(object):
         try:
             jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader)
             jinja2_env.globals.update(notification.valid_tokens)
-            # Extra validation tokens provided on the form_class(... extra_tokens={}) setup
-            if hasattr(field, 'extra_notification_tokens'):
-                jinja2_env.globals.update(field.extra_notification_tokens)
-
             jinja2_env.from_string(joined_data).render()
         except TemplateSyntaxError as e:
             raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
@@ -423,12 +422,6 @@ class quickWatchForm(Form):
 class commonSettingsForm(Form):
     from . import processors
 
-    def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs):
-        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
-        self.notification_body.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
-        self.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
-        self.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
-
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
     fetch_backend = RadioField(u'Fetch Method', choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
@@ -436,8 +429,8 @@ class commonSettingsForm(Form):
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])
     processor = RadioField( label=u"Processor - What do you want to achieve?", choices=processors.available_processors(), default="text_json_diff")
-    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")])
-
+    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1,
+                                                                                                                                    message="Should contain one or more seconds")])
 
 class importForm(Form):
     from . import processors
@@ -597,11 +590,6 @@ class globalSettingsForm(Form):
     # Define these as FormFields/"sub forms", this way it matches the JSON storage
     # datastore.data['settings']['application']..
     # datastore.data['settings']['requests']..
-    def __init__(self, formdata=None, obj=None, prefix="", data=None, meta=None, **kwargs):
-        super().__init__(formdata, obj, prefix, data, meta, **kwargs)
-        self.application.notification_body.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
-        self.application.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
-        self.application.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
 
     requests = FormField(globalSettingsRequestForm)
     application = FormField(globalSettingsApplicationForm)

changedetectionio/model/Tag.py

@@ -1,14 +1,14 @@
 
 from changedetectionio.model import watch_base
 
-
 class model(watch_base):
 
     def __init__(self, *arg, **kw):
-        super(model, self).__init__(*arg, **kw)
 
-        self['overrides_watch'] = kw.get('default', {}).get('overrides_watch')
+        super(model, self).__init__(*arg, **kw)
 
         if kw.get('default'):
             self.update(kw['default'])
             del kw['default']
+
+

changedetectionio/model/Watch.py

@@ -432,17 +432,6 @@ class model(watch_base):
     def toggle_mute(self):
         self['notification_muted'] ^= True
 
-    def extra_notification_token_values(self):
-        # Used for providing extra tokens
-        # return {'widget': 555}
-        return {}
-
-    def extra_notification_token_placeholder_info(self):
-        # Used for providing extra tokens
-        # return [('widget', "Get widget amounts")]
-        return []
-
-
     def extract_regex_from_all_history(self, regex):
         import csv
         import re

changedetectionio/notification.py

@@ -157,7 +157,7 @@ def process_notification(n_object, datastore):
                 logger.warning(f"Process Notification: skipping empty notification URL.")
                 continue
 
-            logger.info(f">> Process Notification: AppRise notifying {url}")
+            logger.info(">> Process Notification: AppRise notifying {}".format(url))
             url = jinja_render(template_str=url, **notification_parameters)
 
             # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
@@ -230,7 +230,6 @@ def process_notification(n_object, datastore):
         log_value = logs.getvalue()
 
         if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
-            logger.critical(log_value)
             raise Exception(log_value)
 
     # Return what was sent for better logging - after the for loop
@@ -273,18 +272,19 @@ def create_notification_parameters(n_object, datastore):
     tokens.update(
         {
             'base_url': base_url,
+            'current_snapshot': n_object.get('current_snapshot', ''),
+            'diff': n_object.get('diff', ''),  # Null default in the case we use a test
+            'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test
+            'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
+            'diff_patch': n_object.get('diff_patch', ''),  # Null default in the case we use a test
+            'diff_removed': n_object.get('diff_removed', ''),  # Null default in the case we use a test
             'diff_url': diff_url,
             'preview_url': preview_url,
+            'triggered_text': n_object.get('triggered_text', ''),
             'watch_tag': watch_tag if watch_tag is not None else '',
             'watch_title': watch_title if watch_title is not None else '',
             'watch_url': watch_url,
             'watch_uuid': uuid,
         })
 
-    # n_object will contain diff, diff_added etc etc
-    tokens.update(n_object)
-
-    if uuid:
-        tokens.update(datastore.data['watching'].get(uuid).extra_notification_token_values())
-
     return tokens

changedetectionio/processors/restock_diff/__init__.py

@@ -45,10 +45,13 @@ class Restock(dict):
 
     def __setitem__(self, key, value):
         # Custom logic to handle setting price and original_price
-        if key == 'price' or key == 'original_price':
+        if key == 'price':
             if isinstance(value, str):
                 value = self.parse_currency(raw_value=value)
 
+            if value and not self.get('original_price'):
+                self['original_price'] = value
+
         super().__setitem__(key, value)
 
 class Watch(BaseWatch):
@@ -56,25 +59,7 @@ class Watch(BaseWatch):
         super().__init__(*arg, **kw)
         self['restock'] = Restock(kw['default']['restock']) if kw.get('default') and kw['default'].get('restock') else Restock()
 
-        self['restock_settings'] = kw['default']['restock_settings'] if kw.get('default',{}).get('restock_settings') else {
-            'follow_price_changes': True,
-            'in_stock_processing' : 'in_stock_only'
-        } #@todo update
-
     def clear_watch(self):
         super().clear_watch()
         self.update({'restock': Restock()})
 
-    def extra_notification_token_values(self):
-        values = super().extra_notification_token_values()
-        values['restock'] = self.get('restock', {})
-        return values
-
-    def extra_notification_token_placeholder_info(self):
-        values = super().extra_notification_token_placeholder_info()
-
-        values.append(('restock.price', "Price detected"))
-        values.append(('restock.original_price', "Original price at first check"))
-
-        return values
-

changedetectionio/processors/restock_diff/forms.py

@@ -1,81 +1,61 @@
+
 from wtforms import (
     BooleanField,
     validators,
     FloatField
 )
-from wtforms.fields.choices import RadioField
-from wtforms.fields.form import FormField
-from wtforms.form import Form
 
 from changedetectionio.forms import processor_text_json_diff_form
 
-
-class RestockSettingsForm(Form):
-    in_stock_processing = RadioField(label='Re-stock detection', choices=[
-        ('in_stock_only', "In Stock only (Out Of Stock -> In Stock only)"),
-        ('all_changes', "Any availability changes"),
-        ('off', "Off, don't follow availability/restock"),
-    ], default="in_stock_only")
-
-    price_change_min = FloatField('Below price to trigger notification', [validators.Optional()],
+class processor_settings_form(processor_text_json_diff_form):
+    in_stock_only = BooleanField('Only trigger when product goes BACK to in-stock', default=True)
+    price_change_min = FloatField('Minimum amount to trigger notification', [validators.Optional()],
                                   render_kw={"placeholder": "No limit", "size": "10"})
-    price_change_max = FloatField('Above price to trigger notification', [validators.Optional()],
+    price_change_max = FloatField('Maximum amount to trigger notification', [validators.Optional()],
                                   render_kw={"placeholder": "No limit", "size": "10"})
-    price_change_threshold_percent = FloatField('Threshold in % for price changes since the original price', validators=[
-
+    price_change_threshold_percent = FloatField('Threshold in % for price changes', validators=[
         validators.Optional(),
         validators.NumberRange(min=0, max=100, message="Should be between 0 and 100"),
     ], render_kw={"placeholder": "0%", "size": "5"})
 
-    follow_price_changes = BooleanField('Follow price changes', default=True)
-
-class processor_settings_form(processor_text_json_diff_form):
-    restock_settings = FormField(RestockSettingsForm)
+    follow_price_changes = BooleanField('Follow price changes', default=False)
 
     def extra_tab_content(self):
         return 'Restock & Price Detection'
 
     def extra_form_content(self):
-        output = ""
-
-        if getattr(self, 'watch', None) and getattr(self, 'datastore'):
-            for tag_uuid in self.watch.get('tags'):
-                tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {})
-                if tag.get('overrides_watch'):
-                    # @todo - Quick and dirty, cant access 'url_for' here because its out of scope somehow
-                    output = f"""<p><strong>Note! A Group tag overrides the restock and price detection here.</strong></p><style>#restock-fieldset-price-group {{ opacity: 0.6; }}</style>"""
-
-        output += """
+        return """
         {% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
         <script>        
             $(document).ready(function () {
-                toggleOpacity('#restock_settings-follow_price_changes', '.price-change-minmax', true);
+                toggleOpacity('#follow_price_changes', '.price-change-minmax', true);
             });
         </script>
 
-        <fieldset id="restock-fieldset-price-group">
+
+        <fieldset>
             <div class="pure-control-group">
-                <fieldset class="pure-group inline-radio">
-                    {{ render_field(form.restock_settings.in_stock_processing) }}
+                <fieldset class="pure-group">
+                    {{ render_checkbox_field(form.in_stock_only) }}
+                    <span class="pure-form-message-inline">Only trigger notifications when page changes from <strong>out of stock</strong> to <strong>back in stock</strong></span>
                 </fieldset>
                 <fieldset class="pure-group">
-                    {{ render_checkbox_field(form.restock_settings.follow_price_changes) }}
+                    {{ render_checkbox_field(form.follow_price_changes) }}
                     <span class="pure-form-message-inline">Changes in price should trigger a notification</span>
+                    <span class="pure-form-message-inline">When OFF - only care about restock detection</span>
                 </fieldset>
                 <fieldset class="pure-group price-change-minmax">               
-                    {{ render_field(form.restock_settings.price_change_min, placeholder=watch.get('restock', {}).get('price')) }}
-                    <span class="pure-form-message-inline">Minimum amount, Trigger a change/notification when the price drops <i>below</i> this value.</span>
+                    {{ render_field(form.price_change_min, placeholder=watch['restock']['price']) }}
+                    <span class="pure-form-message-inline">Minimum amount, only trigger a change when the price is less than this amount.</span>
                 </fieldset>
                 <fieldset class="pure-group price-change-minmax">
-                    {{ render_field(form.restock_settings.price_change_max, placeholder=watch.get('restock', {}).get('price')) }}
-                    <span class="pure-form-message-inline">Maximum amount, Trigger a change/notification when the price rises <i>above</i> this value.</span>
+                    {{ render_field(form.price_change_max, placeholder=watch['restock']['price']) }}
+                    <span class="pure-form-message-inline">Maximum amount, only trigger a change when the price is more than this amount.</span>
                 </fieldset>
                 <fieldset class="pure-group price-change-minmax">
-                    {{ render_field(form.restock_settings.price_change_threshold_percent) }}
-                    <span class="pure-form-message-inline">Price must change more than this % to trigger a change since the first check.</span><br>
-                    <span class="pure-form-message-inline">For example, If the product is $1,000 USD originally, <strong>2%</strong> would mean it has to change more than $20 since the first check.</span><br>
+                    {{ render_field(form.price_change_threshold_percent) }}
+                    <span class="pure-form-message-inline">Price must change more than this % to trigger a change.</span><br>
+                    <span class="pure-form-message-inline">For example, If the product is $1,000 USD, <strong>2%</strong> would mean it has to change more than $20 since the first check.</span><br>
                 </fieldset>                
             </div>
-        </fieldset>
-        """
-        return output
+        </fieldset>"""

changedetectionio/processors/restock_diff/processor.py

@@ -132,18 +132,6 @@ class perform_site_check(difference_detection_processor):
         update_obj['content_type'] = self.fetcher.headers.get('Content-Type', '')
         update_obj["last_check_status"] = self.fetcher.get_last_status_code()
 
-        # Which restock settings to compare against?
-        restock_settings = watch.get('restock_settings', {})
-
-        # See if any tags have 'activate for individual watches in this tag/group?' enabled and use the first we find
-        for tag_uuid in watch.get('tags'):
-            tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {})
-            if tag.get('overrides_watch'):
-                restock_settings = tag.get('restock_settings', {})
-                logger.info(f"Watch {watch.get('uuid')} - Tag '{tag.get('title')}' selected for restock settings override")
-                break
-
-
         itemprop_availability = {}
         try:
             itemprop_availability = get_itemprop_availability(html_content=self.fetcher.content)
@@ -177,11 +165,6 @@ class perform_site_check(difference_detection_processor):
         # Main detection method
         fetched_md5 = None
 
-        # store original price if not set
-        if itemprop_availability and itemprop_availability.get('price') and not itemprop_availability.get('original_price'):
-            itemprop_availability['original_price'] = itemprop_availability.get('price')
-            update_obj['restock']["original_price"] = itemprop_availability.get('price')
-
         if not self.fetcher.instock_data and not itemprop_availability.get('availability'):
             raise ProcessorException(
                 message=f"Unable to extract restock data for this page unfortunately. (Got code {self.fetcher.get_last_status_code()} from server), no embedded stock information was found and nothing interesting in the text, try using this watch with Chrome.",
@@ -200,7 +183,7 @@ class perform_site_check(difference_detection_processor):
 
         # What we store in the snapshot
         price = update_obj.get('restock').get('price') if update_obj.get('restock').get('price') else ""
-        snapshot_content = f"In Stock: {update_obj.get('restock').get('in_stock')} - Price: {price}"
+        snapshot_content = f"{update_obj.get('restock').get('in_stock')} - {price}"
 
         # Main detection method
         fetched_md5 = hashlib.md5(snapshot_content.encode('utf-8')).hexdigest()
@@ -212,14 +195,14 @@ class perform_site_check(difference_detection_processor):
         # out of stock -> back in stock only?
         if watch.get('restock') and watch['restock'].get('in_stock') != update_obj['restock'].get('in_stock'):
             # Yes if we only care about it going to instock, AND we are in stock
-            if restock_settings.get('in_stock_processing') == 'in_stock_only' and update_obj['restock']['in_stock']:
+            if watch.get('in_stock_only') and update_obj['restock']['in_stock']:
                 changed_detected = True
 
-            if restock_settings.get('in_stock_processing') == 'all_changes':
+            if not watch.get('in_stock_only'):
                 # All cases
                 changed_detected = True
 
-        if restock_settings.get('follow_price_changes') and watch.get('restock') and update_obj.get('restock') and update_obj['restock'].get('price'):
+        if watch.get('follow_price_changes') and watch.get('restock') and update_obj.get('restock') and update_obj['restock'].get('price'):
             price = float(update_obj['restock'].get('price'))
             # Default to current price if no previous price found
             if watch['restock'].get('original_price'):
@@ -231,25 +214,26 @@ class perform_site_check(difference_detection_processor):
             # Minimum/maximum price limit
             if update_obj.get('restock') and update_obj['restock'].get('price'):
                 logger.debug(
-                    f"{watch.get('uuid')} - Change was detected, 'price_change_max' is '{restock_settings.get('price_change_max', '')}' 'price_change_min' is '{restock_settings.get('price_change_min', '')}', price from website is '{update_obj['restock'].get('price', '')}'.")
+                    f"{watch.get('uuid')} - Change was detected, 'price_change_max' is '{watch.get('price_change_max', '')}' 'price_change_min' is '{watch.get('price_change_min', '')}', price from website is '{update_obj['restock'].get('price', '')}'.")
                 if update_obj['restock'].get('price'):
-                    min_limit = float(restock_settings.get('price_change_min')) if restock_settings.get('price_change_min') else None
-                    max_limit = float(restock_settings.get('price_change_max')) if restock_settings.get('price_change_max') else None
+                    min_limit = float(watch.get('price_change_min')) if watch.get('price_change_min') else None
+                    max_limit = float(watch.get('price_change_max')) if watch.get('price_change_max') else None
 
                     price = float(update_obj['restock'].get('price'))
                     logger.debug(f"{watch.get('uuid')} after float conversion - Min limit: '{min_limit}' Max limit: '{max_limit}' Price: '{price}'")
                     if min_limit or max_limit:
                         if is_between(number=price, lower=min_limit, upper=max_limit):
-                            # Price was between min/max limit, so there was nothing todo in any case
-                            logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}, nothing to check, forcing changed_detected = False (was {changed_detected})")
-                            changed_detected = False
+                            logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}")
+                            if changed_detected:
+                                logger.debug(f"{watch.get('uuid')} Override change-detected to FALSE because price was inside threshold")
+                                changed_detected = False
                         else:
-                            logger.trace(f"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}, continuing normal comparison")
+                            logger.trace(f"{watch.get('uuid')} {price} is NOT between {min_limit} and {max_limit}")
 
                     # Price comparison by %
-                    if watch['restock'].get('original_price') and changed_detected and restock_settings.get('price_change_threshold_percent'):
+                    if watch['restock'].get('original_price') and changed_detected and watch.get('price_change_threshold_percent'):
                         previous_price = float(watch['restock'].get('original_price'))
-                        pc = float(restock_settings.get('price_change_threshold_percent'))
+                        pc = float(watch.get('price_change_threshold_percent'))
                         change = abs((price - previous_price) / previous_price * 100)
                         if change and change <= pc:
                             logger.debug(f"{watch.get('uuid')} Override change-detected to FALSE because % threshold ({pc}%) was {change:.3f}%")

changedetectionio/run_basic_tests.sh

@@ -35,8 +35,4 @@ pytest tests/test_access_control.py
 pytest tests/test_notification.py
 pytest tests/test_backend.py
 pytest tests/test_rss.py
-pytest tests/test_unique_lines.py
-
-# Check file:// will pickup a file when enabled
-echo "Hello world" > /tmp/test-file.txt
-ALLOW_FILE_URI=yes pytest tests/test_security.py
+pytest tests/test_unique_lines.py

changedetectionio/static/js/watch-settings.js

@@ -27,5 +27,6 @@ $(document).ready(function () {
 
     toggleOpacity('#time_between_check_use_default', '#time_between_check', false);
 
+
 });
 

changedetectionio/static/styles/scss/styles.scss

@@ -1118,11 +1118,6 @@ ul {
     color: #fff;
     opacity: 0.7;
   }
-
-  svg {
-    vertical-align: middle;
-  }
-
   @extend .inline-tag;
 }
 

changedetectionio/static/styles/styles.css

@@ -1194,11 +1194,6 @@ ul {
   color: #fff;
   opacity: 0.7; }
 
-
-.restock-label svg {
-  vertical-align: middle; }
-
-
 #chrome-extension-link {
   padding: 9px;
   border: 1px solid var(--color-grey-800);

changedetectionio/store.py

@@ -83,14 +83,16 @@ class ChangeDetectionStore:
                         self.__data['settings']['application'].update(from_disk['settings']['application'])
 
                 # Convert each existing watch back to the Watch.model object
-                for uuid, watch in self.__data['watching'].items():
-                    self.__data['watching'][uuid] = self.rehydrate_entity(uuid, watch)
-                    logger.info(f"Watching: {uuid} {watch['url']}")
 
-                # And for Tags also, should be Restock type because it has extra settings
-                for uuid, tag in self.__data['settings']['application']['tags'].items():
-                    self.__data['settings']['application']['tags'][uuid] = self.rehydrate_entity(uuid, tag, processor_override='restock_diff')
-                    logger.info(f"Tag: {uuid} {tag['title']}")
+                for uuid, watch in self.__data['watching'].items():
+                    watch['uuid'] = uuid
+                    watch_class = get_custom_watch_obj_for_processor(watch.get('processor'))
+                    if watch.get('uuid') != 'text_json_diff':
+                        logger.trace(f"Loading Watch object '{watch_class.__module__}.{watch_class.__name__}' for UUID {uuid}")
+
+                    self.__data['watching'][uuid] = watch_class(datastore_path=self.datastore_path, default=watch)
+
+                    logger.info(f"Watching: {uuid} {self.__data['watching'][uuid]['url']}")
 
         # First time ran, Create the datastore.
         except (FileNotFoundError):
@@ -145,22 +147,6 @@ class ChangeDetectionStore:
         # Finally start the thread that will manage periodic data saves to JSON
         save_data_thread = threading.Thread(target=self.save_datastore).start()
 
-    def rehydrate_entity(self, uuid, entity, processor_override=None):
-        """Set the dict back to the dict Watch object"""
-        entity['uuid'] = uuid
-
-        if processor_override:
-            watch_class = get_custom_watch_obj_for_processor(processor_override)
-            entity['processor']=processor_override
-        else:
-            watch_class = get_custom_watch_obj_for_processor(entity.get('processor'))
-
-        if entity.get('uuid') != 'text_json_diff':
-            logger.trace(f"Loading Watch object '{watch_class.__module__}.{watch_class.__name__}' for UUID {uuid}")
-
-        entity = watch_class(datastore_path=self.datastore_path, default=entity)
-        return entity
-
     def set_last_viewed(self, uuid, timestamp):
         logger.debug(f"Setting watch UUID: {uuid} last viewed to {int(timestamp)}")
         self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
@@ -199,9 +185,6 @@ class ChangeDetectionStore:
 
     @property
     def has_unviewed(self):
-        if not self.__data.get('watching'):
-            return None
-
         for uuid, watch in self.__data['watching'].items():
             if watch.history_n >= 2 and watch.viewed == False:
                 return True
@@ -631,33 +614,6 @@ class ChangeDetectionStore:
                 return True
         return False
 
-    def get_unique_notification_tokens_available(self):
-        # Ask each type of watch if they have any extra notification token to add to the validation
-        extra_notification_tokens = {}
-        watch_processors_checked = set()
-
-        for watch_uuid, watch in self.__data['watching'].items():
-            processor = watch.get('processor')
-            if processor not in watch_processors_checked:
-                extra_notification_tokens.update(watch.extra_notification_token_values())
-                watch_processors_checked.add(processor)
-
-        return extra_notification_tokens
-
-    def get_unique_notification_token_placeholders_available(self):
-        # The actual description of the tokens, could be combined with get_unique_notification_tokens_available instead of doing this twice
-        extra_notification_tokens = []
-        watch_processors_checked = set()
-
-        for watch_uuid, watch in self.__data['watching'].items():
-            processor = watch.get('processor')
-            if processor not in watch_processors_checked:
-                extra_notification_tokens+=watch.extra_notification_token_placeholder_info()
-                watch_processors_checked.add(processor)
-
-        return extra_notification_tokens
-
-
     def get_updates_available(self):
         import inspect
         updates_available = []
@@ -894,17 +850,4 @@ class ChangeDetectionStore:
                 watch['restock'] = Restock({'in_stock': watch.get('in_stock')})
                 del watch['in_stock']
 
-    # Migrate old restock settings
-    def update_18(self):
-        for uuid, watch in self.data['watching'].items():
-            if not watch.get('restock_settings'):
-                # So we enable price following by default
-                self.data['watching'][uuid]['restock_settings'] = {'follow_price_changes': True}
-
-            # Migrate and cleanoff old value
-            self.data['watching'][uuid]['restock_settings']['in_stock_processing'] = 'in_stock_only' if watch.get(
-                'in_stock_only') else 'all_changes'
-
-            if self.data['watching'][uuid].get('in_stock_only'):
-                del (self.data['watching'][uuid]['in_stock_only'])
 

changedetectionio/templates/_common_fields.html

@@ -1,7 +1,7 @@
 
 {% from '_helpers.html' import render_field %}
 
-{% macro render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) %}
+{% macro render_common_settings_form(form, emailprefix, settings_application) %}
                         <div class="pure-control-group">
                             {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
     Gitter - gitter://token/room
@@ -107,15 +107,7 @@
                                     <tr>
                                         <td><code>{{ '{{triggered_text}}' }}</code></td>
                                         <td>Text that tripped the trigger from filters</td>
-
-                                        {% if extra_notification_token_placeholder_info %}
-                                            {% for token in extra_notification_token_placeholder_info %}
-                                                <tr>
-                                                    <td><code>{{ '{{' }}{{ token[0] }}{{ '}}' }}</code></td>
-                                                    <td>{{ token[1] }}</td>
-                                                </tr>
-                                            {% endfor %}
-                                        {% endif %}
+                                    </tr>
                                     </tbody>
                                 </table>
                                 <div class="pure-form-message-inline">

changedetectionio/templates/edit.html

@@ -246,7 +246,7 @@ User-Agent: wonderbra 1.0") }}
                         {% endif %}
                         <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
 
-                        {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
+                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
                     </div>
                 </fieldset>
             </div>

changedetectionio/templates/settings.html

@@ -92,7 +92,7 @@
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div class="field-group">
-                        {{ render_common_settings_form(form.application.form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}
+                        {{ render_common_settings_form(form.application.form, emailprefix, settings_application) }}
                     </div>
                 </fieldset>
                 <div class="pure-control-group" id="notification-base-url">

changedetectionio/tests/test_notification.py

@@ -3,8 +3,6 @@ import os
 import time
 import re
 from flask import url_for
-from loguru import logger
-
 from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
     set_longer_modified_response
 from . util import  extract_UUID_from_client
@@ -349,81 +347,3 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
         url_for("form_delete", uuid="all"),
         follow_redirects=True
     )
-
-
-#2510
-def test_global_send_test_notification(client, live_server, measure_memory_usage):
-
-
-    #live_server_setup(live_server)
-    set_original_response()
-
-    # otherwise other settings would have already existed from previous tests in this file
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "application-fetch_backend": "html_requests",
-            "application-minutes_between_check": 180,
-            "application-notification_body": 'change detection is cool',
-            "application-notification_format": default_notification_format,
-            "application-notification_urls": "",
-            "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
-        },
-        follow_redirects=True
-    )
-    assert b'Settings updated' in res.data
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
-
-    ######### Test global/system settings
-    res = client.post(
-        url_for("ajax_callback_send_notification_test")+"?mode=global-settings",
-        data={"notification_urls": test_notification_url},
-        follow_redirects=True
-    )
-
-    assert res.status_code != 400
-    assert res.status_code != 500
-
-    # Give apprise time to fire
-    time.sleep(4)
-
-    with open("test-datastore/notification.txt", 'r') as f:
-        x = f.read()
-        assert 'change detection is coo' in x
-
-
-    os.unlink("test-datastore/notification.txt")
-
-    ######### Test group/tag settings
-    res = client.post(
-        url_for("ajax_callback_send_notification_test")+"?mode=group-settings",
-        data={"notification_urls": test_notification_url},
-        follow_redirects=True
-    )
-
-    assert res.status_code != 400
-    assert res.status_code != 500
-
-    # Give apprise time to fire
-    time.sleep(4)
-
-    with open("test-datastore/notification.txt", 'r') as f:
-        x = f.read()
-        # Should come from notification.py default handler when there is no notification body to pull from
-        assert 'change detection is coo' in x
-
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )
-

changedetectionio/tests/test_restock_itemprop.py

@@ -1,10 +1,8 @@
 #!/usr/bin/python3
-import os
 import time
 
 from flask import url_for
 from .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
-from ..notification import default_notification_format
 
 instock_props = [
     # LD+JSON with non-standard list of 'type' https://github.com/dgtlmoon/changedetection.io/issues/1833
@@ -54,8 +52,6 @@ def test_restock_itemprop_basic(client, live_server):
 
     test_url = url_for('test_endpoint', _external=True)
 
-    # By default it should enable ('in_stock_processing') == 'all_changes'
-
     for p in instock_props:
         set_original_response(props_markup=p)
         client.post(
@@ -91,7 +87,6 @@ def test_restock_itemprop_basic(client, live_server):
 def test_itemprop_price_change(client, live_server):
     #live_server_setup(live_server)
 
-    # Out of the box 'Follow price changes' should be ON
     test_url = url_for('test_endpoint', _external=True)
 
     set_original_response(props_markup=instock_props[0], price="190.95")
@@ -119,7 +114,7 @@ def test_itemprop_price_change(client, live_server):
     set_original_response(props_markup=instock_props[0], price='120.45')
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"restock_settings-follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"follow_price_changes": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -133,7 +128,8 @@ def test_itemprop_price_change(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-def _run_test_minmax_limit(client, extra_watch_edit_form):
+def test_itemprop_price_minmax_limit(client, live_server):
+    #live_server_setup(live_server)
 
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
@@ -150,16 +146,17 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
     # A change in price, should trigger a change by default
     wait_for_all_checks(client)
 
-    data = {
-        "tags": "",
-        "url": test_url,
-        "headers": "",
-        'fetch_backend': "html_requests"
-    }
-    data.update(extra_watch_edit_form)
+
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data=data,
+        data={"follow_price_changes": "y",
+              "price_change_min": 900.0,
+              "price_change_max": 1100.10,
+              "url": test_url,
+              "tags": "",
+              "headers": "",
+              'fetch_backend': "html_requests"
+              },
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -167,7 +164,7 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
 
     client.get(url_for("mark_all_viewed"))
 
-    # price changed to something greater than min (900), BUT less than max (1100).. should be no change
+    # price changed to something greater than min (900), and less than max (1100).. should be no change
     set_original_response(props_markup=instock_props[0], price='1000.45')
     client.get(url_for("form_watch_checknow"))
     wait_for_all_checks(client)
@@ -204,44 +201,6 @@ def _run_test_minmax_limit(client, extra_watch_edit_form):
     assert b'Deleted' in res.data
 
 
-def test_restock_itemprop_minmax(client, live_server):
-#    live_server_setup(live_server)
-    extras = {
-        "restock_settings-follow_price_changes": "y",
-        "restock_settings-price_change_min": 900.0,
-        "restock_settings-price_change_max": 1100.10
-    }
-    _run_test_minmax_limit(client, extra_watch_edit_form=extras)
-
-def test_restock_itemprop_with_tag(client, live_server):
-    #live_server_setup(live_server)
-
-    res = client.post(
-        url_for("tags.form_tag_add"),
-        data={"name": "test-tag"},
-        follow_redirects=True
-    )
-    assert b"Tag added" in res.data
-
-    res = client.post(
-        url_for("tags.form_tag_edit_submit", uuid="first"),
-        data={"name": "test-tag",
-              "restock_settings-follow_price_changes": "y",
-              "restock_settings-price_change_min": 900.0,
-              "restock_settings-price_change_max": 1100.10,
-              "overrides_watch": "y", #overrides_watch should be restock_overrides_watch
-              },
-        follow_redirects=True
-    )
-
-    extras = {
-        "tags": "test-tag"
-    }
-
-    _run_test_minmax_limit(client, extra_watch_edit_form=extras)
-
-
-
 def test_itemprop_percent_threshold(client, live_server):
     #live_server_setup(live_server)
 
@@ -262,8 +221,8 @@ def test_itemprop_percent_threshold(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"restock_settings-follow_price_changes": "y",
-              "restock_settings-price_change_threshold_percent": 5.0,
+        data={"follow_price_changes": "y",
+              "price_change_threshold_percent": 5.0,
               "url": test_url,
               "tags": "",
               "headers": "",
@@ -307,70 +266,6 @@ def test_itemprop_percent_threshold(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-
-
-def test_change_with_notification_values(client, live_server):
-    #live_server_setup(live_server)
-
-    if os.path.isfile("test-datastore/notification.txt"):
-        os.unlink("test-datastore/notification.txt")
-
-    test_url = url_for('test_endpoint', _external=True)
-    set_original_response(props_markup=instock_props[0], price='960.45')
-
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
-
-    ######################
-    # You must add a type of 'restock_diff' for its tokens to register as valid in the global settings
-    client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'restock tests', 'processor': 'restock_diff'},
-        follow_redirects=True
-    )
-
-    # A change in price, should trigger a change by default
-    wait_for_all_checks(client)
-
-    # Should see new tokens register
-    res = client.get(url_for("settings_page"))
-    assert b'{{restock.original_price}}' in res.data
-    assert b'Original price at first check' in res.data
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "title new price {{restock.price}}",
-              "application-notification_body": "new price {{restock.price}}",
-              "application-notification_format": default_notification_format,
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    # check tag accepts without error
-
-    # Check the watches in these modes add the tokens for validating
-    assert b"A variable or function is not defined" not in res.data
-
-    assert b"Settings updated." in res.data
-
-
-    set_original_response(props_markup=instock_props[0], price='960.45')
-    # A change in price, should trigger a change by default
-    set_original_response(props_markup=instock_props[0], price='1950.45')
-    client.get(url_for("form_watch_checknow"))
-    wait_for_all_checks(client)
-    time.sleep(3)
-    assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
-    with open("test-datastore/notification.txt", 'r') as f:
-        notification = f.read()
-        assert "new price 1950.45" in notification
-        assert "title new price 1950.45" in notification
-
-
-
 def test_data_sanity(client, live_server):
     #live_server_setup(live_server)
 

changedetectionio/tests/test_security.py

@@ -1,12 +1,7 @@
-import os
-
 from flask import url_for
 from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 import time
 
-from .. import strtobool
-
-
 def test_setup(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
 
@@ -60,33 +55,17 @@ def test_bad_access(client, live_server, measure_memory_usage):
 
     assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
 
-
-def test_file_access(client, live_server, measure_memory_usage):
-    #live_server_setup(live_server)
-
-    test_file_path = "/tmp/test-file.txt"
-
     # file:// is permitted by default, but it will be caught by ALLOW_FILE_URI
+
     client.post(
         url_for("form_quick_watch_add"),
-        data={"url": f"file://{test_file_path}", "tags": ''},
+        data={"url": 'file:///tasty/disk/drive', "tags": ''},
         follow_redirects=True
     )
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
 
-    # If it is enabled at test time
-    if strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
-        res = client.get(
-            url_for("preview_page", uuid="first"),
-            follow_redirects=True
-        )
-
-        # Should see something (this file added by run_basic_tests.sh)
-        assert b"Hello world" in res.data
-    else:
-        # Default should be here
-        assert b'file:// type access is denied for security reasons.' in res.data
+    assert b'file:// type access is denied for security reasons.' in res.data
 
 def test_xss(client, live_server, measure_memory_usage):
     #live_server_setup(live_server)

changedetectionio/update_worker.py

@@ -81,9 +81,6 @@ class update_worker(threading.Thread):
             'uuid': watch.get('uuid') if watch else None,
             'watch_url': watch.get('url') if watch else None,
         })
-
-        n_object.update(watch.extra_notification_token_values())
-
         logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
         logger.debug("Queued notification for sending")
         notification_q.put(n_object)
@@ -500,6 +497,13 @@ class update_worker(threading.Thread):
                     #
                     # Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
                     if process_changedetection_results:
+                        # Always save the screenshot if it's available
+
+                        if update_handler.screenshot:
+                            watch.save_screenshot(screenshot=update_handler.screenshot)
+
+                        if update_handler.xpath_data:
+                            watch.save_xpath_data(data=update_handler.xpath_data)
 
                         # Extract <title> as title if possible/requested.
                         if self.datastore.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
@@ -511,19 +515,12 @@ class update_worker(threading.Thread):
                                     logger.warning(f"UUID: {uuid} Extract <title> as watch title was enabled, but couldn't find a <title>.")
 
                         # Now update after running everything
-                        timestamp = round(time.time())
                         try:
                             self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
-
-                            # Also save the snapshot on the first time checked, "last checked" will always be updated, so we just check history length.
-                            if changed_detected or not watch.history_n:
-
-                                if update_handler.screenshot:
-                                    watch.save_screenshot(screenshot=update_handler.screenshot)
-
-                                if update_handler.xpath_data:
-                                    watch.save_xpath_data(data=update_handler.xpath_data)
+                            # Also save the snapshot on the first time checked
+                            if changed_detected or not watch.get('last_checked'):
+                                timestamp = round(time.time())
 
                                 # Small hack so that we sleep just enough to allow 1 second  between history snapshots
                                 # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
@@ -541,11 +538,15 @@ class update_worker(threading.Thread):
                                 if update_handler.fetcher.content:
                                     watch.save_last_fetched_html(contents=update_handler.fetcher.content, timestamp=timestamp)
 
+                            # A change was detected
+                            if changed_detected:
                                 # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                 if watch.history_n >= 2:
                                     logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
                                     if not watch.get('notification_muted'):
                                         self.send_content_changed_notification(watch_uuid=uuid)
+                                else:
+                                    logger.info(f"Change triggered in UUID {uuid} due to first history saving (no notifications sent) - {watch['url']}")
 
                         except Exception as e:
                             # Catch everything possible here, so that if a worker crashes, we don't lose it until restart!

requirements.txt

@@ -22,7 +22,6 @@ validators~=0.21
 # >= 2.26 also adds Brotli support if brotli is installed
 brotli~=1.0
 requests[socks]
-requests-file
 
 urllib3==1.26.19
 chardet>2.3.0