woops

.gitignore

@@ -10,6 +10,5 @@ dist
 venv
 test-datastore/*
 test-datastore
-test-memory.log
 *.egg-info*
 .vscode/settings.json

MANIFEST.in

@@ -1,5 +1,4 @@
 recursive-include changedetectionio/api *
-recursive-include changedetectionio/apprise_plugin *
 recursive-include changedetectionio/blueprint *
 recursive-include changedetectionio/content_fetchers *
 recursive-include changedetectionio/model *

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.47.05'
+__version__ = '0.46.04'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError

changedetectionio/api/api_v1.py

@@ -58,7 +58,7 @@ class Watch(Resource):
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
 
         if request.args.get('recheck'):
-            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
             return "OK", 200
         if request.args.get('paused', '') == 'paused':
             self.datastore.data['watching'].get(uuid).pause()
@@ -246,7 +246,7 @@ class CreateWatch(Resource):
 
         new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags)
         if new_uuid:
-            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
+            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
             return {'uuid': new_uuid}, 201
         else:
             return "Invalid or unsupported URL", 400
@@ -303,7 +303,7 @@ class CreateWatch(Resource):
 
         if request.args.get('recheck_all'):
             for uuid in self.datastore.data['watching'].keys():
-                self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
             return {'status': "OK"}, 200
 
         return list, 200

changedetectionio/apprise_plugin/__init__.py

@@ -13,7 +13,6 @@ from loguru import logger
 def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     import requests
     import json
-    from urllib.parse import unquote_plus
     from apprise.utils import parse_url as apprise_parse_url
     from apprise import URLBase
 
@@ -48,7 +47,7 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
     if results:
         # Add our headers that the user can potentially over-ride if they wish
         # to to our returned result set and tidy entries by unquoting them
-        headers = {unquote_plus(x): unquote_plus(y)
+        headers = {URLBase.unquote(x): URLBase.unquote(y)
                    for x, y in results['qsd+'].items()}
 
         # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
@@ -56,14 +55,14 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
         # but here we are making straight requests, so we need todo convert this against apprise's logic
         for k, v in results['qsd'].items():
             if not k.strip('+-') in results['qsd+'].keys():
-                params[unquote_plus(k)] = unquote_plus(v)
+                params[URLBase.unquote(k)] = URLBase.unquote(v)
 
         # Determine Authentication
         auth = ''
         if results.get('user') and results.get('password'):
-            auth = (unquote_plus(results.get('user')), unquote_plus(results.get('user')))
+            auth = (URLBase.unquote(results.get('user')), URLBase.unquote(results.get('user')))
         elif results.get('user'):
-            auth = (unquote_plus(results.get('user')))
+            auth = (URLBase.unquote(results.get('user')))
 
     # Try to auto-guess if it's JSON
     h = 'application/json; charset=utf-8'

changedetectionio/blueprint/backups/__init__.py

@@ -1,164 +0,0 @@
-import datetime
-import glob
-import threading
-
-from flask import Blueprint, render_template, send_from_directory, flash, url_for, redirect, abort
-import os
-
-from changedetectionio.store import ChangeDetectionStore
-from changedetectionio.flask_app import login_optionally_required
-from loguru import logger
-
-BACKUP_FILENAME_FORMAT = "changedetection-backup-{}.zip"
-
-
-def create_backup(datastore_path, watches: dict):
-    logger.debug("Creating backup...")
-    import zipfile
-    from pathlib import Path
-
-    # create a ZipFile object
-    timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
-    backupname = BACKUP_FILENAME_FORMAT.format(timestamp)
-    backup_filepath = os.path.join(datastore_path, backupname)
-
-    with zipfile.ZipFile(backup_filepath.replace('.zip', '.tmp'), "w",
-                         compression=zipfile.ZIP_DEFLATED,
-                         compresslevel=8) as zipObj:
-
-        # Add the index
-        zipObj.write(os.path.join(datastore_path, "url-watches.json"), arcname="url-watches.json")
-
-        # Add the flask app secret
-        zipObj.write(os.path.join(datastore_path, "secret.txt"), arcname="secret.txt")
-
-        # Add any data in the watch data directory.
-        for uuid, w in watches.items():
-            for f in Path(w.watch_data_dir).glob('*'):
-                zipObj.write(f,
-                             # Use the full path to access the file, but make the file 'relative' in the Zip.
-                             arcname=os.path.join(f.parts[-2], f.parts[-1]),
-                             compress_type=zipfile.ZIP_DEFLATED,
-                             compresslevel=8)
-
-        # Create a list file with just the URLs, so it's easier to port somewhere else in the future
-        list_file = "url-list.txt"
-        with open(os.path.join(datastore_path, list_file), "w") as f:
-            for uuid in watches:
-                url = watches[uuid]["url"]
-                f.write("{}\r\n".format(url))
-        list_with_tags_file = "url-list-with-tags.txt"
-        with open(
-                os.path.join(datastore_path, list_with_tags_file), "w"
-        ) as f:
-            for uuid in watches:
-                url = watches[uuid].get('url')
-                tag = watches[uuid].get('tags', {})
-                f.write("{} {}\r\n".format(url, tag))
-
-        # Add it to the Zip
-        zipObj.write(
-            os.path.join(datastore_path, list_file),
-            arcname=list_file,
-            compress_type=zipfile.ZIP_DEFLATED,
-            compresslevel=8,
-        )
-        zipObj.write(
-            os.path.join(datastore_path, list_with_tags_file),
-            arcname=list_with_tags_file,
-            compress_type=zipfile.ZIP_DEFLATED,
-            compresslevel=8,
-        )
-
-    # Now it's done, rename it so it shows up finally and its completed being written.
-    os.rename(backup_filepath.replace('.zip', '.tmp'), backup_filepath.replace('.tmp', '.zip'))
-
-
-def construct_blueprint(datastore: ChangeDetectionStore):
-    backups_blueprint = Blueprint('backups', __name__, template_folder="templates")
-    backup_threads = []
-
-    @login_optionally_required
-    @backups_blueprint.route("/request-backup", methods=['GET'])
-    def request_backup():
-        if any(thread.is_alive() for thread in backup_threads):
-            flash("A backup is already running, check back in a few minutes", "error")
-            return redirect(url_for('backups.index'))
-
-        if len(find_backups()) > int(os.getenv("MAX_NUMBER_BACKUPS", 100)):
-            flash("Maximum number of backups reached, please remove some", "error")
-            return redirect(url_for('backups.index'))
-
-        # Be sure we're written fresh
-        datastore.sync_to_json()
-        zip_thread = threading.Thread(target=create_backup, args=(datastore.datastore_path, datastore.data.get("watching")))
-        zip_thread.start()
-        backup_threads.append(zip_thread)
-        flash("Backup building in background, check back in a few minutes.")
-
-        return redirect(url_for('backups.index'))
-
-    def find_backups():
-        backup_filepath = os.path.join(datastore.datastore_path, BACKUP_FILENAME_FORMAT.format("*"))
-        backups = glob.glob(backup_filepath)
-        backup_info = []
-
-        for backup in backups:
-            size = os.path.getsize(backup) / (1024 * 1024)
-            creation_time = os.path.getctime(backup)
-            backup_info.append({
-                'filename': os.path.basename(backup),
-                'filesize': f"{size:.2f}",
-                'creation_time': creation_time
-            })
-
-        backup_info.sort(key=lambda x: x['creation_time'], reverse=True)
-
-        return backup_info
-
-    @login_optionally_required
-    @backups_blueprint.route("/download/<string:filename>", methods=['GET'])
-    def download_backup(filename):
-        import re
-        filename = filename.strip()
-        backup_filename_regex = BACKUP_FILENAME_FORMAT.format("\d+")
-
-        full_path = os.path.join(os.path.abspath(datastore.datastore_path), filename)
-        if not full_path.startswith(os.path.abspath(datastore.datastore_path)):
-            abort(404)
-
-        if filename == 'latest':
-            backups = find_backups()
-            filename = backups[0]['filename']
-
-        if not re.match(r"^" + backup_filename_regex + "$", filename):
-            abort(400)  # Bad Request if the filename doesn't match the pattern
-
-        logger.debug(f"Backup download request for '{full_path}'")
-        return send_from_directory(os.path.abspath(datastore.datastore_path), filename, as_attachment=True)
-
-    @login_optionally_required
-    @backups_blueprint.route("/", methods=['GET'])
-    def index():
-        backups = find_backups()
-        output = render_template("overview.html",
-                                 available_backups=backups,
-                                 backup_running=any(thread.is_alive() for thread in backup_threads)
-                                 )
-
-        return output
-
-    @login_optionally_required
-    @backups_blueprint.route("/remove-backups", methods=['GET'])
-    def remove_backups():
-
-        backup_filepath = os.path.join(datastore.datastore_path, BACKUP_FILENAME_FORMAT.format("*"))
-        backups = glob.glob(backup_filepath)
-        for backup in backups:
-            os.unlink(backup)
-
-        flash("Backups were deleted.")
-
-        return redirect(url_for('backups.index'))
-
-    return backups_blueprint

changedetectionio/blueprint/backups/templates/overview.html

@@ -1,36 +0,0 @@
-{% extends 'base.html' %}
-{% block content %}
-    {% from '_helpers.html' import render_simple_field, render_field %}
-    <div class="edit-form">
-        <div class="box-wrap inner">
-            <h4>Backups</h4>
-            {% if backup_running %}
-                <p>
-                    <strong>A backup is running!</strong>
-                </p>
-            {% endif %}
-            <p>
-                Here you can download and request a new backup, when a backup is completed you will see it listed below.
-            </p>
-            <br>
-                {% if available_backups %}
-                    <ul>
-                    {% for backup in available_backups %}
-                        <li><a href="{{ url_for('backups.download_backup', filename=backup["filename"]) }}">{{ backup["filename"] }}</a> {{  backup["filesize"] }} Mb</li>
-                    {% endfor %}
-                    </ul>
-                {% else %}
-                    <p>
-                    <strong>No backups found.</strong>
-                    </p>
-                {% endif %}
-
-            <a class="pure-button pure-button-primary" href="{{ url_for('backups.request_backup') }}">Create backup</a>
-            {% if available_backups %}
-                <a class="pure-button button-small button-error " href="{{ url_for('backups.remove_backups') }}">Remove backups</a>
-            {% endif %}
-        </div>
-    </div>
-
-
-{% endblock %}

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -19,7 +19,7 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q: PriorityQueue
         datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_ACCEPT
         datastore.data['watching'][uuid]['processor'] = 'restock_diff'
         datastore.data['watching'][uuid].clear_watch()
-        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
         return redirect(url_for("index"))
 
     @login_required

changedetectionio/content_fetchers/res/stock-not-in-stock.js

@@ -30,8 +30,6 @@ function isItemInStock() {
         'dieser artikel ist bald wieder verfügbar',
         'dostępne wkrótce',
         'en rupture de stock',
-        'esgotado',
-        'indisponível',
         'isn\'t in stock right now',
         'isnt in stock right now',
         'isn’t in stock right now',
@@ -59,7 +57,6 @@ function isItemInStock() {
         'notify me when available',
         'notify me',
         'notify when available',
-        'não disponível',
         'não estamos a aceitar encomendas',
         'out of stock',
         'out-of-stock',

changedetectionio/flask_app.py

@@ -67,6 +67,7 @@ FlaskCompress(app)
 
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
+
 app.config.exit = Event()
 
 app.config['NEW_VERSION_AVAILABLE'] = False
@@ -469,7 +470,7 @@ def changedetection_app(config=None, datastore_o=None):
                     continue
             if watch.get('last_error'):
                 errored_count += 1
-
+                
             if search_q:
                 if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
                     sorted_watches.append(watch)
@@ -532,32 +533,24 @@ def changedetection_app(config=None, datastore_o=None):
     @login_optionally_required
     def ajax_callback_send_notification_test(watch_uuid=None):
 
-        # Watch_uuid could be unset in the case it`s used in tag editor, global settings
+        # Watch_uuid could be unset in the case its used in tag editor, global setings
         import apprise
         import random
         from .apprise_asset import asset
         apobj = apprise.Apprise(asset=asset)
-
         # so that the custom endpoints are registered
         from changedetectionio.apprise_plugin import apprise_custom_api_call_wrapper
         is_global_settings_form = request.args.get('mode', '') == 'global-settings'
         is_group_settings_form = request.args.get('mode', '') == 'group-settings'
 
         # Use an existing random one on the global/main settings form
-        if not watch_uuid and (is_global_settings_form or is_group_settings_form) \
-                and datastore.data.get('watching'):
+        if not watch_uuid and (is_global_settings_form or is_group_settings_form):
             logger.debug(f"Send test notification - Choosing random Watch {watch_uuid}")
             watch_uuid = random.choice(list(datastore.data['watching'].keys()))
 
-        if not watch_uuid:
-            return make_response("Error: You must have atleast one watch configured for 'test notification' to work", 400)
-
         watch = datastore.data['watching'].get(watch_uuid)
 
-        notification_urls = None
-
-        if request.form.get('notification_urls'):
-            notification_urls = request.form['notification_urls'].strip().splitlines()
+        notification_urls = request.form['notification_urls'].strip().splitlines()
 
         if not notification_urls:
             logger.debug("Test notification - Trying by group/tag in the edit form if available")
@@ -575,12 +568,12 @@ def changedetection_app(config=None, datastore_o=None):
 
 
         if not notification_urls:
-            return 'Error: No Notification URLs set/found'
+            return 'No Notification URLs set/found'
 
         for n_url in notification_urls:
             if len(n_url.strip()):
                 if not apobj.add(n_url):
-                    return f'Error:  {n_url} is not a valid AppRise URL.'
+                    return f'Error - {n_url} is not a valid AppRise URL.'
 
         try:
             # use the same as when it is triggered, but then override it with the form test values
@@ -599,13 +592,11 @@ def changedetection_app(config=None, datastore_o=None):
             if 'notification_body' in request.form and request.form['notification_body'].strip():
                 n_object['notification_body'] = request.form.get('notification_body', '').strip()
 
-            n_object.update(watch.extra_notification_token_values())
-
             from . import update_worker
             new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
             new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch)
         except Exception as e:
-            return make_response(f"Error: str(e)", 400)
+            return make_response({'error': str(e)}, 400)
 
         return 'OK - Sent test notifications'
 
@@ -803,9 +794,8 @@ def changedetection_app(config=None, datastore_o=None):
             # But in the case something is added we should save straight away
             datastore.needs_write_urgent = True
 
-            if not datastore.data['watching'][uuid].get('paused'):
-                # Queue the watch for immediate recheck, with a higher priority
-                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+            # Queue the watch for immediate recheck, with a higher priority
+            update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
 
             # Diff page [edit] link should go back to diff page
             if request.args.get("next") and request.args.get("next") == 'diff':
@@ -986,7 +976,7 @@ def changedetection_app(config=None, datastore_o=None):
                 importer = import_url_list()
                 importer.run(data=request.values.get('urls'), flash=flash, datastore=datastore, processor=request.values.get('processor', 'text_json_diff'))
                 for uuid in importer.new_uuids:
-                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
 
                 if len(importer.remaining_data) == 0:
                     return redirect(url_for('index'))
@@ -999,7 +989,7 @@ def changedetection_app(config=None, datastore_o=None):
                 d_importer = import_distill_io_json()
                 d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)
                 for uuid in d_importer.new_uuids:
-                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
 
             # XLSX importer
             if request.files and request.files.get('xlsx_file'):
@@ -1023,7 +1013,7 @@ def changedetection_app(config=None, datastore_o=None):
                     w_importer.run(data=file, flash=flash, datastore=datastore)
 
                 for uuid in w_importer.new_uuids:
-                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
 
         # Could be some remaining, or we could be on GET
         form = forms.importForm(formdata=request.form if request.method == 'POST' else None)
@@ -1236,6 +1226,78 @@ def changedetection_app(config=None, datastore_o=None):
 
         return output
 
+    # We're good but backups are even better!
+    @app.route("/backup", methods=['GET'])
+    @login_optionally_required
+    def get_backup():
+
+        import zipfile
+        from pathlib import Path
+
+        # Remove any existing backup file, for now we just keep one file
+
+        for previous_backup_filename in Path(datastore_o.datastore_path).rglob('changedetection-backup-*.zip'):
+            os.unlink(previous_backup_filename)
+
+        # create a ZipFile object
+        timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+        backupname = "changedetection-backup-{}.zip".format(timestamp)
+        backup_filepath = os.path.join(datastore_o.datastore_path, backupname)
+
+        with zipfile.ZipFile(backup_filepath, "w",
+                             compression=zipfile.ZIP_DEFLATED,
+                             compresslevel=8) as zipObj:
+
+            # Be sure we're written fresh
+            datastore.sync_to_json()
+
+            # Add the index
+            zipObj.write(os.path.join(datastore_o.datastore_path, "url-watches.json"), arcname="url-watches.json")
+
+            # Add the flask app secret
+            zipObj.write(os.path.join(datastore_o.datastore_path, "secret.txt"), arcname="secret.txt")
+
+            # Add any data in the watch data directory.
+            for uuid, w in datastore.data['watching'].items():
+                for f in Path(w.watch_data_dir).glob('*'):
+                    zipObj.write(f,
+                                 # Use the full path to access the file, but make the file 'relative' in the Zip.
+                                 arcname=os.path.join(f.parts[-2], f.parts[-1]),
+                                 compress_type=zipfile.ZIP_DEFLATED,
+                                 compresslevel=8)
+
+            # Create a list file with just the URLs, so it's easier to port somewhere else in the future
+            list_file = "url-list.txt"
+            with open(os.path.join(datastore_o.datastore_path, list_file), "w") as f:
+                for uuid in datastore.data["watching"]:
+                    url = datastore.data["watching"][uuid]["url"]
+                    f.write("{}\r\n".format(url))
+            list_with_tags_file = "url-list-with-tags.txt"
+            with open(
+                os.path.join(datastore_o.datastore_path, list_with_tags_file), "w"
+            ) as f:
+                for uuid in datastore.data["watching"]:
+                    url = datastore.data["watching"][uuid].get('url')
+                    tag = datastore.data["watching"][uuid].get('tags', {})
+                    f.write("{} {}\r\n".format(url, tag))
+
+            # Add it to the Zip
+            zipObj.write(
+                os.path.join(datastore_o.datastore_path, list_file),
+                arcname=list_file,
+                compress_type=zipfile.ZIP_DEFLATED,
+                compresslevel=8,
+            )
+            zipObj.write(
+                os.path.join(datastore_o.datastore_path, list_with_tags_file),
+                arcname=list_with_tags_file,
+                compress_type=zipfile.ZIP_DEFLATED,
+                compresslevel=8,
+            )
+
+        # Send_from_directory needs to be the full absolute path
+        return send_from_directory(os.path.abspath(datastore_o.datastore_path), backupname, as_attachment=True)
+
     @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
     def static_content(group, filename):
         from flask import make_response
@@ -1334,7 +1396,7 @@ def changedetection_app(config=None, datastore_o=None):
         url = request.form.get('url').strip()
         if datastore.url_exists(url):
             flash(f'Warning, URL {url} already exists', "notice")
-
+            
         add_paused = request.form.get('edit_and_watch_submit_button') != None
         processor = request.form.get('processor', 'text_json_diff')
         new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
@@ -1380,7 +1442,7 @@ def changedetection_app(config=None, datastore_o=None):
         new_uuid = datastore.clone(uuid)
         if new_uuid:
             if not datastore.data['watching'].get(uuid).get('paused'):
-                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid}))
+                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
             flash('Cloned.')
 
         return redirect(url_for('index'))
@@ -1401,7 +1463,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         if uuid:
             if uuid not in running_uuids:
-                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
             i = 1
 
         elif tag:
@@ -1412,7 +1474,7 @@ def changedetection_app(config=None, datastore_o=None):
                         continue
                     if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                         update_q.put(
-                            queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid})
+                            queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False})
                         )
                         i += 1
 
@@ -1422,7 +1484,7 @@ def changedetection_app(config=None, datastore_o=None):
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                     if with_errors and not watch.get('last_error'):
                         continue
-                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
                     i += 1
         flash(f"{i} watches queued for rechecking.")
         return redirect(url_for('index', tag=tag))
@@ -1480,7 +1542,7 @@ def changedetection_app(config=None, datastore_o=None):
                 uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
                     # Recheck and require a full reprocessing
-                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
             flash("{} watches queued for rechecking".format(len(uuids)))
 
         elif (op == 'clear-errors'):
@@ -1615,9 +1677,6 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.blueprint.check_proxies as check_proxies
     app.register_blueprint(check_proxies.construct_blueprint(datastore=datastore), url_prefix='/check_proxy')
 
-    import changedetectionio.blueprint.backups as backups
-    app.register_blueprint(backups.construct_blueprint(datastore), url_prefix='/backups')
-
 
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
@@ -1807,7 +1866,7 @@ def ticker_thread_check_time_launch_checks():
                         f"{now - watch['last_checked']:0.2f}s since last checked")
 
                     # Into the queue with you
-                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=priority, item={'uuid': uuid}))
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=priority, item={'uuid': uuid, 'skip_when_checksum_same': True}))
 
                     # Reset for next time
                     watch.jitter_seconds = 0

changedetectionio/forms.py

@@ -496,7 +496,7 @@ class processor_text_json_diff_form(commonSettingsForm):
     text_should_not_be_present = StringListField('Block change-detection while text matches', [validators.Optional(), ValidateListRegex()])
     webdriver_js_execute_code = TextAreaField('Execute JavaScript before change detection', render_kw={"rows": "5"}, validators=[validators.Optional()])
 
-    save_button = SubmitField('Save', render_kw={"class": "pure-button button-small pure-button-primary"})
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
 
     proxy = RadioField('Proxy')
     filter_failure_notification_send = BooleanField(
@@ -515,7 +515,6 @@ class processor_text_json_diff_form(commonSettingsForm):
         if not super().validate():
             return False
 
-        from changedetectionio.safe_jinja import render as jinja_render
         result = True
 
         # Fail form validation when a body is set for a GET
@@ -525,46 +524,18 @@ class processor_text_json_diff_form(commonSettingsForm):
 
         # Attempt to validate jinja2 templates in the URL
         try:
+            from changedetectionio.safe_jinja import render as jinja_render
             jinja_render(template_str=self.url.data)
         except ModuleNotFoundError as e:
             # incase jinja2_time or others is missing
             logger.error(e)
-            self.url.errors.append(f'Invalid template syntax configuration: {e}')
+            self.url.errors.append(e)
             result = False
         except Exception as e:
             logger.error(e)
-            self.url.errors.append(f'Invalid template syntax: {e}')
+            self.url.errors.append('Invalid template syntax')
             result = False
 
-        # Attempt to validate jinja2 templates in the body
-        if self.body.data and self.body.data.strip():
-            try:
-                jinja_render(template_str=self.body.data)
-            except ModuleNotFoundError as e:
-                # incase jinja2_time or others is missing
-                logger.error(e)
-                self.body.errors.append(f'Invalid template syntax configuration: {e}')
-                result = False
-            except Exception as e:
-                logger.error(e)
-                self.body.errors.append(f'Invalid template syntax: {e}')
-                result = False
-
-        # Attempt to validate jinja2 templates in the headers
-        if len(self.headers.data) > 0:
-            try:
-                for header, value in self.headers.data.items():
-                    jinja_render(template_str=value)
-            except ModuleNotFoundError as e:
-                # incase jinja2_time or others is missing
-                logger.error(e)
-                self.headers.errors.append(f'Invalid template syntax configuration: {e}')
-                result = False
-            except Exception as e:
-                logger.error(e)
-                self.headers.errors.append(f'Invalid template syntax in "{header}" header: {e}')
-                result = False
-
         return result
 
 class SingleExtraProxy(Form):
@@ -645,7 +616,7 @@ class globalSettingsForm(Form):
 
     requests = FormField(globalSettingsRequestForm)
     application = FormField(globalSettingsApplicationForm)
-    save_button = SubmitField('Save', render_kw={"class": "pure-button button-small pure-button-primary"})
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
 
 
 class extractDataForm(Form):

changedetectionio/html_tools.py

@@ -54,64 +54,29 @@ def include_filters(include_filters, html_content, append_pretty_line_formatting
 def subtractive_css_selector(css_selector, html_content):
     from bs4 import BeautifulSoup
     soup = BeautifulSoup(html_content, "html.parser")
-
-    # So that the elements dont shift their index, build a list of elements here which will be pointers to their place in the DOM
-    elements_to_remove = soup.select(css_selector)
-
-    # Then, remove them in a separate loop
-    for item in elements_to_remove:
+    for item in soup.select(css_selector):
         item.decompose()
-
     return str(soup)
 
-def subtractive_xpath_selector(selectors: List[str], html_content: str) -> str:
-    # Parse the HTML content using lxml
+def subtractive_xpath_selector(xpath_selector, html_content): 
     html_tree = etree.HTML(html_content)
+    elements_to_remove = html_tree.xpath(xpath_selector)
 
-    # First, collect all elements to remove
-    elements_to_remove = []
-
-    # Iterate over the list of XPath selectors
-    for selector in selectors:
-        # Collect elements for each selector
-        elements_to_remove.extend(html_tree.xpath(selector))
-
-    # Then, remove them in a separate loop
     for element in elements_to_remove:
-        if element.getparent() is not None:  # Ensure the element has a parent before removing
-            element.getparent().remove(element)
+        element.getparent().remove(element)
 
-    # Convert the modified HTML tree back to a string
     modified_html = etree.tostring(html_tree, method="html").decode("utf-8")
     return modified_html
 
-
 def element_removal(selectors: List[str], html_content):
-    """Removes elements that match a list of CSS or XPath selectors."""
+    """Removes elements that match a list of CSS or xPath selectors."""
     modified_html = html_content
-    css_selectors = []
-    xpath_selectors = []
-
     for selector in selectors:
         if selector.startswith(('xpath:', 'xpath1:', '//')):
-            # Handle XPath selectors separately
             xpath_selector = selector.removeprefix('xpath:').removeprefix('xpath1:')
-            xpath_selectors.append(xpath_selector)
+            modified_html = subtractive_xpath_selector(xpath_selector, modified_html)
         else:
-            # Collect CSS selectors as one "hit", see comment in subtractive_css_selector
-            css_selectors.append(selector.strip().strip(","))
-
-    if xpath_selectors:
-        modified_html = subtractive_xpath_selector(xpath_selectors, modified_html)
-
-    if css_selectors:
-        # Remove duplicates, then combine all CSS selectors into one string, separated by commas
-        # This stops the elements index shifting
-        unique_selectors = list(set(css_selectors))  # Ensure uniqueness
-        combined_css_selector = " , ".join(unique_selectors)
-        modified_html = subtractive_css_selector(combined_css_selector, modified_html)
-
-
+            modified_html = subtractive_css_selector(selector, modified_html)
     return modified_html
 
 def elementpath_tostring(obj):

changedetectionio/model/Watch.py

@@ -89,10 +89,6 @@ class model(watch_base):
 
         if ready_url.startswith('source:'):
             ready_url=ready_url.replace('source:', '')
-
-        # Also double check it after any Jinja2 formatting just incase
-        if not is_safe_url(ready_url):
-            return 'DISABLED'
         return ready_url
 
     def clear_watch(self):

changedetectionio/processors/__init__.py

@@ -31,15 +31,15 @@ class difference_detection_processor():
 
         from requests.structures import CaseInsensitiveDict
 
-        url = self.watch.link
-
-        # Protect against file:// access, check the real "link" without any meta "source:" etc prepended.
-        if re.search(r'^file://', url, re.IGNORECASE):
+        # Protect against file:// access
+        if re.search(r'^file://', self.watch.get('url', '').strip(), re.IGNORECASE):
             if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):
                 raise Exception(
                     "file:// type access is denied for security reasons."
                 )
 
+        url = self.watch.link
+
         # Requests, playwright, other browser via wss:// etc, fetch_extra_something
         prefer_fetch_backend = self.watch.get('fetch_backend', 'system')
 
@@ -102,7 +102,6 @@ class difference_detection_processor():
             self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))
 
         # Tweak the base config with the per-watch ones
-        from changedetectionio.safe_jinja import render as jinja_render
         request_headers = CaseInsensitiveDict()
 
         ua = self.datastore.data['settings']['requests'].get('default_ua')
@@ -119,15 +118,9 @@ class difference_detection_processor():
         if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
             request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
 
-        for header_name in request_headers:
-            request_headers.update({header_name: jinja_render(template_str=request_headers.get(header_name))})
-
         timeout = self.datastore.data['settings']['requests'].get('timeout')
 
         request_body = self.watch.get('body')
-        if request_body:
-            request_body = jinja_render(template_str=self.watch.get('body'))
-        
         request_method = self.watch.get('method')
         ignore_status_codes = self.watch.get('ignore_status_codes', False)
 
@@ -164,7 +157,7 @@ class difference_detection_processor():
         # After init, call run_changedetection() which will do the actual change-detection
 
     @abstractmethod
-    def run_changedetection(self, watch):
+    def run_changedetection(self, watch, skip_when_checksum_same: bool = True):
         update_obj = {'last_notification_error': False, 'last_error': False}
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()

changedetectionio/processors/restock_diff/processor.py

@@ -40,7 +40,7 @@ def _deduplicate_prices(data):
 
         if isinstance(datum.value, list):
             # Process each item in the list
-            normalized_value = set([float(re.sub(r'[^\d.]', '', str(item))) for item in datum.value if str(item).strip()])
+            normalized_value = set([float(re.sub(r'[^\d.]', '', str(item))) for item in datum.value])
             unique_data.update(normalized_value)
         else:
             # Process single value
@@ -144,7 +144,7 @@ class perform_site_check(difference_detection_processor):
     screenshot = None
     xpath_data = None
 
-    def run_changedetection(self, watch):
+    def run_changedetection(self, watch, skip_when_checksum_same=True):
         import hashlib
 
         if not watch:

changedetectionio/processors/text_json_diff/__init__.py

@@ -11,7 +11,10 @@ def _task(watch, update_handler):
 
     try:
         # The slow process (we run 2 of these in parallel)
-        changed_detected, update_obj, text_after_filter = update_handler.run_changedetection(watch=watch)
+        changed_detected, update_obj, text_after_filter = update_handler.run_changedetection(
+            watch=watch,
+            skip_when_checksum_same=False,
+        )
     except FilterNotFoundInResponse as e:
         text_after_filter = f"Filter not found in HTML: {str(e)}"
     except ReplyWithContentButNoText as e:

changedetectionio/processors/text_json_diff/processor.py

@@ -35,7 +35,7 @@ class PDFToHTMLToolNotFound(ValueError):
 # (set_proxy_from_list)
 class perform_site_check(difference_detection_processor):
 
-    def run_changedetection(self, watch):
+    def run_changedetection(self, watch, skip_when_checksum_same=True):
         changed_detected = False
         html_content = ""
         screenshot = False  # as bytes
@@ -58,6 +58,9 @@ class perform_site_check(difference_detection_processor):
         # Watches added automatically in the queue manager will skip if its the same checksum as the previous run
         # Saves a lot of CPU
         update_obj['previous_md5_before_filters'] = hashlib.md5(self.fetcher.content.encode('utf-8')).hexdigest()
+        if skip_when_checksum_same:
+            if update_obj['previous_md5_before_filters'] == watch.get('previous_md5_before_filters'):
+                raise content_fetchers.exceptions.checksumFromPreviousCheckWasTheSame()
 
         # Fetching complete, now filters
 
@@ -331,21 +334,13 @@ class perform_site_check(difference_detection_processor):
             if result:
                 blocked = True
 
+        # The main thing that all this at the moment comes down to :)
+        if watch.get('previous_md5') != fetched_md5:
+            changed_detected = True
 
         # Looks like something changed, but did it match all the rules?
         if blocked:
             changed_detected = False
-        else:
-            # The main thing that all this at the moment comes down to :)
-            if watch.get('previous_md5') != fetched_md5:
-                changed_detected = True
-
-            # Always record the new checksum
-            update_obj["previous_md5"] = fetched_md5
-
-            # On the first run of a site, watch['previous_md5'] will be None, set it the current one.
-            if not watch.get('previous_md5'):
-                watch['previous_md5'] = fetched_md5
 
         logger.debug(f"Watch UUID {watch.get('uuid')} content check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
 
@@ -365,6 +360,12 @@ class perform_site_check(difference_detection_processor):
                 else:
                     logger.debug(f"check_unique_lines: UUID {watch.get('uuid')} had unique content")
 
+        # Always record the new checksum
+        update_obj["previous_md5"] = fetched_md5
+
+        # On the first run of a site, watch['previous_md5'] will be None, set it the current one.
+        if not watch.get('previous_md5'):
+            watch['previous_md5'] = fetched_md5
 
         # stripped_text_from_html - Everything after filters and NO 'ignored' content
         return changed_detected, update_obj, stripped_text_from_html

changedetectionio/static/js/notifications.js

@@ -28,14 +28,17 @@ $(document).ready(function() {
       url: notification_base_url,
       data : data,
         statusCode: {
-        400: function(data) {
-          // More than likely the CSRF token was lost when the server restarted
-          alert(data.responseText);
+        400: function() {
+            // More than likely the CSRF token was lost when the server restarted
+          alert("There was a problem processing the request, please reload the page.");
         }
       }
     }).done(function(data){
       console.log(data);
       alert(data);
+    }).fail(function(data){
+      console.log(data);
+      alert('There was an error communicating with the server.');
     })
   });
 });

changedetectionio/static/js/tabs.js

@@ -26,7 +26,8 @@ function set_active_tab() {
     if (tab.length) {
         tab[0].parentElement.className = "active";
     }
-
+    // hash could move the page down
+    window.scrollTo(0, 0);
 }
 
 function focus_error_tab() {

changedetectionio/static/styles/diff.css

@@ -153,8 +153,7 @@ html[data-darkmode="true"] {
     border: 1px solid transparent;
     vertical-align: top;
     font: 1em monospace;
-    text-align: left;
-    overflow: clip; }
+    text-align: left; }
   #diff-ui pre {
     white-space: pre-wrap; }
 
@@ -173,9 +172,7 @@ ins {
   text-decoration: none; }
 
 #result {
-  white-space: pre-wrap;
-  word-break: break-word;
-  overflow-wrap: break-word; }
+  white-space: pre-wrap; }
 
 #settings {
   background: rgba(0, 0, 0, 0.05);
@@ -234,12 +231,3 @@ td#diff-col div {
   border-radius: 5px;
   background: var(--color-background);
   box-shadow: 1px 1px 4px var(--color-shadow-jump); }
-
-.pure-form button.reset-margin {
-  margin: 0px; }
-
-.diff-fieldset {
-  display: flex;
-  align-items: center;
-  gap: 4px;
-  flex-wrap: wrap; }

changedetectionio/static/styles/scss/diff.scss

@@ -24,7 +24,6 @@
     vertical-align: top;
     font: 1em monospace;
     text-align: left;
-    overflow: clip; // clip overflowing contents to cell boundariess
   }
 
   pre {
@@ -51,8 +50,6 @@ ins {
 
 #result {
   white-space: pre-wrap;
-  word-break: break-word;
-  overflow-wrap: break-word;
 
   .change {
     span {}
@@ -137,15 +134,3 @@ td#diff-col div {
   background: var(--color-background);
   box-shadow: 1px 1px 4px var(--color-shadow-jump);
 }
-
-// resets button margin to 0px
-.pure-form button.reset-margin {
-  margin: 0px;
-}
-
-.diff-fieldset {
-  display: flex;
-  align-items: center;
-  gap: 4px;
-  flex-wrap: wrap;
-}

changedetectionio/static/styles/scss/parts/_extra_browsers.scss

@@ -11,22 +11,7 @@ ul#requests-extra_browsers {
   /* each proxy entry is a `table` */
   table {
     tr {
-      display: table-row; // default display for small screens
-      input[type=text] {
-        width: 100%;
-      }
-    }
-  }
-  
-  // apply inline display for larger screens
-  @media only screen and (min-width: 1280px) {
-    table {
-      tr {
-        display: inline;
-        input[type=text] {
-          width: 100%;
-        }
-      }
+      display: inline;
     }
   }
 }

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -11,19 +11,7 @@ ul#requests-extra_proxies {
   /* each proxy entry is a `table` */
   table {
     tr {
-      display: table-row; // default display for small screens
-      input[type=text] {
-        width: 100%;
-      }
-    }
-  }
-  
-  // apply inline display for large screens
-  @media only screen and (min-width: 1024px) {
-    table {
-      tr {
-        display: inline;
-      }
+      display: inline;
     }
   }
 }

changedetectionio/static/styles/scss/styles.scss

@@ -147,14 +147,8 @@ body.spinner-active {
   }
 }
 
-
-.tab-pane-inner {
-  // .tab-pane-inner will have the #id that the tab button jumps/anchors to
-  scroll-margin-top: 200px;
-}
-
 section.content {
-  padding-top: 100px;
+  padding-top: 5em;
   padding-bottom: 1em;
   flex-direction: column;
   display: flex;
@@ -937,7 +931,6 @@ $form-edge-padding: 20px;
 }
 
 .tab-pane-inner {
-
   &:not(:target) {
     display: none;
   }

changedetectionio/static/styles/styles.css

@@ -112,12 +112,7 @@ ul#requests-extra_proxies {
   ul#requests-extra_proxies li > label {
     display: none; }
   ul#requests-extra_proxies table tr {
-    display: table-row; }
-    ul#requests-extra_proxies table tr input[type=text] {
-      width: 100%; }
-  @media only screen and (min-width: 1024px) {
-    ul#requests-extra_proxies table tr {
-      display: inline; } }
+    display: inline; }
 
 #request {
   /* Auto proxy scan/checker */ }
@@ -166,14 +161,7 @@ ul#requests-extra_browsers {
   ul#requests-extra_browsers li > label {
     display: none; }
   ul#requests-extra_browsers table tr {
-    display: table-row; }
-    ul#requests-extra_browsers table tr input[type=text] {
-      width: 100%; }
-  @media only screen and (min-width: 1280px) {
-    ul#requests-extra_browsers table tr {
-      display: inline; }
-      ul#requests-extra_browsers table tr input[type=text] {
-        width: 100%; } }
+    display: inline; }
 
 #extra-browsers-setting {
   border: 1px solid var(--color-grey-800);
@@ -617,11 +605,8 @@ body.spinner-active #pure-menu-horizontal-spinner {
     background-color: var(--color-background-menu-link-hover);
     color: var(--color-text-menu-link-hover); }
 
-.tab-pane-inner {
-  scroll-margin-top: 200px; }
-
 section.content {
-  padding-top: 100px;
+  padding-top: 5em;
   padding-bottom: 1em;
   flex-direction: column;
   display: flex;

changedetectionio/templates/base.html

@@ -70,7 +70,7 @@
                 <a href="{{ url_for('import_page')}}" class="pure-menu-link">IMPORT</a>
               </li>
               <li class="pure-menu-item">
-                <a href="{{ url_for('backups.index')}}" class="pure-menu-link">BACKUPS</a>
+                <a href="{{ url_for('get_backup')}}" class="pure-menu-link">BACKUP</a>
               </li>
             {% else %}
               <li class="pure-menu-item">

changedetectionio/templates/diff.html

@@ -14,7 +14,7 @@
 
 <div id="settings">
     <form class="pure-form " action="" method="GET" id="diff-form">
-        <fieldset class="diff-fieldset">
+        <fieldset>
             {% if versions|length >= 1 %}
                 <strong>Compare</strong>
                 <del class="change"><span>from</span></del>
@@ -33,7 +33,7 @@
                         </option>
                     {% endfor %}
                 </select>
-                <button type="submit" class="pure-button pure-button-primary reset-margin">Go</button>
+                <button type="submit" class="pure-button pure-button-primary">Go</button>
             {% endif %}
         </fieldset>
         <fieldset>

changedetectionio/templates/edit.html

@@ -65,8 +65,8 @@
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
-                        <div class="pure-form-message">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></div>
-                        <div class="pure-form-message">Variables are supported in the URL (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a>).</div>
+                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span><br>
+                        <span class="pure-form-message-inline">You can use variables in the URL, perfect for inserting the current date and other logic, <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a></span><br>
                     </div>
                     <div class="pure-control-group inline-radio">
                         {{ render_field(form.processor) }}
@@ -149,24 +149,21 @@
                             {{ render_field(form.method) }}
                         </div>
                         <div id="request-body">
-                                            {{ render_field(form.body, rows=7, placeholder="Example
+                                            {{ render_field(form.body, rows=5, placeholder="Example
 {
    \"name\":\"John\",
    \"age\":30,
-   \"car\":null,
-   \"year\":{% now 'Europe/Berlin', '%Y' %}
+   \"car\":null
 }") }}
                         </div>
-                        <div class="pure-form-message">Variables are supported in the request body (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a>).</div>
                     </div>
                 </fieldset>
             <!-- hmm -->
                 <div class="pure-control-group advanced-options"  style="display: none;">
-                    {{ render_field(form.headers, rows=7, placeholder="Example
+                    {{ render_field(form.headers, rows=5, placeholder="Example
 Cookie: foobar
-User-Agent: wonderbra 1.0
-Math: {{ 1 + 1 }}") }}
-                        <div class="pure-form-message">Variables are supported in the request header values (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a>).</div>
+User-Agent: wonderbra 1.0") }}
+
                         <div class="pure-form-message-inline">
                             {% if has_extra_headers_file %}
                                 <strong>Alert! Extra headers file found and will be added to this watch!</strong>

changedetectionio/templates/settings.html

@@ -276,7 +276,7 @@ nav
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
                     <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
-                    <a href="{{url_for('clear_all_history')}}" class="pure-button button-small button-error">Clear Snapshot History</a>
+                    <a href="{{url_for('clear_all_history')}}" class="pure-button button-small button-cancel">Clear Snapshot History</a>
                 </div>
             </div>
         </form>

changedetectionio/tests/proxy_list/test_proxy.py

@@ -16,4 +16,4 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     )
 
     assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
+    time.sleep(3)

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -1,8 +1,7 @@
 #!/usr/bin/env python3
-import json
 import os
 from flask import url_for
-from changedetectionio.tests.util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
+from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
 
 
 def set_response():
@@ -19,6 +18,7 @@ def set_response():
         f.write(data)
     time.sleep(1)
 
+
 def test_socks5(client, live_server, measure_memory_usage):
     live_server_setup(live_server)
     set_response()
@@ -79,24 +79,3 @@ def test_socks5(client, live_server, measure_memory_usage):
 
     # Should see the proper string
     assert "Awesome, you made it".encode('utf-8') in res.data
-
-    # PROXY CHECKER WIDGET CHECK - this needs more checking
-    uuid = extract_UUID_from_client(client)
-
-    res = client.get(
-        url_for("check_proxies.start_check", uuid=uuid),
-        follow_redirects=True
-    )
-    # It's probably already finished super fast :(
-    #assert b"RUNNING" in res.data
-    
-    wait_for_all_checks(client)
-    res = client.get(
-        url_for("check_proxies.get_recheck_status", uuid=uuid),
-        follow_redirects=True
-    )
-    assert b"OK" in res.data
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-

changedetectionio/tests/test_backup.py

@@ -26,24 +26,8 @@ def test_backup(client, live_server, measure_memory_usage):
     assert b"1 Imported" in res.data
     wait_for_all_checks(client)
 
-    # Launch the thread in the background to create the backup
     res = client.get(
-        url_for("backups.request_backup"),
-        follow_redirects=True
-    )
-    time.sleep(2)
-
-    res = client.get(
-        url_for("backups.index"),
-        follow_redirects=True
-    )
-    # Can see the download link to the backup
-    assert b'<a href="/backups/download/changedetection-backup-20' in res.data
-    assert b'Remove backups' in res.data
-
-    # Get the latest one
-    res = client.get(
-        url_for("backups.download_backup", filename="latest"),
+        url_for("get_backup"),
         follow_redirects=True
     )
 
@@ -60,11 +44,3 @@ def test_backup(client, live_server, measure_memory_usage):
 
     # Should be two txt files in the archive (history and the snapshot)
     assert len(newlist) == 2
-
-    # Get the latest one
-    res = client.get(
-        url_for("backups.remove_backups"),
-        follow_redirects=True
-    )
-
-    assert b'No backups found.' in res.data

changedetectionio/tests/test_block_while_text_present.py

@@ -65,8 +65,11 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     live_server_setup(live_server)
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "out of stoCk\r\nfoobar"
+
     set_original_ignore_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -124,24 +127,13 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
-    # 2548
-    # Going back to the ORIGINAL should NOT trigger a change
-    set_original_ignore_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'unviewed' not in res.data
 
-
-    # Now we set a change where the text is gone AND its different content, it should now trigger
+    # Now we set a change where the text is gone, it should now trigger
     set_modified_response_minus_block_text()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
-
-
-
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_element_removal.py

@@ -11,35 +11,6 @@ from .util import live_server_setup, wait_for_all_checks
 def test_setup(live_server):
     live_server_setup(live_server)
 
-def set_response_with_multiple_index():
-    data= """<!DOCTYPE html>
-<html>
-<body>
-
-<!-- NOTE!! CHROME WILL ADD TBODY HERE IF ITS NOT THERE!! -->
-<table style="width:100%">
-  <tr>
-    <th>Person 1</th>
-    <th>Person 2</th>
-    <th>Person 3</th>
-  </tr>
-  <tr>
-    <td>Emil</td>
-    <td>Tobias</td>
-    <td>Linus</td>
-  </tr>
-  <tr>
-    <td>16</td>
-    <td>14</td>
-    <td>10</td>
-  </tr>
-</table>
-</body>
-</html>
-"""
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(data)
-
 
 def set_original_response():
     test_return_data = """<html>
@@ -206,61 +177,3 @@ def test_element_removal_full(client, live_server, measure_memory_usage):
     # There should not be an unviewed change, as changes should be removed
     res = client.get(url_for("index"))
     assert b"unviewed" not in res.data
-
-# Re #2752
-def test_element_removal_nth_offset_no_shift(client, live_server, measure_memory_usage):
-    #live_server_setup(live_server)
-
-    set_response_with_multiple_index()
-    subtractive_selectors_data = ["""
-body > table > tr:nth-child(1) > th:nth-child(2)
-body > table >  tr:nth-child(2) > td:nth-child(2)
-body > table > tr:nth-child(3) > td:nth-child(2)
-body > table > tr:nth-child(1) > th:nth-child(3)
-body > table >  tr:nth-child(2) > td:nth-child(3)
-body > table > tr:nth-child(3) > td:nth-child(3)""",
-"""//body/table/tr[1]/th[2]
-//body/table/tr[2]/td[2]
-//body/table/tr[3]/td[2]
-//body/table/tr[1]/th[3]
-//body/table/tr[2]/td[3]
-//body/table/tr[3]/td[3]"""]
-
-    for selector_list in subtractive_selectors_data:
-
-        res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-        assert b'Deleted' in res.data
-
-        # Add our URL to the import page
-        test_url = url_for("test_endpoint", _external=True)
-        res = client.post(
-            url_for("import_page"), data={"urls": test_url}, follow_redirects=True
-        )
-        assert b"1 Imported" in res.data
-        wait_for_all_checks(client)
-
-        res = client.post(
-            url_for("edit_page", uuid="first"),
-            data={
-                "subtractive_selectors": selector_list,
-                "url": test_url,
-                "tags": "",
-                "fetch_backend": "html_requests",
-            },
-            follow_redirects=True,
-        )
-        assert b"Updated watch." in res.data
-        wait_for_all_checks(client)
-
-        res = client.get(
-            url_for("preview_page", uuid="first"),
-            follow_redirects=True
-        )
-
-        assert b"Tobias" not in res.data
-        assert b"Linus" not in res.data
-        assert b"Person 2" not in res.data
-        assert b"Person 3" not in res.data
-        # First column should exist
-        assert b"Emil" in res.data
-

changedetectionio/tests/test_live_preview.py

@@ -1,78 +0,0 @@
-#!/usr/bin/env python3
-
-from flask import url_for
-from changedetectionio.tests.util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
-
-
-def set_response():
-
-    data = f"""<html>
-       <body>Awesome, you made it<br>
-yeah the socks request worked<br>
-something to ignore<br>
-something to trigger<br>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(data)
-
-def test_content_filter_live_preview(client, live_server, measure_memory_usage):
-    live_server_setup(live_server)
-    set_response()
-
-    test_url = url_for('test_endpoint', _external=True)
-
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": ''},
-        follow_redirects=True
-    )
-    uuid = extract_UUID_from_client(client)
-    res = client.post(
-        url_for("edit_page", uuid=uuid),
-        data={
-            "include_filters": "",
-            "fetch_backend": 'html_requests',
-            "ignore_text": "something to ignore",
-            "trigger_text": "something to trigger",
-            "url": test_url,
-        },
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-
-    # The endpoint is a POST and accepts the form values to override the watch preview
-    import json
-
-    # DEFAULT OUTPUT WITHOUT ANYTHING UPDATED/CHANGED - SHOULD SEE THE WATCH DEFAULTS
-    res = client.post(
-        url_for("watch_get_preview_rendered", uuid=uuid)
-    )
-    default_return = json.loads(res.data.decode('utf-8'))
-    assert default_return.get('after_filter')
-    assert default_return.get('before_filter')
-    assert default_return.get('ignore_line_numbers') == [3] # "something to ignore" line 3
-    assert default_return.get('trigger_line_numbers') == [4] # "something to trigger" line 4
-
-    # SEND AN UPDATE AND WE SHOULD SEE THE OUTPUT CHANGE SO WE KNOW TO HIGHLIGHT NEW STUFF
-    res = client.post(
-        url_for("watch_get_preview_rendered", uuid=uuid),
-        data={
-            "include_filters": "",
-            "fetch_backend": 'html_requests',
-            "ignore_text": "sOckS", # Also be sure case insensitive works
-            "trigger_text": "AweSOme",
-            "url": test_url,
-        },
-    )
-    reply = json.loads(res.data.decode('utf-8'))
-    assert reply.get('after_filter')
-    assert reply.get('before_filter')
-    assert reply.get('ignore_line_numbers') == [2]  # Ignored - "socks" on line 2
-    assert reply.get('trigger_line_numbers') == [1]  # Triggers "Awesome" in line 1
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_notification.py

@@ -284,7 +284,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     # CUSTOM JSON BODY CHECK for POST://
     set_original_response()
     # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#header-manipulation
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22"
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}&+custom-header=123"
 
     res = client.post(
         url_for("settings_page"),
@@ -326,7 +326,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
         assert j['secret'] == 444
         assert j['somebug'] == '网站监测 内容更新了'
 
-
     # URL check, this will always be converted to lowercase
     assert os.path.isfile("test-datastore/notification-url.txt")
     with open("test-datastore/notification-url.txt", 'r') as f:
@@ -338,7 +337,6 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me
     with open("test-datastore/notification-headers.txt", 'r') as f:
         notification_headers = f.read()
         assert 'custom-header: 123' in notification_headers.lower()
-        assert 'second: hello world "space"' in notification_headers.lower()
 
 
     # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
@@ -431,15 +429,3 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage
         follow_redirects=True
     )
 
-    ######### Test global/system settings - When everything is deleted it should give a helpful error
-    # See #2727
-    res = client.post(
-        url_for("ajax_callback_send_notification_test")+"?mode=global-settings",
-        data={"notification_urls": test_notification_url},
-        follow_redirects=True
-    )
-    assert res.status_code == 400
-    assert b"Error: You must have atleast one watch configured for 'test notification' to work" in res.data
-
-
-

changedetectionio/tests/test_request.py

@@ -45,7 +45,7 @@ def test_headers_in_request(client, live_server, measure_memory_usage):
               "url": test_url,
               "tags": "",
               "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
-              "headers": "jinja2:{{ 1+1 }}\nxxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
+              "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -61,7 +61,6 @@ def test_headers_in_request(client, live_server, measure_memory_usage):
     )
 
     # Flask will convert the header key to uppercase
-    assert b"Jinja2:2" in res.data
     assert b"Xxx:ooo" in res.data
     assert b"Cool:yeah" in res.data
 
@@ -118,8 +117,7 @@ def test_body_in_request(client, live_server, measure_memory_usage):
     wait_for_all_checks(client)
 
     # Now the change which should trigger a change
-    body_value = 'Test Body Value {{ 1+1 }}'
-    body_value_formatted = 'Test Body Value 2'
+    body_value = 'Test Body Value'
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
@@ -142,9 +140,8 @@ def test_body_in_request(client, live_server, measure_memory_usage):
 
     # If this gets stuck something is wrong, something should always be there
     assert b"No history found" not in res.data
-    # We should see the formatted value of what we sent in the reply
-    assert str.encode(body_value) not in res.data
-    assert str.encode(body_value_formatted) in res.data
+    # We should see what we sent in the reply
+    assert str.encode(body_value) in res.data
 
     ####### data sanity checks
     # Add the test URL twice, we will check

changedetectionio/tests/test_restock_itemprop.py

@@ -3,7 +3,7 @@ import os
 import time
 
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output, extract_UUID_from_client
+from .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output
 from ..notification import default_notification_format
 
 instock_props = [
@@ -367,12 +367,6 @@ def test_change_with_notification_values(client, live_server):
         assert "new price 1950.45" in notification
         assert "title new price 1950.45" in notification
 
-    ## Now test the "SEND TEST NOTIFICATION" is working
-    os.unlink("test-datastore/notification.txt")
-    uuid = extract_UUID_from_client(client)
-    res = client.post(url_for("ajax_callback_send_notification_test", watch_uuid=uuid), data={}, follow_redirects=True)
-    time.sleep(5)
-    assert os.path.isfile("test-datastore/notification.txt"), "Notification received"
 
 
 def test_data_sanity(client, live_server):

changedetectionio/update_worker.py

@@ -81,8 +81,7 @@ class update_worker(threading.Thread):
             'watch_url': watch.get('url') if watch else None,
         })
 
-        if watch:
-            n_object.update(watch.extra_notification_token_values())
+        n_object.update(watch.extra_notification_token_values())
 
         logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
         logger.debug("Queued notification for sending")
@@ -261,6 +260,9 @@ class update_worker(threading.Thread):
                     try:
                         # Processor is what we are using for detecting the "Change"
                         processor = watch.get('processor', 'text_json_diff')
+                        # Abort processing when the content was the same as the last fetch
+                        skip_when_same_checksum = queued_item_data.item.get('skip_when_checksum_same')
+
 
                         # Init a new 'difference_detection_processor', first look in processors
                         processor_module_name = f"changedetectionio.processors.{processor}.processor"
@@ -276,7 +278,10 @@ class update_worker(threading.Thread):
 
                         update_handler.call_browser()
 
-                        changed_detected, update_obj, contents = update_handler.run_changedetection(watch=watch)
+                        changed_detected, update_obj, contents = update_handler.run_changedetection(
+                            watch=watch,
+                            skip_when_checksum_same=skip_when_same_checksum,
+                        )
 
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.

requirements.txt

@@ -59,9 +59,7 @@ elementpath==4.1.5
 
 selenium~=4.14.0
 
-# https://github.com/pallets/werkzeug/issues/2985
-# Maybe related to pytest?
-werkzeug==3.0.6
+werkzeug~=3.0
 
 # Templating, so far just in the URLs but in the future can be for the notifications also
 jinja2~=3.1