did not need this in its own block

Dockerfile

@@ -20,11 +20,6 @@ COPY requirements.txt /requirements.txt
 
 RUN pip install --target=/dependencies -r /requirements.txt
 
-# Playwright is an alternative to Selenium
-# Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
-RUN pip install --target=/dependencies playwright~=1.20 \
-    || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
-
 # Final image stage
 FROM python:3.8-slim
 

README.md

@@ -9,7 +9,7 @@ _Know when web pages change! Stay ontop of new information!_
 
 Live your data-life *pro-actively* instead of *re-actively*.
 
-Free, Open-source web page monitoring, notification and change detection. Don't have time? [**Try our $6.99/month subscription - unlimited checks and watches!**](https://lemonade.changedetection.io/start)
+Free, Open-source web page monitoring, notification and change detection. Don't have time? [Try our $6.99/month plan - unlimited checks and watches!](https://lemonade.changedetection.io/start)
 
 
 [<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start)
@@ -17,7 +17,10 @@ Free, Open-source web page monitoring, notification and change detection. Don't
 
 **Get your own private instance now! Let us host it for you!**
 
-[**Try our $6.99/month subscription - unlimited checks and watches!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
+[![Deploy to Lemonade](https://lemonade.changedetection.io/static/images/lemonade.svg)](https://lemonade.changedetection.io/start)
+
+
+[_Let us host your own private instance - We accept PayPal and Bitcoin, Support the further development of changedetection.io!_](https://lemonade.changedetection.io/start)
 
 
 
@@ -36,14 +39,13 @@ Free, Open-source web page monitoring, notification and change detection. Don't
 - COVID related news from government websites
 - University/organisation news from their website
 - Detect and monitor changes in JSON API responses 
-- JSON API monitoring and alerting
+- API monitoring and alerting
 - Changes in legal and other documents
 - Trigger API calls via notifications when text appears on a website
 - Glue together APIs using the JSON filter and JSON notifications
 - Create RSS feeds based on changes in web content
-- Monitor HTML source code for unexpected changes, strengthen your PCI compliance
 - You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
-
+ 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver!</a>_
 
 ## Screenshots
@@ -170,12 +172,9 @@ Raspberry Pi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! See the
 
 Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 
+Please support us, even small amounts help a LOT.
 
-Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://lemonade.changedetection.io/start) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
+BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
-
-Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)
-
-Or BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />
 

changedetectionio/__init__.py

@@ -32,7 +32,6 @@ from flask import (
     render_template,
     request,
     send_from_directory,
-    session,
     url_for,
 )
 from flask_login import login_required
@@ -95,6 +94,16 @@ def init_app_secret(datastore_path):
 
     return secret
 
+# Remember python is by reference
+# populate_form in wtfors didnt work for me. (try using a setattr() obj type on datastore.watch?)
+def populate_form_from_watch(form, watch):
+    for i in form.__dict__.keys():
+        if i[0] != '_':
+            p = getattr(form, i)
+            if hasattr(p, 'data') and i in watch:
+                setattr(p, "data", watch[i])
+
+
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
 # running or something similar.
 @app.template_filter('format_last_checked_time')
@@ -311,7 +320,6 @@ def changedetection_app(config=None, datastore_o=None):
                 guid = "{}/{}".format(watch['uuid'], watch['last_changed'])
                 fe = fg.add_entry()
 
-
                 # Include a link to the diff page, they will have to login here to see if password protection is enabled.
                 # Description is the page you watch, link takes you to the diff JS UI page
                 base_url = datastore.data['settings']['application']['base_url']
@@ -343,8 +351,6 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/", methods=['GET'])
     @login_required
     def index():
-        from changedetectionio import forms
-
         limit_tag = request.args.get('tag')
         pause_uuid = request.args.get('pause')
 
@@ -381,6 +387,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         existing_tags = datastore.get_all_tags()
 
+        from changedetectionio import forms
         form = forms.quickWatchForm(request.form)
 
         output = render_template("watch-overview.html",
@@ -392,10 +399,8 @@ def changedetection_app(config=None, datastore_o=None):
                                  has_unviewed=datastore.data['has_unviewed'],
                                  # Don't link to hosting when we're on the hosting environment
                                  hosted_sticky=os.getenv("SALTED_PASS", False) == False,
-                                 guid=datastore.data['app_guid'],
+                                 guid=datastore.data['app_guid'])
-                                 queued_uuids=update_q.queue)
+
-        if session.get('share-link'):
-            del(session['share-link'])
         return output
 
 
@@ -434,21 +439,48 @@ def changedetection_app(config=None, datastore_o=None):
     @login_required
     def scrub_page():
 
+        import re
+
         if request.method == 'POST':
             confirmtext = request.form.get('confirmtext')
+            limit_date = request.form.get('limit_date')
+            limit_timestamp = 0
+
+            # Re #149 - allow empty/0 timestamp limit
+            if len(limit_date):
+                try:
+                    limit_date = limit_date.replace('T', ' ')
+                    # I noticed chrome will show '/' but actually submit '-'
+                    limit_date = limit_date.replace('-', '/')
+                    # In the case that :ss seconds are supplied
+                    limit_date = re.sub(r'(\d\d:\d\d)(:\d\d)', '\\1', limit_date)
+
+                    str_to_dt = datetime.datetime.strptime(limit_date, '%Y/%m/%d %H:%M')
+                    limit_timestamp = int(str_to_dt.timestamp())
+
+                    if limit_timestamp > time.time():
+                        flash("Timestamp is in the future, cannot continue.", 'error')
+                        return redirect(url_for('scrub_page'))
+
+                except ValueError:
+                    flash('Incorrect date format, cannot continue.', 'error')
+                    return redirect(url_for('scrub_page'))
 
             if confirmtext == 'scrub':
                 changes_removed = 0
-                for uuid in datastore.data['watching'].keys():
+                for uuid, watch in datastore.data['watching'].items():
-                    datastore.scrub_watch(uuid)
+                    if limit_timestamp:
+                        changes_removed += datastore.scrub_watch(uuid, limit_timestamp=limit_timestamp)
+                    else:
+                        changes_removed += datastore.scrub_watch(uuid)
 
-                flash("Cleared all snapshot history")
+                flash("Cleared snapshot history ({} snapshots removed)".format(changes_removed))
             else:
                 flash('Incorrect confirmation text.', 'error')
 
             return redirect(url_for('index'))
 
-        output = render_template("scrub.html")
+        output =  render_template("scrub.html")
         return output
 
 
@@ -488,58 +520,49 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
     @login_required
-    # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists
-    # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ?
-
     def edit_page(uuid):
         from changedetectionio import forms
+        form = forms.watchForm(request.form)
 
-        using_default_check_time = True
         # More for testing, possible to return the first/only
-        if not datastore.data['watching'].keys():
-            flash("No watches to edit", "error")
-            return redirect(url_for('index'))
-
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
-        if not uuid in datastore.data['watching']:
-            flash("No watch with the UUID %s found." % (uuid), "error")
-            return redirect(url_for('index'))
 
-        # be sure we update with a copy instead of accidently editing the live object by reference
+        if request.method == 'GET':
-        default = deepcopy(datastore.data['watching'][uuid])
+            if not uuid in datastore.data['watching']:
+                flash("No watch with the UUID %s found." % (uuid), "error")
+                return redirect(url_for('index'))
 
-        # Show system wide default if nothing configured
+            populate_form_from_watch(form, datastore.data['watching'][uuid])
-        if datastore.data['watching'][uuid]['fetch_backend'] is None:
-            default['fetch_backend'] = datastore.data['settings']['application']['fetch_backend']
-
-        # Show system wide default if nothing configured
-        if all(value == 0 or value == None for value in datastore.data['watching'][uuid]['time_between_check'].values()):
-            default['time_between_check'] = deepcopy(datastore.data['settings']['requests']['time_between_check'])
-
-        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                                        data=default
-                                        )
 
+            if datastore.data['watching'][uuid]['fetch_backend'] is None:
+                form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
 
         if request.method == 'POST' and form.validate():
-            extra_update_obj = {}
 
             # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
-            # Assume we use the default value, unless something relevant is different, then use the form value
+            if form.minutes_between_check.data == datastore.data['settings']['requests']['minutes_between_check']:
-            # values could be None, 0 etc.
+                form.minutes_between_check.data = None
-            # Set to None unless the next for: says that something is different
-            extra_update_obj['time_between_check'] = dict.fromkeys(form.time_between_check.data)
-            for k, v in form.time_between_check.data.items():
-                if v and v != datastore.data['settings']['requests']['time_between_check'][k]:
-                    extra_update_obj['time_between_check'] = form.time_between_check.data
-                    using_default_check_time = False
-                    break
 
-            # Use the default if its the same as system wide
             if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']:
-                extra_update_obj['fetch_backend'] = None
+                form.fetch_backend.data = None
+
+            update_obj = {'url': form.url.data.strip(),
+                          'minutes_between_check': form.minutes_between_check.data,
+                          'tag': form.tag.data.strip(),
+                          'title': form.title.data.strip(),
+                          'headers': form.headers.data,
+                          'body': form.body.data,
+                          'method': form.method.data,
+                          'ignore_status_codes': form.ignore_status_codes.data,
+                          'fetch_backend': form.fetch_backend.data,
+                          'trigger_text': form.trigger_text.data,
+                          'notification_title': form.notification_title.data,
+                          'notification_body': form.notification_body.data,
+                          'notification_format': form.notification_format.data,
+                          'extract_title_as_title': form.extract_title_as_title.data,
+                          }
 
             # Notification URLs
             datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data
@@ -551,21 +574,24 @@ def changedetection_app(config=None, datastore_o=None):
             # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
             if form_ignore_text:
                 if len(datastore.data['watching'][uuid]['history']):
-                    extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
+                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
+
+
+            datastore.data['watching'][uuid]['css_filter'] = form.css_filter.data.strip()
+            datastore.data['watching'][uuid]['subtractive_selectors'] = form.subtractive_selectors.data
 
             # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
             if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']:
                 if len(datastore.data['watching'][uuid]['history']):
-                    extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
+                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
-            datastore.data['watching'][uuid].update(form.data)
+            datastore.data['watching'][uuid].update(update_obj)
-            datastore.data['watching'][uuid].update(extra_update_obj)
 
             flash("Updated watch.")
 
             # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
             # But in the case something is added we should save straight away
-            datastore.needs_write_urgent = True
+            datastore.sync_to_json()
 
             # Queue the watch for immediate recheck
             update_q.put(uuid)
@@ -584,12 +610,17 @@ def changedetection_app(config=None, datastore_o=None):
             if request.method == 'POST' and not form.validate():
                 flash("An error occurred, please see below.", "error")
 
+            # Re #110 offer the default minutes
+            using_default_minutes = False
+            if form.minutes_between_check.data == None:
+                form.minutes_between_check.data = datastore.data['settings']['requests']['minutes_between_check']
+                using_default_minutes = True
 
             output = render_template("edit.html",
                                      uuid=uuid,
                                      watch=datastore.data['watching'][uuid],
                                      form=form,
-                                     has_empty_checktime=using_default_check_time,
+                                     using_default_minutes=using_default_minutes,
                                      current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)
                                      )
@@ -599,39 +630,61 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/settings", methods=['GET', "POST"])
     @login_required
     def settings_page():
+
         from changedetectionio import content_fetcher, forms
 
-        # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status
+        form = forms.globalSettingsForm(request.form)
-        form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,
-                                        data=datastore.data['settings']
-                                        )
 
-        if request.method == 'POST':
+        if request.method == 'GET':
+            form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check'])
+            form.notification_urls.data = datastore.data['settings']['application']['notification_urls']
+            form.global_subtractive_selectors.data = datastore.data['settings']['application']['global_subtractive_selectors']
+            form.global_ignore_text.data = datastore.data['settings']['application']['global_ignore_text']
+            form.ignore_whitespace.data = datastore.data['settings']['application']['ignore_whitespace']
+            form.render_anchor_tag_content.data = datastore.data['settings']['application']['render_anchor_tag_content']
+            form.extract_title_as_title.data = datastore.data['settings']['application']['extract_title_as_title']
+            form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
+            form.notification_title.data = datastore.data['settings']['application']['notification_title']
+            form.notification_body.data = datastore.data['settings']['application']['notification_body']
+            form.notification_format.data = datastore.data['settings']['application']['notification_format']
+            form.base_url.data = datastore.data['settings']['application']['base_url']
+            form.real_browser_save_screenshot.data = datastore.data['settings']['application']['real_browser_save_screenshot']
+
+        if request.method == 'POST' and form.data.get('removepassword_button') == True:
             # Password unset is a GET, but we can lock the session to a salted env password to always need the password
-            if form.application.form.data.get('removepassword_button', False):
+            if not os.getenv("SALTED_PASS", False):
-                # SALTED_PASS means the password is "locked" to what we set in the Env var
+                datastore.data['settings']['application']['password'] = False
-                if not os.getenv("SALTED_PASS", False):
+                flash("Password protection removed.", 'notice')
-                    datastore.remove_password()
+                flask_login.logout_user()
-                    flash("Password protection removed.", 'notice')
+                return redirect(url_for('settings_page'))
-                    flask_login.logout_user()
-                    return redirect(url_for('settings_page'))
 
-            if form.validate():
+        if request.method == 'POST' and form.validate():
-                datastore.data['settings']['application'].update(form.data['application'])
+            datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
-                datastore.data['settings']['requests'].update(form.data['requests'])
+            datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data
+            datastore.data['settings']['application']['extract_title_as_title'] = form.extract_title_as_title.data
+            datastore.data['settings']['application']['fetch_backend'] = form.fetch_backend.data
+            datastore.data['settings']['application']['notification_title'] = form.notification_title.data
+            datastore.data['settings']['application']['notification_body'] = form.notification_body.data
+            datastore.data['settings']['application']['notification_format'] = form.notification_format.data
+            datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
+            datastore.data['settings']['application']['base_url'] = form.base_url.data
+            datastore.data['settings']['application']['global_subtractive_selectors'] = form.global_subtractive_selectors.data
+            datastore.data['settings']['application']['global_ignore_text'] =  form.global_ignore_text.data
+            datastore.data['settings']['application']['ignore_whitespace'] = form.ignore_whitespace.data
+            datastore.data['settings']['application']['real_browser_save_screenshot'] = form.real_browser_save_screenshot.data
+            datastore.data['settings']['application']['render_anchor_tag_content'] = form.render_anchor_tag_content.data
 
-                if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password):
+            if not os.getenv("SALTED_PASS", False) and form.password.encrypted_password:
-                    datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password
+                datastore.data['settings']['application']['password'] = form.password.encrypted_password
-                    datastore.needs_write_urgent = True
+                flash("Password protection enabled.", 'notice')
-                    flash("Password protection enabled.", 'notice')
+                flask_login.logout_user()
-                    flask_login.logout_user()
+                return redirect(url_for('index'))
-                    return redirect(url_for('index'))
 
-                datastore.needs_write_urgent = True
+            datastore.needs_write = True
-                flash("Settings updated.")
+            flash("Settings updated.")
 
-            else:
+        if request.method == 'POST' and not form.validate():
-                flash("An error occurred, please see below.", "error")
+            flash("An error occurred, please see below.", "error")
 
         output = render_template("settings.html",
                                  form=form,
@@ -650,30 +703,21 @@ def changedetection_app(config=None, datastore_o=None):
         good = 0
 
         if request.method == 'POST':
-            now=time.time()
             urls = request.values.get('urls').split("\n")
-
-            if (len(urls) > 5000):
-                flash("Importing 5,000 of the first URLs from your list, the rest can be imported again.")
-
             for url in urls:
                 url = url.strip()
                 url, *tags = url.split(" ")
                 # Flask wtform validators wont work with basic auth, use validators package
-                # Up to 5000 per batch so we dont flood the server
+                if len(url) and validators.url(url):
-                if len(url) and validators.url(url.replace('source:', '')) and good < 5000:
+                    new_uuid = datastore.add_watch(url=url.strip(), tag=" ".join(tags))
-                    new_uuid = datastore.add_watch(url=url.strip(), tag=" ".join(tags), write_to_disk_now=False)
+                    # Straight into the queue.
-                    if new_uuid:
+                    update_q.put(new_uuid)
-                        # Straight into the queue.
+                    good += 1
-                        update_q.put(new_uuid)
+                else:
-                        good += 1
+                    if len(url):
-                        continue
+                        remaining_urls.append(url)
 
-                if len(url.strip()):
+            flash("{} Imported, {} Skipped.".format(good, len(remaining_urls)))
-                    remaining_urls.append(url)
-
-            flash("{} Imported in {:.2f}s, {} Skipped.".format(good, time.time()-now,len(remaining_urls)))
-            datastore.needs_write = True
 
             if len(remaining_urls) == 0:
                 # Looking good, redirect to index.
@@ -977,35 +1021,29 @@ def changedetection_app(config=None, datastore_o=None):
         from changedetectionio import forms
         form = forms.quickWatchForm(request.form)
 
-        if not form.validate():
+        if form.validate():
-            flash("Error")
-            return redirect(url_for('index'))
 
-        url = request.form.get('url').strip()
+            url = request.form.get('url').strip()
-        if datastore.url_exists(url):
+            if datastore.url_exists(url):
-            flash('The URL {} already exists'.format(url), "error")
+                flash('The URL {} already exists'.format(url), "error")
-            return redirect(url_for('index'))
+                return redirect(url_for('index'))
 
-        # @todo add_watch should throw a custom Exception for validation etc
+            # @todo add_watch should throw a custom Exception for validation etc
-        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
+            new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
-        if new_uuid:
             # Straight into the queue.
             update_q.put(new_uuid)
+
             flash("Watch added.")
-
+            return redirect(url_for('index'))
-        return redirect(url_for('index'))
+        else:
-
+            flash("Error")
+            return redirect(url_for('index'))
 
 
     @app.route("/api/delete", methods=['GET'])
     @login_required
     def api_delete():
         uuid = request.args.get('uuid')
-
-        if uuid != 'all' and not uuid in datastore.data['watching'].keys():
-            flash('The watch by UUID {} does not exist.'.format(uuid), 'error')
-            return redirect(url_for('index'))
-
         # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
@@ -1065,59 +1103,6 @@ def changedetection_app(config=None, datastore_o=None):
         flash("{} watches are queued for rechecking.".format(i))
         return redirect(url_for('index', tag=tag))
 
-    @app.route("/api/share-url", methods=['GET'])
-    @login_required
-    def api_share_put_watch():
-        """Given a watch UUID, upload the info and return a share-link
-           the share-link can be imported/added"""
-        import requests
-        import json
-        tag = request.args.get('tag')
-        uuid = request.args.get('uuid')
-
-        # more for testing
-        if uuid == 'first':
-            uuid = list(datastore.data['watching'].keys()).pop()
-
-        # copy it to memory as trim off what we dont need (history)
-        watch = deepcopy(datastore.data['watching'][uuid])
-        if (watch.get('history')):
-            del (watch['history'])
-
-        # for safety/privacy
-        for k in list(watch.keys()):
-            if k.startswith('notification_'):
-                del watch[k]
-
-        for r in['uuid', 'last_checked', 'last_changed']:
-            if watch.get(r):
-                del (watch[r])
-
-        # Add the global stuff which may have an impact
-        watch['ignore_text'] += datastore.data['settings']['application']['global_ignore_text']
-        watch['subtractive_selectors'] += datastore.data['settings']['application']['global_subtractive_selectors']
-
-        watch_json = json.dumps(watch)
-
-        try:
-            r = requests.request(method="POST",
-                                 data={'watch': watch_json},
-                                 url="https://changedetection.io/share/share",
-                                 headers={'App-Guid': datastore.data['app_guid']})
-            res = r.json()
-
-            session['share-link'] = "https://changedetection.io/share/{}".format(res['share_key'])
-
-
-        except Exception as e:
-            flash("Could not share, something went wrong while communicating with the share server.", 'error')
-
-        # https://changedetection.io/share/VrMv05wpXyQa
-        # in the browser - should give you a nice info page - wtf
-        # paste in etc
-        return redirect(url_for('index'))
-
-
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
 
@@ -1187,6 +1172,8 @@ def notification_runner():
                 notification_debug_log = notification_debug_log[-100:]
 
 
+
+
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
     from changedetectionio import update_worker
@@ -1223,9 +1210,7 @@ def ticker_thread_check_time_launch_checks():
 
         # Check for watches outside of the time threshold to put in the thread queue.
         now = time.time()
-
+        max_system_wide = int(copied_datastore.data['settings']['requests']['minutes_between_check']) * 60
-        recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
-        recheck_time_system_seconds = datastore.threshold_seconds
 
         for uuid, watch in copied_datastore.data['watching'].items():
 
@@ -1234,15 +1219,18 @@ def ticker_thread_check_time_launch_checks():
                 continue
 
             # If they supplied an individual entry minutes to threshold.
-            threshold = now
+            watch_minutes_between_check = watch.get('minutes_between_check', None)
-            watch_threshold_seconds = watch.threshold_seconds()
+            if watch_minutes_between_check is not None:
-            if watch_threshold_seconds:
+                # Cast to int just incase
-                threshold -= watch_threshold_seconds
+                max_time = int(watch_minutes_between_check) * 60
             else:
-                threshold -= recheck_time_system_seconds
+                # Default system wide.
+                max_time = max_system_wide
+
+            threshold = now - max_time
 
             # Yeah, put it in the queue, it's more than time
-            if watch['last_checked'] <= max(threshold, recheck_time_minimum_seconds):
+            if watch['last_checked'] <= threshold:
                 if not uuid in running_uuids and uuid not in update_q.queue:
                     update_q.put(uuid)
 
@@ -1250,4 +1238,4 @@ def ticker_thread_check_time_launch_checks():
         time.sleep(3)
 
         # Should be low so we can break this out in testing
-        app.config.exit.wait(1)
+        app.config.exit.wait(1)

changedetectionio/changedetection.py

@@ -8,7 +8,7 @@ import sys
 
 import eventlet
 import eventlet.wsgi
-from . import store, changedetection_app, content_fetcher
+from . import store, changedetection_app
 from . import __version__
 
 def main():

changedetectionio/content_fetcher.py

@@ -1,10 +1,13 @@
 from abc import ABC, abstractmethod
 import chardet
 import os
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
+from selenium.common.exceptions import WebDriverException
 import requests
 import time
 import urllib3.exceptions
-import sys
 
 
 class EmptyReply(Exception):
@@ -16,17 +19,13 @@ class EmptyReply(Exception):
 
     pass
 
-
 class Fetcher():
     error = None
     status_code = None
     content = None
     headers = None
-    # Will be needed in the future by the VisualSelector, always get this where possible.
+
-    screenshot = False
+    fetcher_description ="No description"
-    fetcher_description = "No description"
-    system_http_proxy = os.getenv('HTTP_PROXY')
-    system_https_proxy = os.getenv('HTTPS_PROXY')
 
     @abstractmethod
     def get_error(self):
@@ -47,6 +46,10 @@ class Fetcher():
     def quit(self):
         return
 
+    @abstractmethod
+    def screenshot(self):
+        return
+
     @abstractmethod
     def get_last_status_code(self):
         return self.status_code
@@ -56,105 +59,29 @@ class Fetcher():
     def is_ready(self):
         return True
 
-
 #   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
 #   the current one would return javascript output (as we use JS to generate the diff)
 #
+#   Returns tuple(mime_type, stream)
+#    @abstractmethod
+#    def return_diff(self, stream_a, stream_b):
+#        return
+
 def available_fetchers():
-    # See the if statement at the bottom of this file for how we switch between playwright and webdriver
+        import inspect
-    import inspect
+        from changedetectionio import content_fetcher
-    p = []
+        p=[]
-    for name, obj in inspect.getmembers(sys.modules[__name__], inspect.isclass):
+        for name, obj in inspect.getmembers(content_fetcher):
-        if inspect.isclass(obj):
+            if inspect.isclass(obj):
-            # @todo html_ is maybe better as fetcher_ or something
+                # @todo html_ is maybe better as fetcher_ or something
-            # In this case, make sure to edit the default one in store.py and fetch_site_status.py
+                # In this case, make sure to edit the default one in store.py and fetch_site_status.py
-            if name.startswith('html_'):
+                if "html_" in name:
-                t = tuple([name, obj.fetcher_description])
+                    t=tuple([name,obj.fetcher_description])
-                p.append(t)
+                    p.append(t)
 
-    return p
+        return p
 
-
+class html_webdriver(Fetcher):
-class base_html_playwright(Fetcher):
-    fetcher_description = "Playwright {}/Javascript".format(
-        os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
-    )
-    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
-        fetcher_description += " via '{}'".format(os.getenv("PLAYWRIGHT_DRIVER_URL"))
-
-    browser_type = ''
-    command_executor = ''
-
-    # Configs for Proxy setup
-    # In the ENV vars, is prefixed with "playwright_proxy_", so it is for example "playwright_proxy_server"
-    playwright_proxy_settings_mappings = ['bypass', 'server', 'username', 'password']
-
-    proxy = None
-
-    def __init__(self):
-        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
-        self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
-        self.command_executor = os.getenv(
-            "PLAYWRIGHT_DRIVER_URL",
-            'ws://playwright-chrome:3000/playwright'
-        ).strip('"')
-
-        # If any proxy settings are enabled, then we should setup the proxy object
-        proxy_args = {}
-        for k in self.playwright_proxy_settings_mappings:
-            v = os.getenv('playwright_proxy_' + k, False)
-            if v:
-                proxy_args[k] = v.strip('"')
-
-        if proxy_args:
-            self.proxy = proxy_args
-
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False):
-
-        from playwright.sync_api import sync_playwright
-
-        with sync_playwright() as p:
-            browser_type = getattr(p, self.browser_type)
-
-            # Seemed to cause a connection Exception even tho I can see it connect
-            # self.browser = browser_type.connect(self.command_executor, timeout=timeout*1000)
-            browser = browser_type.connect_over_cdp(self.command_executor, timeout=timeout * 1000)
-
-            # Set user agent to prevent Cloudflare from blocking the browser
-            # Use the default one configured in the App.py model that's passed from fetch_site_status.py
-            context = browser.new_context(
-                user_agent=request_headers['User-Agent'] if request_headers.get('User-Agent') else 'Mozilla/5.0',
-                proxy=self.proxy
-            )
-            page = context.new_page()
-            page.set_viewport_size({"width": 1280, "height": 1024})
-            response = page.goto(url, timeout=timeout * 1000)
-
-            extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))
-            page.wait_for_timeout(extra_wait * 1000)
-
-            if response is None:
-                raise EmptyReply(url=url, status_code=None)
-
-            self.status_code = response.status
-            self.content = page.content()
-            self.headers = response.all_headers()
-
-            # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
-            # JPEG is better here because the screenshots can be very very large
-            page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024})
-            self.screenshot = page.screenshot(type='jpeg', full_page=True, quality=90)
-            context.close()
-            browser.close()
-
-
-class base_html_webdriver(Fetcher):
     if os.getenv("WEBDRIVER_URL"):
         fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))
     else:
@@ -167,11 +94,12 @@ class base_html_webdriver(Fetcher):
     selenium_proxy_settings_mappings = ['proxyType', 'ftpProxy', 'httpProxy', 'noProxy',
                                         'proxyAutoconfigUrl', 'sslProxy', 'autodetect',
                                         'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
-    proxy = None
+
+
+
+    proxy=None
 
     def __init__(self):
-        from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
-
         # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
         self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
 
@@ -182,12 +110,6 @@ class base_html_webdriver(Fetcher):
             if v:
                 proxy_args[k] = v.strip('"')
 
-        # Map back standard HTTP_ and HTTPS_PROXY to webDriver httpProxy/sslProxy
-        if not proxy_args.get('webdriver_httpProxy') and self.system_http_proxy:
-            proxy_args['httpProxy'] = self.system_http_proxy
-        if not proxy_args.get('webdriver_sslProxy') and self.system_https_proxy:
-            proxy_args['httpsProxy'] = self.system_https_proxy
-
         if proxy_args:
             self.proxy = SeleniumProxy(raw=proxy_args)
 
@@ -199,9 +121,6 @@ class base_html_webdriver(Fetcher):
             request_method,
             ignore_status_codes=False):
 
-        from selenium import webdriver
-        from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
-        from selenium.common.exceptions import WebDriverException
         # request_body, request_method unused for now, until some magic in the future happens.
 
         # check env for WEBDRIVER_URL
@@ -226,8 +145,9 @@ class base_html_webdriver(Fetcher):
         time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
         self.content = self.driver.page_source
         self.headers = {}
-        self.screenshot = self.driver.get_screenshot_as_png()
+
-        self.quit()
+    def screenshot(self):
+        return self.driver.get_screenshot_as_png()
 
     # Does the connection to the webdriver work? run a test connection.
     def is_ready(self):
@@ -250,7 +170,6 @@ class base_html_webdriver(Fetcher):
             except Exception as e:
                 print("Exception in chrome shutdown/quit" + str(e))
 
-
 # "html_requests" is listed as the default fetcher in store.py!
 class html_requests(Fetcher):
     fetcher_description = "Basic fast Plaintext/HTTP Client"
@@ -263,20 +182,12 @@ class html_requests(Fetcher):
             request_method,
             ignore_status_codes=False):
 
-        # Map back standard HTTP_ and HTTPS_PROXY to requests http/https proxy
-        proxies={}
-        if self.system_http_proxy:
-            proxies['http'] = self.system_http_proxy
-        if self.system_https_proxy:
-            proxies['https'] = self.system_https_proxy
-
         r = requests.request(method=request_method,
-                             data=request_body,
+                         data=request_body,
-                             url=url,
+                         url=url,
-                             headers=request_headers,
+                         headers=request_headers,
-                             timeout=timeout,
+                         timeout=timeout,
-                             proxies=proxies,
+                         verify=False)
-                             verify=False)
 
         # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
         # For example - some sites don't tell us it's utf-8, but return utf-8 content
@@ -296,11 +207,3 @@ class html_requests(Fetcher):
         self.content = r.text
         self.headers = r.headers
 
-
-# Decide which is the 'real' HTML webdriver, this is more a system wide config
-# rather than site-specific.
-use_playwright_as_chrome_fetcher = os.getenv('PLAYWRIGHT_DRIVER_URL', False)
-if use_playwright_as_chrome_fetcher:
-    html_webdriver = base_html_playwright
-else:
-    html_webdriver = base_html_webdriver

changedetectionio/fetch_site_status.py

@@ -20,7 +20,7 @@ class perform_site_check():
         timestamp = int(time.time())  # used for storage etc too
 
         changed_detected = False
-        screenshot = False  # as bytes
+        screenshot = False # as bytes
         stripped_text_from_html = ""
 
         watch = self.datastore.data['watching'][uuid]
@@ -52,12 +52,6 @@ class perform_site_check():
         request_method = self.datastore.get_val(uuid, 'method')
         ignore_status_code = self.datastore.get_val(uuid, 'ignore_status_codes')
 
-        # source: support
-        is_source = False
-        if url.startswith('source:'):
-            url = url.replace('source:', '')
-            is_source = True
-
         # Pluggable content fetcher
         prefer_backend = watch['fetch_backend']
         if hasattr(content_fetcher, prefer_backend):
@@ -66,9 +60,9 @@ class perform_site_check():
             # If the klass doesnt exist, just use a default
             klass = getattr(content_fetcher, "html_requests")
 
+
         fetcher = klass()
         fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code)
-
         # Fetching complete, now filters
         # @todo move to class / maybe inside of fetcher abstract base?
 
@@ -81,12 +75,6 @@ class perform_site_check():
 
         is_json = 'application/json' in fetcher.headers.get('Content-Type', '')
         is_html = not is_json
-
-        # source: support, basically treat it as plaintext
-        if is_source:
-            is_html = False
-            is_json = False
-
         css_filter_rule = watch['css_filter']
         subtractive_selectors = watch.get(
             "subtractive_selectors", []
@@ -106,7 +94,7 @@ class perform_site_check():
                 stripped_text_from_html = html_tools.extract_json_as_string(content=fetcher.content, jsonpath_filter=css_filter_rule)
                 is_html = False
 
-        if is_html or is_source:
+        if is_html:
             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
             html_content = fetcher.content
 
@@ -125,24 +113,15 @@ class perform_site_check():
                         html_content = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
                 if has_subtractive_selectors:
                     html_content = html_tools.element_removal(subtractive_selectors, html_content)
-
+                # extract text
-                if not is_source:
+                stripped_text_from_html = \
-                    # extract text
+                    html_tools.html_to_text(
-                    stripped_text_from_html = \
+                        html_content,
-                        html_tools.html_to_text(
+                        render_anchor_tag_content=self.datastore.data["settings"][
-                            html_content,
+                            "application"].get(
-                            render_anchor_tag_content=self.datastore.data["settings"][
+                            "render_anchor_tag_content", False)
-                                "application"].get(
+                    )
-                                "render_anchor_tag_content", False)
+                
-                        )
-
-                elif is_source:
-                    stripped_text_from_html = html_content
-
-            # Re #340 - return the content before the 'ignore text' was applied
-            text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
-
-
         # Re #340 - return the content before the 'ignore text' was applied
         text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
@@ -165,8 +144,8 @@ class perform_site_check():
         else:
             fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest()
 
-        # On the first run of a site, watch['previous_md5'] will be None, set it the current one.
+        # On the first run of a site, watch['previous_md5'] will be an empty string, set it the current one.
-        if not watch.get('previous_md5'):
+        if not len(watch['previous_md5']):
             watch['previous_md5'] = fetched_md5
             update_obj["previous_md5"] = fetched_md5
 
@@ -182,15 +161,22 @@ class perform_site_check():
             if result:
                 blocked_by_not_found_trigger_text = False
 
+
         if not blocked_by_not_found_trigger_text and watch['previous_md5'] != fetched_md5:
             changed_detected = True
             update_obj["previous_md5"] = fetched_md5
             update_obj["last_changed"] = timestamp
 
+
         # Extract title as title
         if is_html:
             if self.datastore.data['settings']['application']['extract_title_as_title'] or watch['extract_title_as_title']:
                 if not watch['title'] or not len(watch['title']):
                     update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
 
-        return changed_detected, update_obj, text_content_before_ignored_filter, fetcher.screenshot
+        if self.datastore.data['settings']['application'].get('real_browser_save_screenshot', True):
+            screenshot = fetcher.screenshot()
+
+        fetcher.quit()
+
+        return changed_detected, update_obj, text_content_before_ignored_filter, screenshot

changedetectionio/forms.py

@@ -25,8 +25,6 @@ from changedetectionio.notification import (
     valid_notification_formats,
 )
 
-from wtforms.fields import FormField
-
 valid_method = {
     'GET',
     'POST',
@@ -37,31 +35,27 @@ valid_method = {
 
 default_method = 'GET'
 
-
 class StringListField(StringField):
     widget = widgets.TextArea()
 
     def _value(self):
         if self.data:
-            # ignore empty lines in the storage
+            return "\r\n".join(self.data)
-            data = list(filter(lambda x: len(x.strip()), self.data))
-            # Apply strip to each line
-            data = list(map(lambda x: x.strip(), data))
-            return "\r\n".join(data)
         else:
             return u''
 
     # incoming
     def process_formdata(self, valuelist):
-        if valuelist and len(valuelist[0].strip()):
+        if valuelist:
-            # Remove empty strings, stripping and splitting \r\n, only \n etc.
+            # Remove empty strings
-            self.data = valuelist[0].splitlines()
+            cleaned = list(filter(None, valuelist[0].split("\n")))
-            # Remove empty lines from the final data
+            self.data = [x.strip() for x in cleaned]
-            self.data = list(filter(lambda x: len(x.strip()), self.data))
+            p = 1
         else:
             self.data = []
 
 
+
 class SaltyPasswordField(StringField):
     widget = widgets.PasswordInput()
     encrypted_password = ""
@@ -89,13 +83,6 @@ class SaltyPasswordField(StringField):
         else:
             self.data = False
 
-class TimeBetweenCheckForm(Form):
-    weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
-    days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
-    hours = IntegerField('Hours', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
-    minutes = IntegerField('Minutes', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
-    seconds = IntegerField('Seconds', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
-    # @todo add total seconds minimum validatior = minimum_seconds_recheck_time
 
 # Separated by  key:value
 class StringDictKeyValue(StringField):
@@ -134,6 +121,7 @@ class ValidateContentFetcherIsReady(object):
 
     def __call__(self, form, field):
         import urllib3.exceptions
+
         from changedetectionio import content_fetcher
 
         # Better would be a radiohandler that keeps a reference to each class
@@ -309,7 +297,6 @@ class quickWatchForm(Form):
     url = fields.URLField('URL', validators=[validateURL()])
     tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)])
 
-# Common to a single watch and the global settings
 class commonSettingsForm(Form):
 
     notification_urls = StringListField('Notification URL list', validators=[validators.Optional(), ValidateNotificationBodyAndTitleWhenURLisSet(), ValidateAppRiseServers()])
@@ -324,7 +311,8 @@ class watchForm(commonSettingsForm):
     url = fields.URLField('URL', validators=[validateURL()])
     tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)], default='')
 
-    time_between_check = FormField(TimeBetweenCheckForm)
+    minutes_between_check = fields.IntegerField('Maximum time in minutes until recheck',
+                                               [validators.Optional(), validators.NumberRange(min=1)])
 
     css_filter = StringField('CSS/JSON/XPATH Filter', [ValidateCSSJSONXPATHInput()], default='')
 
@@ -354,32 +342,19 @@ class watchForm(commonSettingsForm):
 
         return result
 
-
+class globalSettingsForm(commonSettingsForm):
-# datastore.data['settings']['requests']..
+    password = SaltyPasswordField()
-class globalSettingsRequestForm(Form):
+    minutes_between_check = fields.IntegerField('Maximum time in minutes until recheck',
-    time_between_check = FormField(TimeBetweenCheckForm)
+                                               [validators.NumberRange(min=1)])
-
+    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title')
-
-# datastore.data['settings']['application']..
-class globalSettingsApplicationForm(commonSettingsForm):
-
     base_url = StringField('Base URL', validators=[validators.Optional()])
     global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
-    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
+    global_ignore_text = StringListField('Ignore text', [ValidateListRegex()])
     ignore_whitespace = BooleanField('Ignore whitespace')
-    real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome?')
-    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
-    render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
-    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
-    password = SaltyPasswordField()
 
+    render_anchor_tag_content = BooleanField('Render anchor tag content',
+                                             default=False)
 
-class globalSettingsForm(Form):
-    # Define these as FormFields/"sub forms", this way it matches the JSON storage
-    # datastore.data['settings']['application']..
-    # datastore.data['settings']['requests']..
-
-    requests = FormField(globalSettingsRequestForm)
-    application = FormField(globalSettingsApplicationForm)
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-
+    real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome')
+    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})

changedetectionio/model/App.py

@@ -1,50 +0,0 @@
-import collections
-import os
-
-import uuid as uuid_builder
-
-from changedetectionio.notification import (
-    default_notification_body,
-    default_notification_format,
-    default_notification_title,
-)
-
-class model(dict):
-    base_config = {
-            'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
-            'watching': {},
-            'settings': {
-                'headers': {
-                    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
-                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
-                    'Accept-Encoding': 'gzip, deflate',  # No support for brolti in python requests yet.
-                    'Accept-Language': 'en-GB,en-US;q=0.9,en;'
-                },
-                'requests': {
-                    'timeout': 15,  # Default 15 seconds
-                    'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
-                    'workers': 10  # Number of threads, lower is better for slow connections
-                },
-                'application': {
-                    'password': False,
-                    'base_url' : None,
-                    'extract_title_as_title': False,
-                    'fetch_backend': os.getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
-                    'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
-                    'global_subtractive_selectors': [],
-                    'ignore_whitespace': False,
-                    'render_anchor_tag_content': False,
-                    'notification_urls': [], # Apprise URL list
-                    # Custom notification content
-                    'notification_title': default_notification_title,
-                    'notification_body': default_notification_body,
-                    'notification_format': default_notification_format,
-                    'real_browser_save_screenshot': True,
-                    'schema_version' : 0
-                }
-            }
-        }
-
-    def __init__(self, *arg, **kw):
-        super(model, self).__init__(*arg, **kw)
-        self.update(self.base_config)

changedetectionio/model/Watch.py

@@ -1,68 +0,0 @@
-import os
-
-import uuid as uuid_builder
-
-minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
-
-from changedetectionio.notification import (
-    default_notification_body,
-    default_notification_format,
-    default_notification_title,
-)
-
-
-class model(dict):
-    base_config = {
-            'url': None,
-            'tag': None,
-            'last_checked': 0,
-            'last_changed': 0,
-            'paused': False,
-            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
-            'newest_history_key': 0,
-            'title': None,
-            'previous_md5': False,
-#           UUID not needed, should be generated only as a key
-#            'uuid':
-            'headers': {},  # Extra headers to send
-            'body': None,
-            'method': 'GET',
-            'history': {},  # Dict of timestamp and output stripped filename
-            'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
-            # Custom notification content
-            'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
-            'notification_title': default_notification_title,
-            'notification_body': default_notification_body,
-            'notification_format': default_notification_format,
-            'css_filter': "",
-            'subtractive_selectors': [],
-            'trigger_text': [],  # List of text or regex to wait for until a change is detected
-            'fetch_backend': None,
-            'extract_title_as_title': False,
-            # Re #110, so then if this is set to None, we know to use the default value instead
-            # Requires setting to None on submit if it's the same as the default
-            # Should be all None by default, so we use the system default in this case.
-            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None}
-        }
-
-    def __init__(self, *arg, **kw):
-        self.update(self.base_config)
-        # goes at the end so we update the default object with the initialiser
-        super(model, self).__init__(*arg, **kw)
-
-
-    @property
-    def has_empty_checktime(self):
-        # using all() + dictionary comprehension
-        # Check if all values are 0 in dictionary
-        res = all(x == None or x == False or x==0 for x in self.get('time_between_check', {}).values())
-        return res
-
-    def threshold_seconds(self):
-        seconds = 0
-        mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
-        for m, n in mtable.items():
-            x = self.get('time_between_check', {}).get(m, None)
-            if x:
-                seconds += x * n
-        return seconds

changedetectionio/notification.py

@@ -66,26 +66,13 @@ def process_notification(n_object, datastore):
                 if not 'avatar_url' in url:
                     url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
 
-                if url.startswith('tgram://'):
+                body = n_body[0:1800-len(n_title)-len(url)] if 'discord://' in url else n_body
-                    # real limit is 4096, but minus some for extra metadata
-                    payload_max_size = 3600
-                    body_limit = max(0, payload_max_size - len(n_title))
-                    n_title = n_title[0:payload_max_size]
-                    n_body = n_body[0:body_limit]
-
-                elif url.startswith('discord://'):
-                    # real limit is 2000, but minus some for extra metadata
-                    payload_max_size = 1700
-                    body_limit = max(0, payload_max_size - len(n_title))
-                    n_title = n_title[0:payload_max_size]
-                    n_body = n_body[0:body_limit]
-
                 apobj.add(url)
 
                 apobj.notify(
-                    title=n_title,
+                title=n_title,
-                    body=n_body,
+                body=body,
-                    body_format=n_format)
+                body_format=n_format)
 
                 apobj.clear()
 

changedetectionio/static/images/copy.svg

@@ -1,40 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   version="1.1"
-   id="Layer_1"
-   x="0px"
-   y="0px"
-   viewBox="0 0 115.77 122.88"
-   style="enable-background:new 0 0 115.77 122.88"
-   xml:space="preserve"
-   sodipodi:docname="copy.svg"
-   inkscape:version="1.1.1 (1:1.1+202109281949+c3084ef5ed)"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg"><defs
-     id="defs11" /><sodipodi:namedview
-     id="namedview9"
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1.0"
-     inkscape:pageshadow="2"
-     inkscape:pageopacity="0.0"
-     inkscape:pagecheckerboard="0"
-     showgrid="false"
-     inkscape:zoom="5.5501303"
-     inkscape:cx="57.83648"
-     inkscape:cy="61.439999"
-     inkscape:window-width="1920"
-     inkscape:window-height="1056"
-     inkscape:window-x="1920"
-     inkscape:window-y="0"
-     inkscape:window-maximized="1"
-     inkscape:current-layer="g6" /><style
-     type="text/css"
-     id="style2">.st0{fill-rule:evenodd;clip-rule:evenodd;}</style><g
-     id="g6"><path
-       class="st0"
-       d="M89.62,13.96v7.73h12.19h0.01v0.02c3.85,0.01,7.34,1.57,9.86,4.1c2.5,2.51,4.06,5.98,4.07,9.82h0.02v0.02 v73.27v0.01h-0.02c-0.01,3.84-1.57,7.33-4.1,9.86c-2.51,2.5-5.98,4.06-9.82,4.07v0.02h-0.02h-61.7H40.1v-0.02 c-3.84-0.01-7.34-1.57-9.86-4.1c-2.5-2.51-4.06-5.98-4.07-9.82h-0.02v-0.02V92.51H13.96h-0.01v-0.02c-3.84-0.01-7.34-1.57-9.86-4.1 c-2.5-2.51-4.06-5.98-4.07-9.82H0v-0.02V13.96v-0.01h0.02c0.01-3.85,1.58-7.34,4.1-9.86c2.51-2.5,5.98-4.06,9.82-4.07V0h0.02h61.7 h0.01v0.02c3.85,0.01,7.34,1.57,9.86,4.1c2.5,2.51,4.06,5.98,4.07,9.82h0.02V13.96L89.62,13.96z M79.04,21.69v-7.73v-0.02h0.02 c0-0.91-0.39-1.75-1.01-2.37c-0.61-0.61-1.46-1-2.37-1v0.02h-0.01h-61.7h-0.02v-0.02c-0.91,0-1.75,0.39-2.37,1.01 c-0.61,0.61-1,1.46-1,2.37h0.02v0.01v64.59v0.02h-0.02c0,0.91,0.39,1.75,1.01,2.37c0.61,0.61,1.46,1,2.37,1v-0.02h0.01h12.19V35.65 v-0.01h0.02c0.01-3.85,1.58-7.34,4.1-9.86c2.51-2.5,5.98-4.06,9.82-4.07v-0.02h0.02H79.04L79.04,21.69z M105.18,108.92V35.65v-0.02 h0.02c0-0.91-0.39-1.75-1.01-2.37c-0.61-0.61-1.46-1-2.37-1v0.02h-0.01h-61.7h-0.02v-0.02c-0.91,0-1.75,0.39-2.37,1.01 c-0.61,0.61-1,1.46-1,2.37h0.02v0.01v73.27v0.02h-0.02c0,0.91,0.39,1.75,1.01,2.37c0.61,0.61,1.46,1,2.37,1v-0.02h0.01h61.7h0.02 v0.02c0.91,0,1.75-0.39,2.37-1.01c0.61-0.61,1-1.46,1-2.37h-0.02V108.92L105.18,108.92z"
-       id="path4"
-       style="fill:#ffffff;fill-opacity:1" /></g></svg>

changedetectionio/static/images/spread.svg

@@ -1,46 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   width="18"
-   height="19.92"
-   viewBox="0 0 18 19.92"
-   version="1.1"
-   id="svg6"
-   sodipodi:docname="spread.svg"
-   inkscape:version="1.1.1 (1:1.1+202109281949+c3084ef5ed)"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg">
-  <defs
-     id="defs10" />
-  <sodipodi:namedview
-     id="namedview8"
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1.0"
-     inkscape:pageshadow="2"
-     inkscape:pageopacity="0.0"
-     inkscape:pagecheckerboard="0"
-     showgrid="false"
-     fit-margin-top="0"
-     fit-margin-left="0"
-     fit-margin-right="0"
-     fit-margin-bottom="0"
-     inkscape:zoom="28.416667"
-     inkscape:cx="9.0087975"
-     inkscape:cy="9.9941348"
-     inkscape:window-width="1920"
-     inkscape:window-height="1056"
-     inkscape:window-x="1920"
-     inkscape:window-y="0"
-     inkscape:window-maximized="1"
-     inkscape:current-layer="svg6" />
-  <path
-     d="M -3,-2 H 21 V 22 H -3 Z"
-     fill="none"
-     id="path2" />
-  <path
-     d="m 15,14.08 c -0.76,0 -1.44,0.3 -1.96,0.77 L 5.91,10.7 C 5.96,10.47 6,10.24 6,10 6,9.76 5.96,9.53 5.91,9.3 L 12.96,5.19 C 13.5,5.69 14.21,6 15,6 16.66,6 18,4.66 18,3 18,1.34 16.66,0 15,0 c -1.66,0 -3,1.34 -3,3 0,0.24 0.04,0.47 0.09,0.7 L 5.04,7.81 C 4.5,7.31 3.79,7 3,7 1.34,7 0,8.34 0,10 c 0,1.66 1.34,3 3,3 0.79,0 1.5,-0.31 2.04,-0.81 l 7.12,4.16 c -0.05,0.21 -0.08,0.43 -0.08,0.65 0,1.61 1.31,2.92 2.92,2.92 1.61,0 2.92,-1.31 2.92,-2.92 0,-1.61 -1.31,-2.92 -2.92,-2.92 z"
-     id="path4"
-     style="fill:#0078e7;fill-opacity:1" />
-</svg>

changedetectionio/static/js/notifications.js

@@ -4,7 +4,7 @@ $(document).ready(function() {
     e.preventDefault();
     email = prompt("Destination email");
     if(email) {
-      var n = $(".notification-urls");
+      var n = $("#notification_urls");
       var p=email_notification_prefix;
       $(n).val( $.trim( $(n).val() )+"\n"+email_notification_prefix+email );
     }
@@ -25,10 +25,10 @@ $(document).ready(function() {
 
     data = {
         window_url : window.location.href,
-        notification_urls : $('.notification-urls').val(),
+        notification_urls : $('#notification_urls').val(),
-        notification_title : $('.notification-title').val(),
+        notification_title : $('#notification_title').val(),
-        notification_body : $('.notification-body').val(),
+        notification_body : $('#notification_body').val(),
-        notification_format : $('.notification-format').val(),
+        notification_format : $('#notification_format').val(),
     }
     for (key in data) {
       if (!data[key].length) {

changedetectionio/static/js/settings.js

@@ -0,0 +1,13 @@
+window.addEventListener("load", (event) => {
+  // just an example for now
+  function toggleVisible(elem) {
+    // theres better ways todo this
+    var x = document.getElementById(elem);
+    if (x.style.display === "block") {
+      x.style.display = "none";
+    } else {
+      x.style.display = "block";
+    }
+  }
+});
+

changedetectionio/static/js/watch-overview.js

@@ -3,22 +3,4 @@ $(function () {
   $('.diff-link').click(function () {
     $(this).closest('.unviewed').removeClass('unviewed');
   });
-
-  $('.with-share-link > *').click(function () {
-      $("#copied-clipboard").remove();
-
-      var range = document.createRange();
-      var n=$("#share-link")[0];
-      range.selectNode(n);
-      window.getSelection().removeAllRanges();
-      window.getSelection().addRange(range);
-      document.execCommand("copy");
-      window.getSelection().removeAllRanges();
-
-      $('.with-share-link').append('<span style="font-size: 80%; color: #fff;" id="copied-clipboard">Copied to clipboard</span>');
-      $("#copied-clipboard").fadeOut(2500, function() {
-       $(this).remove();
-      });
-  });
 });
-

changedetectionio/static/js/watch-settings.js

@@ -1,14 +0,0 @@
-$(document).ready(function() {
-    function toggle() {
-        if ($('input[name="fetch_backend"]:checked').val() != 'html_requests') {
-            $('#requests-override-options').hide();
-        } else {
-            $('#requests-override-options').show();
-        }
-    }
-    $('input[name="fetch_backend"]').click(function (e) {
-        toggle();
-    });
-    toggle();
-
-});

changedetectionio/static/styles/styles.css

@@ -180,9 +180,6 @@ body:after, body:before {
   .messages li.notice {
     background: rgba(255, 255, 255, 0.5); }
 
-.messages.with-share-link > *:hover {
-  cursor: pointer; }
-
 #notification-customisation {
   border: 1px solid #ccc;
   padding: 0.5rem;
@@ -309,10 +306,10 @@ footer {
     font-weight: bold; }
   .pure-form textarea {
     width: 100%; }
-  .pure-form ul.fetch-backend {
+  .pure-form ul#fetch_backend {
     margin: 0px;
     list-style: none; }
-    .pure-form ul.fetch-backend li > * {
+    .pure-form ul#fetch_backend > li > * {
       display: inline-block; }
 
 @media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
@@ -443,8 +440,3 @@ ul {
   padding-left: 1em;
   padding-top: 0px;
   margin-top: 4px; }
-
-.time-check-widget tr {
-  display: inline; }
-  .time-check-widget tr input[type="number"] {
-    width: 4em; }

changedetectionio/static/styles/styles.scss

@@ -237,11 +237,6 @@ body:after, body:before {
             background: rgba(255, 255, 255, .5);
         }
     }
-    &.with-share-link {
-     > *:hover {
-       cursor:pointer;
-     }
-    }
 }
 
 #notification-customisation {
@@ -418,10 +413,10 @@ footer {
   textarea {
     width: 100%;
   }
-  ul.fetch-backend {
+  ul#fetch_backend {
     margin: 0px;
     list-style: none;
-    li {
+    > li {
         > * {
             display: inline-block;
         }
@@ -629,13 +624,4 @@ ul {
     padding-left: 1em;
     padding-top: 0px;
     margin-top: 4px;
-}
-
-.time-check-widget {
-    tr {
-        display: inline;
-        input[type="number"] {
-            width: 4em;
-        }
-    }
 }

changedetectionio/store.py

@@ -1,6 +1,3 @@
-from flask import (
-    flash
-)
 import json
 import logging
 import os
@@ -10,10 +7,12 @@ import uuid as uuid_builder
 from copy import deepcopy
 from os import mkdir, path, unlink
 from threading import Lock
-import re
-import requests
 
-from changedetectionio.model import Watch, App
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+)
 
 
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
@@ -21,11 +20,6 @@ from changedetectionio.model import Watch, App
 # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
 class ChangeDetectionStore:
     lock = Lock()
-    # For general updates/writes that can wait a few seconds
-    needs_write = False
-
-    # For when we edit, we should write to disk
-    needs_write_urgent = False
 
     def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
         # Should only be active for docker
@@ -35,11 +29,71 @@ class ChangeDetectionStore:
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
         self.stop_thread = False
 
-        self.__data = App.model()
+        self.__data = {
+            'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
+            'watching': {},
+            'settings': {
+                'headers': {
+                    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
+                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+                    'Accept-Encoding': 'gzip, deflate',  # No support for brolti in python requests yet.
+                    'Accept-Language': 'en-GB,en-US;q=0.9,en;'
+                },
+                'requests': {
+                    'timeout': 15,  # Default 15 seconds
+                    'minutes_between_check': 3 * 60,  # Default 3 hours
+                    'workers': 10  # Number of threads, lower is better for slow connections
+                },
+                'application': {
+                    'password': False,
+                    'base_url' : None,
+                    'extract_title_as_title': False,
+                    'fetch_backend': 'html_requests',
+                    'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
+                    'global_subtractive_selectors': [],
+                    'ignore_whitespace': False,
+                    'render_anchor_tag_content': False,
+                    'notification_urls': [], # Apprise URL list
+                    # Custom notification content
+                    'notification_title': default_notification_title,
+                    'notification_body': default_notification_body,
+                    'notification_format': default_notification_format,
+                    'real_browser_save_screenshot': True,
+                }
+            }
+        }
 
         # Base definition for all watchers
-        # deepcopy part of #569 - not sure why its needed exactly
+        self.generic_definition = {
-        self.generic_definition = deepcopy(Watch.model())
+            'url': None,
+            'tag': None,
+            'last_checked': 0,
+            'last_changed': 0,
+            'paused': False,
+            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
+            'newest_history_key': "",
+            'title': None,
+            # Re #110, so then if this is set to None, we know to use the default value instead
+            # Requires setting to None on submit if it's the same as the default
+            'minutes_between_check': None,
+            'previous_md5': "",
+            'uuid': str(uuid_builder.uuid4()),
+            'headers': {},  # Extra headers to send
+            'body': None,
+            'method': 'GET',
+            'history': {},  # Dict of timestamp and output stripped filename
+            'ignore_text': [], # List of text to ignore when calculating the comparison checksum
+            # Custom notification content
+            'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise)
+            'notification_title': default_notification_title,
+            'notification_body': default_notification_body,
+            'notification_format': default_notification_format,
+            'css_filter': "",
+            'subtractive_selectors': [],
+            'trigger_text': [],  # List of text or regex to wait for until a change is detected
+            'fetch_backend': None,
+            'extract_title_as_title': False
+        }
 
         if path.isfile('changedetectionio/source.txt'):
             with open('changedetectionio/source.txt') as f:
@@ -111,9 +165,6 @@ class ChangeDetectionStore:
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['rss_access_token'] = secret
 
-        # Bump the update version by running updates
-        self.run_updates()
-
         self.needs_write = True
 
         # Finally start the thread that will manage periodic data saves to JSON
@@ -139,16 +190,8 @@ class ChangeDetectionStore:
         self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
         self.needs_write = True
 
-    def remove_password(self):
-        self.__data['settings']['application']['password'] = False
-        self.needs_write = True
-
     def update_watch(self, uuid, update_obj):
 
-        # It's possible that the watch could be deleted before update
-        if not self.__data['watching'].get(uuid):
-            return
-
         with self.lock:
 
             # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
@@ -163,17 +206,6 @@ class ChangeDetectionStore:
 
         self.needs_write = True
 
-    @property
-    def threshold_seconds(self):
-        seconds = 0
-        mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
-        minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
-        for m, n in mtable.items():
-            x = self.__data['settings']['requests']['time_between_check'].get(m)
-            if x:
-                seconds += x * n
-        return max(seconds, minimum_seconds_recheck_time)
-
     @property
     def data(self):
         has_unviewed = False
@@ -236,7 +268,7 @@ class ChangeDetectionStore:
 
                 del self.data['watching'][uuid]
 
-            self.needs_write_urgent = True
+            self.needs_write = True
 
     # Clone a watch by UUID
     def clone(self, uuid):
@@ -260,64 +292,69 @@ class ChangeDetectionStore:
         return self.data['watching'][uuid].get(val)
 
     # Remove a watchs data but keep the entry (URL etc)
-    def scrub_watch(self, uuid):
+    def scrub_watch(self, uuid, limit_timestamp = False):
-        import pathlib
 
-        self.__data['watching'][uuid].update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'newest_history_key': 0, 'previous_md5': False})
+        import hashlib
-        self.needs_write_urgent = True
+        del_timestamps = []
 
-        for item in pathlib.Path(self.datastore_path).rglob(uuid+"/*.txt"):
+        changes_removed = 0
-            unlink(item)
 
-    def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
+        for timestamp, path in self.data['watching'][uuid]['history'].items():
+            if not limit_timestamp or (limit_timestamp is not False and int(timestamp) > limit_timestamp):
+                self.unlink_history_file(path)
+                del_timestamps.append(timestamp)
+                changes_removed += 1
+
+        if not limit_timestamp:
+            self.data['watching'][uuid]['last_checked'] = 0
+            self.data['watching'][uuid]['last_changed'] = 0
+            self.data['watching'][uuid]['previous_md5'] = ""
+
+
+        for timestamp in del_timestamps:
+            del self.data['watching'][uuid]['history'][str(timestamp)]
+
+            # If there was a limitstamp, we need to reset some meta data about the entry
+            # This has to happen after we remove the others from the list
+            if limit_timestamp:
+                newest_key = self.get_newest_history_key(uuid)
+                if newest_key:
+                    self.data['watching'][uuid]['last_checked'] = int(newest_key)
+                    # @todo should be the original value if it was less than newest key
+                    self.data['watching'][uuid]['last_changed'] = int(newest_key)
+                    try:
+                        with open(self.data['watching'][uuid]['history'][str(newest_key)], "rb") as fp:
+                            content = fp.read()
+                        self.data['watching'][uuid]['previous_md5'] = hashlib.md5(content).hexdigest()
+                    except (FileNotFoundError, IOError):
+                        self.data['watching'][uuid]['previous_md5'] = ""
+                        pass
+
+        self.needs_write = True
+        return changes_removed
+
+    def add_watch(self, url, tag="", extras=None):
         if extras is None:
             extras = {}
-        # Incase these are copied across, assume it's a reference and deepcopy()
-        apply_extras = deepcopy(extras)
-
-        # Was it a share link? try to fetch the data
-        if (url.startswith("https://changedetection.io/share/")):
-            try:
-                r = requests.request(method="GET",
-                                     url=url,
-                                     # So we know to return the JSON instead of the human-friendly "help" page
-                                     headers={'App-Guid': self.__data['app_guid']})
-                res = r.json()
-
-                # List of permisable stuff we accept from the wild internet
-                for k in ['url', 'tag',
-                                   'paused', 'title',
-                                   'previous_md5', 'headers',
-                                   'body', 'method',
-                                   'ignore_text', 'css_filter',
-                                   'subtractive_selectors', 'trigger_text',
-                                   'extract_title_as_title']:
-                    if res.get(k):
-                        apply_extras[k] = res[k]
-
-            except Exception as e:
-                logging.error("Error fetching metadata for shared watch link", url, str(e))
-                flash("Error fetching metadata for {}".format(url), 'error')
-                return False
 
         with self.lock:
             # @todo use a common generic version of this
             new_uuid = str(uuid_builder.uuid4())
-            # #Re 569
+            _blank = deepcopy(self.generic_definition)
-            # Not sure why deepcopy was needed here, sometimes new watches would appear to already have 'history' set
+            _blank.update({
-            # I assumed this would instantiate a new object but somehow an existing dict was getting used
-            new_watch = deepcopy(Watch.model({
                 'url': url,
                 'tag': tag
-            }))
+            })
-
 
+            # Incase these are copied across, assume it's a reference and deepcopy()
+            apply_extras = deepcopy(extras)
             for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
                 if k in apply_extras:
                     del apply_extras[k]
 
-            new_watch.update(apply_extras)
+            _blank.update(apply_extras)
-            self.__data['watching'][new_uuid]=new_watch
+
+            self.data['watching'][new_uuid] = _blank
 
         # Get the directory ready
         output_path = "{}/{}".format(self.datastore_path, new_uuid)
@@ -326,8 +363,7 @@ class ChangeDetectionStore:
         except FileExistsError:
             print(output_path, "already exists.")
 
-        if write_to_disk_now:
+        self.sync_to_json()
-            self.sync_to_json()
         return new_uuid
 
     # Save some text file to the appropriate path and bump the history
@@ -365,7 +401,7 @@ class ChangeDetectionStore:
 
     def sync_to_json(self):
         logging.info("Saving JSON..")
-        print("Saving JSON..")
+
         try:
             data = deepcopy(self.__data)
         except RuntimeError as e:
@@ -387,7 +423,6 @@ class ChangeDetectionStore:
                 logging.error("Error writing JSON!! (Main JSON file save was skipped) : %s", str(e))
 
             self.needs_write = False
-            self.needs_write_urgent = False
 
     # Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON
     # by just running periodically in one thread, according to python, dict updates are threadsafe.
@@ -398,14 +433,14 @@ class ChangeDetectionStore:
                 print("Shutting down datastore thread")
                 return
 
-            if self.needs_write or self.needs_write_urgent:
+            if self.needs_write:
                 self.sync_to_json()
 
             # Once per minute is enough, more and it can cause high CPU usage
             # better here is to use something like self.app.config.exit.wait(1), but we cant get to 'app' from here
-            for i in range(120):
+            for i in range(30):
-                time.sleep(0.5)
+                time.sleep(2)
-                if self.stop_thread or self.needs_write_urgent:
+                if self.stop_thread:
                     break
 
     # Go through the datastore path and remove any snapshots that are not mentioned in the index
@@ -421,54 +456,7 @@ class ChangeDetectionStore:
         import pathlib
 
         # Only in the sub-directories
-        for uuid in self.data['watching']:
+        for item in pathlib.Path(self.datastore_path).rglob("*/*txt"):
-            for item in pathlib.Path(self.datastore_path).rglob(uuid+"/*.txt"):
+            if not str(item) in index:
-                if not str(item) in index:
+                print ("Removing",item)
-                    print ("Removing",item)
+                unlink(item)
-                    unlink(item)
-
-    # Run all updates
-    # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
-    #             So therefor - each `update_n` should be very careful about checking if it needs to actually run
-    #             Probably we should bump the current update schema version with each tag release version?
-    def run_updates(self):
-        import inspect
-        import shutil
-
-        updates_available = []
-        for i, o in inspect.getmembers(self, predicate=inspect.ismethod):
-            m = re.search(r'update_(\d+)$', i)
-            if m:
-                updates_available.append(int(m.group(1)))
-        updates_available.sort()
-
-        for update_n in updates_available:
-            if update_n > self.__data['settings']['application']['schema_version']:
-                print ("Applying update_{}".format((update_n)))
-                # Wont exist on fresh installs
-                if os.path.exists(self.json_store_path):
-                    shutil.copyfile(self.json_store_path, self.datastore_path+"/url-watches-before-{}.json".format(update_n))
-
-                try:
-                    update_method = getattr(self, "update_{}".format(update_n))()
-                except Exception as e:
-                    print("Error while trying update_{}".format((update_n)))
-                    print(e)
-                    # Don't run any more updates
-                    return
-                else:
-                    # Bump the version, important
-                    self.__data['settings']['application']['schema_version'] = update_n
-
-    # Convert minutes to seconds on settings and each watch
-    def update_1(self):
-        if self.data['settings']['requests'].get('minutes_between_check'):
-            self.data['settings']['requests']['time_between_check']['minutes'] = self.data['settings']['requests']['minutes_between_check']
-            # Remove the default 'hours' that is set from the model
-            self.data['settings']['requests']['time_between_check']['hours'] = None
-
-        for uuid, watch in self.data['watching'].items():
-            if 'minutes_between_check' in watch:
-                # Only upgrade individual watch time if it was set
-                if watch.get('minutes_between_check', False):
-                    self.data['watching'][uuid]['time_between_check']['minutes'] = watch['minutes_between_check']

changedetectionio/templates/_common_fields.jinja

@@ -8,12 +8,12 @@
     Gitter - gitter://token/room
     Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
     AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
-    SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com", class="notification-urls")
+    SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com")
                             }}
                             <div class="pure-form-message-inline">
                               <ul>
                                 <li>Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.</li>
-                                <li><code>discord://</code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code>discord://</code> notifications are cut at 2,000 characters in length.</li>
                                 <li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
                                 <li>Go here for <a href="{{url_for('notification_logs')}}">notification debug logs</a></li>
                               </ul>
@@ -22,19 +22,20 @@
                             <a id="send-test-notification" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Send test notification</a>
 {% if emailprefix %}
                             <a id="add-email-helper" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Add email</a>
+
 {% endif %}
                         </div>
                         <div id="notification-customisation" class="pure-control-group">
                             <div class="pure-control-group">
-                                {{ render_field(form.notification_title, class="m-d notification-title") }}
+                                {{ render_field(form.notification_title, class="m-d") }}
                                 <span class="pure-form-message-inline">Title for all notifications</span>
                             </div>
                             <div class="pure-control-group">
-                                {{ render_field(form.notification_body , rows=5, class="notification-body") }}
+                                {{ render_field(form.notification_body , rows=5) }}
                                 <span class="pure-form-message-inline">Body for all notifications</span>
                             </div>
                             <div class="pure-control-group">
-                                {{ render_field(form.notification_format , rows=5, class="notification-format") }}
+                                {{ render_field(form.notification_format , rows=5) }}
                                 <span class="pure-form-message-inline">Format for all notifications</span>
                             </div>
                             <div class="pure-controls">

changedetectionio/templates/base.html

@@ -94,13 +94,6 @@
         </ul>
       {% endif %}
     {% endwith %}
-
-    {% if session['share-link'] %}
-        <ul class="messages with-share-link">
-          <li class="message">Share this link: <span id="share-link">{{ session['share-link'] }}</span> <img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='copy.svg')}}" /></li>
-        </ul>
-    {% endif %}
-
     {% block content %}
 
     {% endblock %}

changedetectionio/templates/edit.html

@@ -9,7 +9,6 @@
     const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
 </script>
-<script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 
 <div class="edit-form monospaced-textarea">
@@ -42,8 +41,8 @@
                         <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
                     <div class="pure-control-group">
-                        {{ render_field(form.time_between_check, class="time-check-widget") }}
+                        {{ render_field(form.minutes_between_check) }}
-                        {% if has_empty_checktime %}
+                        {% if using_default_minutes %}
                         <span class="pure-form-message-inline">Currently using the <a
                                 href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
                         {% else %}
@@ -59,17 +58,19 @@
 
             <div class="tab-pane-inner" id="request">
                     <div class="pure-control-group">
-                        {{ render_field(form.fetch_backend, class="fetch-backend") }}
+                        {{ render_field(form.fetch_backend) }}
                         <span class="pure-form-message-inline">
                             <p>Use the <strong>Basic</strong> method (default) where your watched site doesn't need Javascript to render.</p>
                             <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                         </span>
                     </div>
 
-                <fieldset class="pure-group" id="requests-override-options">
+                <hr/>
-                    <div class="pure-form-message-inline">
+                <fieldset class="pure-group">
+
+                    <span class="pure-form-message-inline">
                         <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
-                    </div>
+                    </span>
                     <div class="pure-control-group">
                         {{ render_field(form.method) }}
                     </div>

changedetectionio/templates/scrub.html

@@ -7,7 +7,7 @@
         <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <div class="pure-control-group">
-                This will remove ALL version snapshots/data, but keep your list of URLs. <br/>
+                This will remove all version snapshots/data, but keep your list of URLs. <br/>
                 You may like to use the <strong>BACKUP</strong> link first.<br/>
             </div>
             <br/>
@@ -17,6 +17,12 @@
                 <span class="pure-form-message-inline">Type in the word <strong>scrub</strong> to confirm that you understand!</span>
             </div>
             <br/>
+            <div class="pure-control-group">
+                <label for="confirmtext">Optional: Limit deletion of snapshots to snapshots <i>newer</i> than date/time</label>
+                <input type="datetime-local" id="limit_date" name="limit_date"  />
+                <span class="pure-form-message-inline">dd/mm/yyyy hh:mm (24 hour format)</span>
+            </div>
+            <br/>
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Scrub!</button>
             </div>

changedetectionio/templates/settings.html

@@ -9,6 +9,7 @@
     const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}
 </script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='settings.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 
@@ -27,15 +28,15 @@
             <div class="tab-pane-inner" id="general">
                 <fieldset>
                     <div class="pure-control-group">
-                        {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
+                        {{ render_field(form.minutes_between_check) }}
                         <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
                     </div>
                     <div class="pure-control-group">
                         {% if not hide_remove_pass %}
                             {% if current_user.is_authenticated %}
-                                {{ render_button(form.application.form.removepassword_button) }}
+                                {{ render_button(form.removepassword_button) }}
                             {% else %}
-                            {{ render_field(form.application.form.password) }}
+                            {{ render_field(form.password) }}
                             <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
                             {% endif %}
                         {% else %}
@@ -43,7 +44,7 @@
                         {% endif %}
                     </div>
                     <div class="pure-control-group">
-                        {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
+                        {{ render_field(form.base_url, placeholder="http://yoursite.com:5000/",
                         class="m-d") }}
                         <span class="pure-form-message-inline">
                             Base URL used for the {base_url} token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{current_base_url}}"),
@@ -52,12 +53,12 @@
                     </div>
 
                     <div class="pure-control-group">
-                        {{ render_checkbox_field(form.application.form.extract_title_as_title) }}
+                        {{ render_checkbox_field(form.extract_title_as_title) }}
                         <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
                     </div>
 
                     <div class="pure-control-group">
-                        {{ render_checkbox_field(form.application.form.real_browser_save_screenshot) }}
+                        {{ render_checkbox_field(form.real_browser_save_screenshot) }}
                         <span class="pure-form-message-inline">When using a Chrome browser, a screenshot from the last check will be available on the Diff page</span>
                     </div>
 
@@ -67,14 +68,14 @@
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div class="field-group">
-                        {{ render_common_settings_form(form.application.form, current_base_url, emailprefix) }}
+                        {{ render_common_settings_form(form, current_base_url, emailprefix) }}
                     </div>
                 </fieldset>
             </div>
 
             <div class="tab-pane-inner" id="fetching">
                 <div class="pure-control-group">
-                    {{ render_field(form.application.form.fetch_backend, class="fetch-backend") }}
+                    {{ render_field(form.fetch_backend) }}
                     <span class="pure-form-message-inline">
                         <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
                         <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
@@ -86,20 +87,20 @@
             <div class="tab-pane-inner" id="filters">
 
                     <fieldset class="pure-group">
-                    {{ render_checkbox_field(form.application.form.ignore_whitespace) }}
+                    {{ render_checkbox_field(form.ignore_whitespace) }}
                     <span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br/>
                     <i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc.
                     </span>
                     </fieldset>
                 <fieldset class="pure-group">
-                    {{ render_checkbox_field(form.application.form.render_anchor_tag_content) }}
+                    {{ render_checkbox_field(form.render_anchor_tag_content) }}
                     <span class="pure-form-message-inline">Render anchor tag content, default disabled, when enabled renders links as <code>(link text)[https://somesite.com]</code>
                         <br/>
                     <i>Note:</i> Changing this could affect the content of your existing watches, possibly trigger alerts etc.
                     </span>
                     </fieldset>
                     <fieldset class="pure-group">
-                      {{ render_field(form.application.form.global_subtractive_selectors, rows=5, placeholder="header
+                      {{ render_field(form.global_subtractive_selectors, rows=5, placeholder="header
 footer
 nav
 .stockticker") }}
@@ -111,7 +112,7 @@ nav
                       </span>
                     </fieldset>
                     <fieldset class="pure-group">
-                    {{ render_field(form.application.form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
+                    {{ render_field(form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
 /some.regex\d{2}/ for case-INsensitive regex
                     ") }}
                     <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br/>

changedetectionio/templates/watch-overview.html

@@ -10,10 +10,11 @@
         <fieldset>
             <legend>Add a new change detection watch</legend>
                 {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
-                {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch group") }}
+                {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="tag") }}
             <button type="submit" class="pure-button pure-button-primary">Watch</button>
         </fieldset>
-        <span style="color:#eee; font-size: 80%;"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /> Tip: You can also add 'shared' watches. <a href="#">More info</a></a></span>
+        <!-- add extra stuff, like do a http POST and send headers -->
+        <!-- user/pass r = requests.get('https://api.github.com/user', auth=('user', 'pass')) -->
     </form>
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
@@ -45,15 +46,12 @@
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
                 {% if watch.last_notification_error is defined and watch.last_notification_error != False %}error{% endif %}
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
-                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}
+                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
-                {% if watch.uuid in queued_uuids %}queued{% endif %}">
                 <td class="inline">{{ loop.index }}</td>
                 <td class="inline paused-state state-{{watch.paused}}"><a href="{{url_for('index', pause=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause" title="Pause"/></a></td>
 
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
-                    <a class="external" target="_blank" rel="noopener" href="{{ watch.url.replace('source:','') }}"></a>
+                    <a class="external" target="_blank" rel="noopener" href="{{ watch.url }}"></a>
-                    <a href="{{url_for('api_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
-
                     {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
 
                     {% if watch.last_error is defined and watch.last_error != False %}
@@ -74,8 +72,8 @@
                     {% endif %}
                 </td>
                 <td>
-                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('api_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
+                    <a href="{{ url_for('api_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
-                       class="recheck pure-button button-small pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
+                       class="pure-button button-small pure-button-primary">Recheck</a>
                     <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
                     {% if watch.history|length >= 2 %}
                     <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary diff-link">Diff</a>

changedetectionio/tests/conftest.py

@@ -16,7 +16,6 @@ def cleanup(datastore_path):
     # Unlink test output files
     files = ['output.txt',
              'url-watches.json',
-             'secret.txt',
              'notification.txt',
              'count.txt',
              'endpoint-content.txt'

changedetectionio/tests/test_access_control.py

@@ -1,5 +1,5 @@
 from flask import url_for
-from . util import live_server_setup
+
 
 def test_check_access_control(app, client):
     # Still doesnt work, but this is closer.
@@ -12,9 +12,9 @@ def test_check_access_control(app, client):
         # Enable password check.
         res = c.post(
             url_for("settings_page"),
-            data={"application-password": "foobar",
+            data={"password": "foobar",
-                  "requests-time_between_check-minutes": 180,
+                  "minutes_between_check": 180,
-                  'application-fetch_backend': "html_requests"},
+                  'fetch_backend': "html_requests"},
             follow_redirects=True
         )
 
@@ -46,34 +46,77 @@ def test_check_access_control(app, client):
         assert b"BACKUP" in res.data
         assert b"IMPORT" in res.data
         assert b"LOG OUT" in res.data
-        assert b"time_between_check-minutes" in res.data
+        assert b"minutes_between_check" in res.data
         assert b"fetch_backend" in res.data
 
-        ##################################################
-        # Remove password button, and check that it worked
-        ##################################################
         res = c.post(
             url_for("settings_page"),
             data={
-                "requests-time_between_check-minutes": 180,
+                "minutes_between_check": 180,
-                "application-fetch_backend": "html_webdriver",
+                "tag": "",
-                "application-removepassword_button": "Remove password"
+                "headers": "",
+                "fetch_backend": "html_webdriver",
+                "removepassword_button": "Remove password"
             },
             follow_redirects=True,
         )
-        assert b"Password protection removed." in res.data
-        assert b"LOG OUT" not in res.data
 
-        ############################################################
+# There was a bug where saving the settings form would submit a blank password
-        # Be sure a blank password doesnt setup password protection
+def test_check_access_control_no_blank_password(app, client):
-        ############################################################
+    # Still doesnt work, but this is closer.
+
+    with app.test_client() as c:
+        # Check we dont have any password protection enabled yet.
+        res = c.get(url_for("settings_page"))
+        assert b"Remove password" not in res.data
+
+        # Enable password check.
         res = c.post(
             url_for("settings_page"),
-            data={"application-password": "",
+            data={"password": "",
-                  "requests-time_between_check-minutes": 180,
+                  "minutes_between_check": 180,
-                  'application-fetch_backend': "html_requests"},
+                  'fetch_backend': "html_requests"},
             follow_redirects=True
         )
 
-        assert b"Password protection enabled" not in res.data
+        assert b"Password protection enabled." not in res.data
+        assert b"Login" not in res.data
 
+
+# There was a bug where saving the settings form would submit a blank password
+def test_check_access_no_remote_access_to_remove_password(app, client):
+    # Still doesnt work, but this is closer.
+
+    with app.test_client() as c:
+        # Check we dont have any password protection enabled yet.
+        res = c.get(url_for("settings_page"))
+        assert b"Remove password" not in res.data
+
+        # Enable password check.
+        res = c.post(
+            url_for("settings_page"),
+            data={"password": "password",
+                  "minutes_between_check": 180,
+                  'fetch_backend': "html_requests"},
+            follow_redirects=True
+        )
+
+        assert b"Password protection enabled." in res.data
+        assert b"Login" in res.data
+
+        res = c.post(
+            url_for("settings_page"),
+            data={
+                "minutes_between_check": 180,
+                "tag": "",
+                "headers": "",
+                "fetch_backend": "html_webdriver",
+                "removepassword_button": "Remove password"
+            },
+            follow_redirects=True,
+        )
+        assert b"Password protection removed." not in res.data
+
+        res = c.get(url_for("index"),
+              follow_redirects=True)
+        assert b"watch-table-wrapper" not in res.data

changedetectionio/tests/test_backend.py

@@ -50,14 +50,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     #####################
 
-    # Check HTML conversion detected and workd
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    # Check this class does not appear (that we didnt see the actual source)
-    assert b'foobar-detection' not in res.data
-
     # Make a change
     set_modified_response()
 
@@ -109,7 +101,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # Enable auto pickup of <title> in settings
     res = client.post(
         url_for("settings_page"),
-        data={"application-extract_title_as_title": "1", "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"},
+        data={"extract_title_as_title": "1", "minutes_between_check": 180, 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
 

changedetectionio/tests/test_ignore_text.py

@@ -171,24 +171,11 @@ def test_check_ignore_text_functionality(client, live_server):
 def test_check_global_ignore_text_functionality(client, live_server):
     sleep_time_for_fetch_thread = 3
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
     set_original_ignore_response()
 
-    # Goto the settings page, add our ignore text
+    # Give the endpoint time to spin up
-    res = client.post(
+    time.sleep(1)
-        url_for("settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-global_ignore_text": ignore_text,
-            'application-fetch_backend': "html_requests"
-        },
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
-
 
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -205,6 +192,17 @@ def test_check_global_ignore_text_functionality(client, live_server):
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
+    # Goto the settings page, add our ignore text
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "minutes_between_check": 180,
+            "global_ignore_text": ignore_text,
+            'fetch_backend': "html_requests"
+        },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
 
     # Goto the edit page of the item, add our ignore text
     # Add our URL to the import page
@@ -227,16 +225,12 @@ def test_check_global_ignore_text_functionality(client, live_server):
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
-    # so that we are sure everything is viewed and in a known 'nothing changed' state
-    res = client.get(url_for("diff_history_page", uuid="first"))
-
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
-
+    #  Make a change
-    #  Make a change which includes the ignore text
     set_modified_ignore_response()
 
     # Trigger a check
@@ -249,7 +243,7 @@ def test_check_global_ignore_text_functionality(client, live_server):
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
-    # Just to be sure.. set a regular modified change that will trigger it
+    # Just to be sure.. set a regular modified change..
     set_modified_original_ignore_response()
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)

changedetectionio/tests/test_ignorehyperlinks.py

@@ -51,12 +51,13 @@ def test_render_anchor_tag_content_true(client, live_server):
     # set original html text
     set_original_ignore_response()
 
-    # Goto the settings page, choose to ignore links (dont select/send "application-render_anchor_tag_content")
+    # Goto the settings page, choose not to ignore links
     res = client.post(
         url_for("settings_page"),
         data={
-            "requests-time_between_check-minutes": 180,
+            "minutes_between_check": 180,
-            "application-fetch_backend": "html_requests",
+            "render_anchor_tag_content": "true",
+            "fetch_backend": "html_requests",
         },
         follow_redirects=True,
     )
@@ -84,30 +85,6 @@ def test_render_anchor_tag_content_true(client, live_server):
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
-    # We should not see the rendered anchor tag
-    res = client.get(url_for("preview_page", uuid="first"))
-    assert '(/modified_link)' not in res.data.decode()
-
-    # Goto the settings page, ENABLE render anchor tag
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-render_anchor_tag_content": "true",
-            "application-fetch_backend": "html_requests",
-        },
-        follow_redirects=True,
-    )
-    assert b"Settings updated." in res.data
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-
-
     # check that the anchor tag content is rendered
     res = client.get(url_for("preview_page", uuid="first"))
     assert '(/modified_link)' in res.data.decode()
@@ -123,3 +100,120 @@ def test_render_anchor_tag_content_true(client, live_server):
                      follow_redirects=True)
     assert b'Deleted' in res.data
 
+
+def test_render_anchor_tag_content_false(client, live_server):
+    """Testing that anchor tag content changes are ignored when
+    render_anchor_tag_content setting is set to false"""
+    sleep_time_for_fetch_thread = 3
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # set the original html text
+    set_original_ignore_response()
+
+    # Goto the settings page, choose to ignore hyperlinks
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "minutes_between_check": 180,
+            "render_anchor_tag_content": "false",
+            "fetch_backend": "html_requests",
+        },
+        follow_redirects=True,
+    )
+    assert b"Settings updated." in res.data
+
+    # Add our URL to the import page
+    test_url = url_for("test_endpoint", _external=True)
+    res = client.post(
+        url_for("import_page"), data={"urls": test_url}, follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # set a new html text, with a modified link
+    set_modified_ignore_response()
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # check that the anchor tag content is not rendered
+    res = client.get(url_for("preview_page", uuid="first"))
+    assert '(/modified_link)' not in res.data.decode()
+
+    # even though the link has changed, we shouldn't detect a change since
+    # we selected to not render anchor tag content (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b"unviewed" not in res.data
+    assert b"/test-endpoint" in res.data
+
+    # Cleanup everything
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+
+def test_render_anchor_tag_content_default(client, live_server):
+    """Testing that anchor tag content changes are ignored when the
+    render_anchor_tag_content setting is not explicitly selected"""
+    sleep_time_for_fetch_thread = 3
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # set the original html text
+    set_original_ignore_response()
+
+    # Goto the settings page, not passing the render_anchor_tag_content setting
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "minutes_between_check": 180,
+            "fetch_backend": "html_requests",
+        },
+        follow_redirects=True,
+    )
+    assert b"Settings updated." in res.data
+
+    # Add our URL to the import page
+    test_url = url_for("test_endpoint", _external=True)
+    res = client.post(
+        url_for("import_page"), data={"urls": test_url}, follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # set a new html text, with a modified link
+    set_modified_ignore_response()
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # check that the anchor tag content is not rendered
+    res = client.get(url_for("preview_page", uuid="first"))
+    assert '(/modified_link)' not in res.data.decode()
+
+    # even though the link has changed, we shouldn't detect a change since
+    # we did not select the setting and the default behaviour is to not
+    # render anchor tag content (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b"unviewed" not in res.data
+    assert b"/test-endpoint" in res.data
+
+    # Cleanup everything
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetectionio/tests/test_ignorestatuscode.py

@@ -51,9 +51,9 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-            "requests-time_between_check-minutes": 180,
+            "minutes_between_check": 180,
-            "application-ignore_status_codes": "y",
+            "ignore_status_codes": "y",
-            'application-fetch_backend': "html_requests"
+            'fetch_backend': "html_requests"
         },
         follow_redirects=True
     )

changedetectionio/tests/test_ignorewhitespace.py

@@ -61,9 +61,9 @@ def test_check_ignore_whitespace(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-            "requests-time_between_check-minutes": 180,
+            "minutes_between_check": 180,
-            "application-ignore_whitespace": "y",
+            "ignore_whitespace": "y",
-            "application-fetch_backend": "html_requests"
+            'fetch_backend': "html_requests"
         },
         follow_redirects=True
     )

changedetectionio/tests/test_jsonpath_selector.py

@@ -270,7 +270,6 @@ def test_check_json_filter_bool_val(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(3)
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
@@ -285,7 +284,6 @@ def test_check_json_filter_bool_val(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    time.sleep(3)
 
     # Trigger a check
     client.get(url_for("api_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_notification.py

@@ -156,7 +156,7 @@ def test_check_notification(client, live_server):
 
     # cleanup for the next
     client.get(
-        url_for("api_delete", uuid="all"),
+        url_for("api_delete", uuid="first"),
         follow_redirects=True
     )
 
@@ -172,7 +172,8 @@ def test_notification_validation(client, live_server):
         data={"url": test_url, "tag": 'nice one'},
         follow_redirects=True
     )
-
+    with open("xxx.bin", "wb") as f:
+        f.write(res.data)
     assert b"Watch added" in res.data
 
     # Re #360 some validation
@@ -194,11 +195,11 @@ def test_notification_validation(client, live_server):
     # Now adding a wrong token should give us an error
     res = client.post(
         url_for("settings_page"),
-        data={"application-notification_title": "New ChangeDetection.io Notification - {watch_url}",
+        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
-              "application-notification_body": "Rubbish: {rubbish}\n",
+              "notification_body": "Rubbish: {rubbish}\n",
-              "application-notification_format": "Text",
+              "notification_format": "Text",
-              "application-notification_urls": "json://localhost/foobar",
+              "notification_urls": "json://localhost/foobar",
-              "requests-time_between_check-minutes": 180,
+              "time_between_check": {'seconds': 180},
               "fetch_backend": "html_requests"
               },
         follow_redirects=True
@@ -208,6 +209,6 @@ def test_notification_validation(client, live_server):
 
     # cleanup for the next
     client.get(
-        url_for("api_delete", uuid="all"),
+        url_for("api_delete", uuid="first"),
         follow_redirects=True
     )

changedetectionio/tests/test_notification_errors.py

@@ -35,7 +35,7 @@ def test_check_notification_error_handling(client, live_server):
               "tag": "",
               "title": "",
               "headers": "",
-              "time_between_check-minutes": "180",
+              "minutes_between_check": "180",
               "fetch_backend": "html_requests",
               "trigger_check": "y"},
         follow_redirects=True

changedetectionio/tests/test_request.py

@@ -27,7 +27,6 @@ def test_headers_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(3)
     cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;'
 
 
@@ -85,25 +84,9 @@ def test_body_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(3)
-
-    # add the first 'version'
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-              "url": test_url,
-              "tag": "",
-              "method": "POST",
-              "fetch_backend": "html_requests",
-              "body": "something something"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
-    time.sleep(3)
-
-    # Now the change which should trigger a change
     body_value = 'Test Body Value'
+
+    # Add a properly formatted body with a proper method
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={

changedetectionio/tests/test_share_watch.py

@@ -1,76 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup
-import re
-
-sleep_time_for_fetch_thread = 3
-
-
-def test_share_watch(client, live_server):
-    set_original_response()
-    live_server_setup(live_server)
-
-    test_url = url_for('test_endpoint', _external=True)
-    css_filter = ".nice-filter"
-
-    # Add our URL to the import page
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-
-    # Goto the edit page, add our ignore text
-    # Add our URL to the import page
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    # Check it saved
-    res = client.get(
-        url_for("edit_page", uuid="first"),
-    )
-    assert bytes(css_filter.encode('utf-8')) in res.data
-
-    # click share the link
-    res = client.get(
-        url_for("api_share_put_watch", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b"Share this link:" in res.data
-    assert b"https://changedetection.io/share/" in res.data
-
-    html = res.data.decode()
-    share_link_search = re.search('<span id="share-link">(.*)</span>', html, re.IGNORECASE)
-    assert share_link_search
-
-    # Now delete what we have, we will try to re-import it
-    # Cleanup everything
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-    # Add our URL to the import page
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": share_link_search.group(1)},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-
-    # Now hit edit, we should see what we expect
-    # that the import fetched the meta-data
-
-    # Check it saved
-    res = client.get(
-        url_for("edit_page", uuid="first"),
-    )
-    assert bytes(css_filter.encode('utf-8')) in res.data

changedetectionio/tests/test_source.py

@@ -1,95 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup
-
-sleep_time_for_fetch_thread = 3
-
-def test_setup(live_server):
-    live_server_setup(live_server)
-
-def test_check_basic_change_detection_functionality_source(client, live_server):
-    set_original_response()
-    test_url = 'source:'+url_for('test_endpoint', _external=True)
-    # Add our URL to the import page
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-
-    time.sleep(sleep_time_for_fetch_thread)
-
-    #####################
-
-    # Check HTML conversion detected and workd
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    # Check this class DOES appear (that we didnt see the actual source)
-    assert b'foobar-detection' in res.data
-
-    # Make a change
-    set_modified_response()
-
-    # Force recheck
-    res = client.get(url_for("api_watch_checknow"), follow_redirects=True)
-    assert b'1 watches are queued for rechecking.' in res.data
-
-    time.sleep(5)
-
-    # Now something should be ready, indicated by having a 'unviewed' class
-    res = client.get(url_for("index"))
-    assert b'unviewed' in res.data
-
-    res = client.get(
-        url_for("diff_history_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b'<title>modified head title' in res.data
-
-
-
-
-def test_check_ignore_elements(client, live_server):
-    set_original_response()
-
-    time.sleep(2)
-    test_url = 'source:'+url_for('test_endpoint', _external=True)
-    # Add our URL to the import page
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-
-    time.sleep(sleep_time_for_fetch_thread)
-
-    #####################
-    # We want <span> and <p> ONLY, but ignore span with .foobar-detection
-
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"css_filter": 'span,p', "url": test_url, "tag": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-
-    time.sleep(sleep_time_for_fetch_thread)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b'foobar-detection' not in res.data
-    assert b'&lt;br' not in res.data
-    assert b'&lt;p' in res.data

changedetectionio/tests/test_watch_fields_storage.py

@@ -20,8 +20,8 @@ def test_check_watch_field_storage(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={ "notification_urls": "json://127.0.0.1:30000\r\njson://128.0.0.1\r\n",
+        data={ "notification_urls": "json://myapi.com",
-               "time_between_check-minutes": 126,
+               "minutes_between_check": 126,
                "css_filter" : ".fooclass",
                "title" : "My title",
                "ignore_text" : "ignore this",
@@ -38,14 +38,8 @@ def test_check_watch_field_storage(client, live_server):
         url_for("edit_page", uuid="first"),
         follow_redirects=True
     )
-    # checks that we dont get an error when using blank lines in the field value
-    assert not b"json://127.0.0.1\n\njson" in res.data
-    assert not b"json://127.0.0.1\r\n\njson" in res.data
-    assert not b"json://127.0.0.1\r\n\rjson" in res.data
-
-    assert b"json://127.0.0.1" in res.data
-    assert b"json://128.0.0.1" in res.data
 
+    assert b"json://myapi.com" in res.data
     assert b"126" in res.data
     assert b".fooclass" in res.data
     assert b"My title" in res.data
@@ -62,8 +56,8 @@ def test_check_recheck_global_setting(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-               "requests-time_between_check-minutes": 1566,
+               "minutes_between_check": 1566,
-               'application-fetch_backend': "html_requests"
+               'fetch_backend': "html_requests"
                },
         follow_redirects=True
     )
@@ -94,8 +88,8 @@ def test_check_recheck_global_setting(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-               "requests-time_between_check-minutes": 222,
+               "minutes_between_check": 222,
-                'application-fetch_backend': "html_requests"
+                'fetch_backend': "html_requests"
                },
         follow_redirects=True
     )
@@ -114,7 +108,7 @@ def test_check_recheck_global_setting(client, live_server):
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"url": test_url,
-              "time_between_check-minutes": 55,
+              "minutes_between_check": 55,
               'fetch_backend': "html_requests"
               },
         follow_redirects=True
@@ -130,8 +124,8 @@ def test_check_recheck_global_setting(client, live_server):
     res = client.post(
         url_for("settings_page"),
         data={
-               "requests-time_between_check-minutes": 666,
+               "minutes_between_check": 666,
-                "application-fetch_backend": "html_requests"
+                'fetch_backend': "html_requests"
                },
         follow_redirects=True
     )
@@ -140,7 +134,7 @@ def test_check_recheck_global_setting(client, live_server):
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"url": test_url,
-              "time_between_check-minutes": "",
+              "minutes_between_check": "",
               'fetch_backend': "html_requests"
               },
         follow_redirects=True
@@ -153,3 +147,4 @@ def test_check_recheck_global_setting(client, live_server):
         follow_redirects=True
     )
     assert b"666" in res.data
+

changedetectionio/tests/util.py

@@ -10,7 +10,6 @@ def set_original_response():
      <p>Which is across multiple lines</p>
      </br>
      So let's see what happens.  </br>
-     <span class="foobar-detection" style='display:none'></span>
      </body>
      </html>
     """
@@ -85,7 +84,6 @@ def live_server_setup(live_server):
     # Just return the body in the request
     @live_server.app.route('/test-body', methods=['POST', 'GET'])
     def test_body():
-        print ("TEST-BODY GOT", request.data, "returning")
         return request.data
 
     # Just return the verb in the request

changedetectionio/update_worker.py

@@ -2,7 +2,6 @@ import threading
 import queue
 import time
 
-from changedetectionio import content_fetcher
 # A single update worker
 #
 # Requests for checking on a single site(watch) from a queue of watches
@@ -33,6 +32,7 @@ class update_worker(threading.Thread):
 
             else:
                 self.current_uuid = uuid
+                from changedetectionio import content_fetcher
 
                 if uuid in list(self.datastore.data['watching'].keys()):
 

docker-compose.yml

@@ -17,19 +17,12 @@ services:
   #       Alternative WebDriver/selenium URL, do not use "'s or 's!
   #      - WEBDRIVER_URL=http://browser-chrome:4444/wd/hub
   #
-  #       WebDriver proxy settings webdriver_proxyType, webdriver_ftpProxy, webdriver_noProxy,
+  #       WebDriver proxy settings webdriver_proxyType, webdriver_ftpProxy, webdriver_httpProxy, webdriver_noProxy,
-  #                                webdriver_proxyAutoconfigUrl, webdriver_autodetect,
+  #                                webdriver_proxyAutoconfigUrl, webdriver_sslProxy, webdriver_autodetect,
   #                                webdriver_socksProxy, webdriver_socksUsername, webdriver_socksVersion, webdriver_socksPassword
   #
   #             https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.proxy
   #
-  #       Alternative Playwright URL, do not use "'s or 's!
-  #      - PLAYWRIGHT_DRIVER_URL=ws://playwright-chrome:3000/playwright
-  #
-  #       Playwright proxy settings playwright_proxy_server, playwright_proxy_bypass, playwright_proxy_username, playwright_proxy_password
-  #
-  #             https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch-option-proxy
-  #
   #        Plain requsts - proxy support example.
   #      - HTTP_PROXY=socks5h://10.10.1.10:1080
   #      - HTTPS_PROXY=socks5h://10.10.1.10:1080
@@ -65,13 +58,6 @@ services:
 #            # Workaround to avoid the browser crashing inside a docker container
 #            # See https://github.com/SeleniumHQ/docker-selenium#quick-start
 #            - /dev/shm:/dev/shm
-#        restart: unless-stopped
-
-     # Used for fetching pages via Playwright+Chrome where you need Javascript support.
-
-#    playwright-chrome:
-#        hostname: playwright-chrome
-#        image: browserless/chrome
 #        restart: unless-stopped
 
 volumes:

requirements.txt

@@ -17,7 +17,7 @@ wtforms ~= 3.0
 jsonpath-ng ~= 1.5.3
 
 # Notification library
-apprise ~= 0.9.8.3
+apprise ~= 0.9.7
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
@@ -40,4 +40,3 @@ selenium ~= 4.1.0
 # need to revisit flask login versions
 werkzeug ~= 2.0.0
 
-# playwright is installed at Dockerfile build time because it's not available on all platforms