0.39.12

Co-authored-by: Ara Hayrabedian <ara.hayrabedian@gmail.com>

.github/workflows/containers.yml

@@ -2,16 +2,20 @@ name: Build and push containers
 
 on:
   # Automatically triggered by a testing workflow passing, but this is only checked when it lands in the `master`/default branch
-  workflow_run:
-    workflows: ["ChangeDetection.io Test"]
-    branches: [master]
-    tags: ['0.*']
-    types: [completed]
+#  workflow_run:
+#    workflows: ["ChangeDetection.io Test"]
+#    branches: [master]
+#    tags: ['0.*']
+#    types: [completed]
 
   # Or a new tagged release
   release:
     types: [published, edited]
 
+  push:
+    branches:
+      - master
+
 jobs:
   metadata:
     runs-on: ubuntu-latest
@@ -91,8 +95,7 @@ jobs:
           file: ./Dockerfile
           push: true
           tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
-            ghcr.io/${{ github.repository }}:latest
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest,ghcr.io/${{ github.repository }}:latest
           platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
@@ -107,8 +110,7 @@ jobs:
           file: ./Dockerfile
           push: true
           tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
-            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }},ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
           platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache

.gitignore

@@ -7,4 +7,6 @@ __pycache__
 .pytest_cache
 build
 dist
+venv
+*.egg-info*
 .vscode/settings.json

MANIFEST.in

@@ -2,5 +2,5 @@ recursive-include changedetectionio/templates *
 recursive-include changedetectionio/static *
 include changedetection.py
 global-exclude *.pyc
-global-exclude *node_modules*
+global-exclude node_modules
 global-exclude venv

README.md

@@ -9,10 +9,10 @@ _Know when web pages change! Stay ontop of new information!_
 
 Live your data-life *pro-actively* instead of *re-actively*.
 
-Open source web page monitoring, notification and change detection.
+Free, Open-source web page monitoring, notification and change detection. Don't have time? [Try our $6.99/month plan - unlimited checks and watches!](https://lemonade.changedetection.io/start)
 
 
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start)
 
 
 **Get your own private instance now! Let us host it for you!**
@@ -70,6 +70,10 @@ Docker standalone
 $ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```
 
+### Windows
+
+See the install instructions at the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Microsoft-Windows
+
 ### Python Pip
 
 Check out our pypi page https://pypi.org/project/changedetection.io/
@@ -163,9 +167,6 @@ See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configura
 
 Raspberry Pi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! See the wiki for [details](https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver)
 
-## Windows native support?
-
-Sorry not yet :( https://github.com/dgtlmoon/changedetection.io/labels/windows
 
 ## Support us
 

changedetection.py

@@ -1,110 +1,11 @@
 #!/usr/bin/python3
 
-# Launch as a eventlet.wsgi server instance.
-
-import getopt
-import os
-import sys
-
-import eventlet
-import eventlet.wsgi
-import changedetectionio
-
-from changedetectionio import store
-
-def main():
-    ssl_mode = False
-    host = ''
-    port = os.environ.get('PORT') or 5000
-    do_cleanup = False
-
-    # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
-    datastore_path = os.path.join(os.getcwd(), "datastore")
-
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "Ccsd:h:p:", "port")
-    except getopt.GetoptError:
-        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
-        sys.exit(2)
-
-    create_datastore_dir = False
-
-    for opt, arg in opts:
-        #        if opt == '--purge':
-        # Remove history, the actual files you need to delete manually.
-        #            for uuid, watch in datastore.data['watching'].items():
-        #                watch.update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': None})
-
-        if opt == '-s':
-            ssl_mode = True
-
-        if opt == '-h':
-            host = arg
-
-        if opt == '-p':
-            port = int(arg)
-
-        if opt == '-d':
-            datastore_path = arg
-
-        # Cleanup (remove text files that arent in the index)
-        if opt == '-c':
-            do_cleanup = True
-
-        # Create the datadir if it doesnt exist
-        if opt == '-C':
-            create_datastore_dir = True
-
-    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
-    app_config = {'datastore_path': datastore_path}
-
-    if not os.path.isdir(app_config['datastore_path']):
-        if create_datastore_dir:
-            os.mkdir(app_config['datastore_path'])
-        else:
-            print ("ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists.\n"
-                   "Alternatively, use the -C parameter.".format(app_config['datastore_path']),file=sys.stderr)
-            sys.exit(2)
-
-    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=changedetectionio.__version__)
-    app = changedetectionio.changedetection_app(app_config, datastore)
-
-    # Go into cleanup mode
-    if do_cleanup:
-        datastore.remove_unused_snapshots()
-
-    app.config['datastore_path'] = datastore_path
-
-
-    @app.context_processor
-    def inject_version():
-        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
-                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
-                    has_password=datastore.data['settings']['application']['password'] != False
-                    )
-
-    # Proxy sub-directory support
-    # Set environment var USE_X_SETTINGS=1 on this script
-    # And then in your proxy_pass settings
-    #
-    #         proxy_set_header Host "localhost";
-    #         proxy_set_header X-Forwarded-Prefix /app;
-
-    if os.getenv('USE_X_SETTINGS'):
-        print ("USE_X_SETTINGS is ENABLED\n")
-        from werkzeug.middleware.proxy_fix import ProxyFix
-        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
-
-    if ssl_mode:
-        # @todo finalise SSL config, but this should get you in the right direction if you need it.
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port)),
-                                               certfile='cert.pem',
-                                               keyfile='privkey.pem',
-                                               server_side=True), app)
-
-    else:
-        eventlet.wsgi.server(eventlet.listen((host, int(port))), app)
+# Entry-point for running from the CLI when not installed via Pip, Pip will handle the console_scripts entry_points's from setup.py
+# It's recommended to use `pip3 install changedetection.io` and start with `changedetection.py` instead, it will be linkd to your global path.
+# or Docker.
+# Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
+from changedetectionio import changedetection
 
 if __name__ == '__main__':
-    main()
+    changedetection.main()

changedetectionio/.gitignore

@@ -0,0 +1 @@
+test-datastore

changedetectionio/__init__.py

@@ -35,9 +35,11 @@ from flask import (
     url_for,
 )
 from flask_login import login_required
+from flask_wtf import CSRFProtect
+
 from changedetectionio import html_tools
 
-__version__ = '0.39.9'
+__version__ = '0.39.12'
 
 datastore = None
 
@@ -51,11 +53,10 @@ update_q = queue.Queue()
 
 notification_q = queue.Queue()
 
-# Needs to be set this way because we also build and publish via pip
-base_path = os.path.dirname(os.path.realpath(__file__))
 app = Flask(__name__,
-            static_url_path="{}/static".format(base_path),
-            template_folder="{}/templates".format(base_path))
+            static_url_path="",
+            static_folder="static",
+            template_folder="templates")
 
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
@@ -71,6 +72,9 @@ app.config['LOGIN_DISABLED'] = False
 # Disables caching of the templates
 app.config['TEMPLATES_AUTO_RELOAD'] = True
 
+csrf = CSRFProtect()
+csrf.init_app(app)
+
 notification_debug_log=[]
 
 def init_app_secret(datastore_path):
@@ -127,7 +131,7 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
     # return timeago.format(timestamp, time.time())
     # return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
 
-
+# When nobody is logged in Flask-Login's current_user is set to an AnonymousUser object.
 class User(flask_login.UserMixin):
     id=None
 
@@ -136,7 +140,6 @@ class User(flask_login.UserMixin):
     def get_user(self, email="defaultuser@changedetection.io"):
         return self
     def is_authenticated(self):
-
         return True
     def is_active(self):
         return True
@@ -215,6 +218,10 @@ def changedetection_app(config=None, datastore_o=None):
             return redirect(url_for('index'))
 
         if request.method == 'GET':
+            if flask_login.current_user.is_authenticated:
+                flash("Already logged in")
+                return redirect(url_for("index"))
+
             output = render_template("login.html")
             return output
 
@@ -250,6 +257,11 @@ def changedetection_app(config=None, datastore_o=None):
         # (No password in settings or env var)
         app.config['LOGIN_DISABLED'] = datastore.data['settings']['application']['password'] == False and os.getenv("SALTED_PASS", False) == False
 
+        # Set the auth cookie path if we're running as X-settings/X-Forwarded-Prefix
+        if os.getenv('USE_X_SETTINGS') and 'X-Forwarded-Prefix' in request.headers:
+            app.config['REMEMBER_COOKIE_PATH'] = request.headers['X-Forwarded-Prefix']
+            app.config['SESSION_COOKIE_PATH'] = request.headers['X-Forwarded-Prefix']
+
         # For the RSS path, allow access via a token
         if request.path == '/rss' and request.args.get('token'):
             app_rss_token = datastore.data['settings']['application']['rss_access_token']
@@ -260,7 +272,7 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/rss", methods=['GET'])
     @login_required
     def rss():
-
+        from . import diff
         limit_tag = request.args.get('tag')
 
         # Sort by last_changed and add the uuid which is usually the key..
@@ -289,13 +301,25 @@ def changedetection_app(config=None, datastore_o=None):
         fg.link(href='https://changedetection.io')
 
         for watch in sorted_watches:
+
+            dates = list(watch['history'].keys())
+            # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected.
+            if len(dates) < 2:
+                continue
+
+            # Convert to int, sort and back to str again
+            # @todo replace datastore getter that does this automatically
+            dates = [int(i) for i in dates]
+            dates.sort(reverse=True)
+            dates = [str(i) for i in dates]
+            prev_fname = watch['history'][dates[1]]
+
             if not watch['viewed']:
                 # Re #239 - GUID needs to be individual for each event
                 # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228)
                 guid = "{}/{}".format(watch['uuid'], watch['last_changed'])
                 fe = fg.add_entry()
 
-
                 # Include a link to the diff page, they will have to login here to see if password protection is enabled.
                 # Description is the page you watch, link takes you to the diff JS UI page
                 base_url = datastore.data['settings']['application']['base_url']
@@ -304,12 +328,16 @@ def changedetection_app(config=None, datastore_o=None):
 
                 diff_link = {'href': "{}{}".format(base_url, url_for('diff_history_page', uuid=watch['uuid']))}
 
-                # @todo use title if it exists
                 fe.link(link=diff_link)
-                fe.title(title=watch['url'])
 
-                # @todo in the future <description><![CDATA[<html><body>Any code html is valid.</body></html>]]></description>
-                fe.description(description=watch['url'])
+                # @todo watch should be a getter - watch.get('title') (internally if URL else..)
+
+                watch_title = watch.get('title') if watch.get('title') else watch.get('url')
+                fe.title(title=watch_title)
+                latest_fname = watch['history'][dates[0]]
+
+                html_diff = diff.render_diff(prev_fname, latest_fname, include_equal=False, line_feed_sep="</br>")
+                fe.description(description="<![CDATA[<html><body><h4>{}</h4>{}</body></html>".format(watch_title, html_diff))
 
                 fe.guid(guid, permalink=False)
                 dt = datetime.datetime.fromtimestamp(int(watch['newest_history_key']))
@@ -323,7 +351,6 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/", methods=['GET'])
     @login_required
     def index():
-
         limit_tag = request.args.get('tag')
         pause_uuid = request.args.get('pause')
 
@@ -333,17 +360,12 @@ def changedetection_app(config=None, datastore_o=None):
 
         if pause_uuid:
             try:
-                if pause_uuid == 'pause-all' or pause_uuid == 'resume-all':
-                    action = True if pause_uuid == 'pause-all' else False
-                    for watch_uuid, watch in datastore.data['watching'].items():
-                        if datastore.data['watching'][watch_uuid]['tag'] == limit_tag or limit_tag is None :
-                            datastore.data['watching'][watch_uuid]['paused'] = action
-                else :
-                    datastore.data['watching'][pause_uuid]['paused'] ^= True
+                datastore.data['watching'][pause_uuid]['paused'] ^= True
                 datastore.needs_write = True
+
+                return redirect(url_for('index', tag = limit_tag))
             except KeyError:
-                flash("No watch by that UUID found, or error setting paused state.", 'error');
-            return redirect(url_for('index', tag = limit_tag))
+                pass
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
@@ -368,12 +390,6 @@ def changedetection_app(config=None, datastore_o=None):
         from changedetectionio import forms
         form = forms.quickWatchForm(request.form)
 
-        # Extra page <title> (n) unviewed
-        extra_title = ""
-        if datastore.data['unviewed_count'] > 0:
-            extra_title = " ({})".format(str(datastore.data['unviewed_count']))
-
-
         output = render_template("watch-overview.html",
                                  form=form,
                                  watches=sorted_watches,
@@ -383,12 +399,42 @@ def changedetection_app(config=None, datastore_o=None):
                                  has_unviewed=datastore.data['has_unviewed'],
                                  # Don't link to hosting when we're on the hosting environment
                                  hosted_sticky=os.getenv("SALTED_PASS", False) == False,
-                                 guid=datastore.data['app_guid'],
-                                 extra_title=extra_title
-                                )
+                                 guid=datastore.data['app_guid'])
 
         return output
 
+
+    # AJAX endpoint for sending a test
+    @app.route("/notification/send-test", methods=['POST'])
+    @login_required
+    def ajax_callback_send_notification_test():
+
+        import apprise
+        apobj = apprise.Apprise()
+
+        # validate URLS
+        if not len(request.form['notification_urls'].strip()):
+            return make_response({'error': 'No Notification URLs set'}, 400)
+
+        for server_url in request.form['notification_urls'].splitlines():
+            if len(server_url.strip()):
+                if not apobj.add(server_url):
+                    message = '{} is not a valid AppRise URL.'.format(server_url)
+                    return make_response({'error': message}, 400)
+
+        try:
+            n_object = {'watch_url': request.form['window_url'],
+                        'notification_urls': request.form['notification_urls'].splitlines(),
+                        'notification_title': request.form['notification_title'].strip(),
+                        'notification_body': request.form['notification_body'].strip(),
+                        'notification_format': request.form['notification_format'].strip()
+                        }
+            notification_q.put(n_object)
+        except Exception as e:
+            return make_response({'error': str(e)}, 400)
+
+        return 'OK'
+
     @app.route("/scrub", methods=['GET', 'POST'])
     @login_required
     def scrub_page():
@@ -509,13 +555,13 @@ def changedetection_app(config=None, datastore_o=None):
                           'headers': form.headers.data,
                           'body': form.body.data,
                           'method': form.method.data,
+                          'ignore_status_codes': form.ignore_status_codes.data,
                           'fetch_backend': form.fetch_backend.data,
                           'trigger_text': form.trigger_text.data,
                           'notification_title': form.notification_title.data,
                           'notification_body': form.notification_body.data,
                           'notification_format': form.notification_format.data,
-                          'extract_title_as_title': form.extract_title_as_title.data
-
+                          'extract_title_as_title': form.extract_title_as_title.data,
                           }
 
             # Notification URLs
@@ -532,6 +578,7 @@ def changedetection_app(config=None, datastore_o=None):
 
 
             datastore.data['watching'][uuid]['css_filter'] = form.css_filter.data.strip()
+            datastore.data['watching'][uuid]['subtractive_selectors'] = form.subtractive_selectors.data
 
             # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
             if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']:
@@ -549,20 +596,6 @@ def changedetection_app(config=None, datastore_o=None):
             # Queue the watch for immediate recheck
             update_q.put(uuid)
 
-            if form.trigger_check.data:
-                if len(form.notification_urls.data):
-                    n_object = {'watch_url': form.url.data.strip(),
-                                'notification_urls': form.notification_urls.data,
-                                'notification_title': form.notification_title.data,
-                                'notification_body': form.notification_body.data,
-                                'notification_format': form.notification_format.data,
-                                'uuid': uuid
-                                }
-                    notification_q.put(n_object)
-                    flash('Test notification queued.')
-                else:
-                    flash('No notification URLs set, cannot send test.', 'error')
-
             # Diff page [edit] link should go back to diff page
             if request.args.get("next") and request.args.get("next") == 'diff' and not form.save_and_preview_button.data:
                 return redirect(url_for('diff_history_page', uuid=uuid))
@@ -588,7 +621,8 @@ def changedetection_app(config=None, datastore_o=None):
                                      watch=datastore.data['watching'][uuid],
                                      form=form,
                                      using_default_minutes=using_default_minutes,
-                                     current_base_url = datastore.data['settings']['application']['base_url']
+                                     current_base_url=datastore.data['settings']['application']['base_url'],
+                                     emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)
                                      )
 
         return output
@@ -604,25 +638,27 @@ def changedetection_app(config=None, datastore_o=None):
         if request.method == 'GET':
             form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check'])
             form.notification_urls.data = datastore.data['settings']['application']['notification_urls']
+            form.global_subtractive_selectors.data = datastore.data['settings']['application']['global_subtractive_selectors']
             form.global_ignore_text.data = datastore.data['settings']['application']['global_ignore_text']
             form.ignore_whitespace.data = datastore.data['settings']['application']['ignore_whitespace']
+            form.render_anchor_tag_content.data = datastore.data['settings']['application']['render_anchor_tag_content']
             form.extract_title_as_title.data = datastore.data['settings']['application']['extract_title_as_title']
             form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
             form.notification_title.data = datastore.data['settings']['application']['notification_title']
             form.notification_body.data = datastore.data['settings']['application']['notification_body']
             form.notification_format.data = datastore.data['settings']['application']['notification_format']
             form.base_url.data = datastore.data['settings']['application']['base_url']
+            form.real_browser_save_screenshot.data = datastore.data['settings']['application']['real_browser_save_screenshot']
 
-            # Password unset is a GET, but we can lock the session to always need the password
-            if not os.getenv("SALTED_PASS", False) and request.values.get('removepassword') == 'yes':
-                from pathlib import Path
+        if request.method == 'POST' and form.data.get('removepassword_button') == True:
+            # Password unset is a GET, but we can lock the session to a salted env password to always need the password
+            if not os.getenv("SALTED_PASS", False):
                 datastore.data['settings']['application']['password'] = False
                 flash("Password protection removed.", 'notice')
                 flask_login.logout_user()
                 return redirect(url_for('settings_page'))
 
         if request.method == 'POST' and form.validate():
-
             datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
             datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data
             datastore.data['settings']['application']['extract_title_as_title'] = form.extract_title_as_title.data
@@ -632,21 +668,11 @@ def changedetection_app(config=None, datastore_o=None):
             datastore.data['settings']['application']['notification_format'] = form.notification_format.data
             datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
             datastore.data['settings']['application']['base_url'] = form.base_url.data
+            datastore.data['settings']['application']['global_subtractive_selectors'] = form.global_subtractive_selectors.data
             datastore.data['settings']['application']['global_ignore_text'] =  form.global_ignore_text.data
             datastore.data['settings']['application']['ignore_whitespace'] = form.ignore_whitespace.data
-
-            if form.trigger_check.data:
-                if len(form.notification_urls.data):
-                    n_object = {'watch_url': "Test from changedetection.io!",
-                                'notification_urls': form.notification_urls.data,
-                                'notification_title': form.notification_title.data,
-                                'notification_body': form.notification_body.data,
-                                'notification_format': form.notification_format.data,
-                                }
-                    notification_q.put(n_object)
-                    flash('Test notification queued.')
-                else:
-                    flash('No notification URLs set, cannot send test.', 'error')
+            datastore.data['settings']['application']['real_browser_save_screenshot'] = form.real_browser_save_screenshot.data
+            datastore.data['settings']['application']['render_anchor_tag_content'] = form.render_anchor_tag_content.data
 
             if not os.getenv("SALTED_PASS", False) and form.password.encrypted_password:
                 datastore.data['settings']['application']['password'] = form.password.encrypted_password
@@ -663,7 +689,8 @@ def changedetection_app(config=None, datastore_o=None):
         output = render_template("settings.html",
                                  form=form,
                                  current_base_url = datastore.data['settings']['application']['base_url'],
-                                 hide_remove_pass=os.getenv("SALTED_PASS", False))
+                                 hide_remove_pass=os.getenv("SALTED_PASS", False),
+                                 emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False))
 
         return output
 
@@ -678,10 +705,8 @@ def changedetection_app(config=None, datastore_o=None):
         if request.method == 'POST':
             urls = request.values.get('urls').split("\n")
             for url in urls:
-              
-                url, *tags = url.split(" ")
                 url = url.strip()
-
+                url, *tags = url.split(" ")
                 # Flask wtform validators wont work with basic auth, use validators package
                 if len(url) and validators.url(url):
                     new_uuid = datastore.add_watch(url=url.strip(), tag=" ".join(tags))
@@ -709,130 +734,12 @@ def changedetection_app(config=None, datastore_o=None):
     @login_required
     def mark_all_viewed():
 
-        limit_tag = request.args.get('tag')
-
         # Save the current newest history as the most recently viewed
-        try: 
-            for watch_uuid, watch in datastore.data['watching'].items():
-                if datastore.data['watching'][watch_uuid]['tag'] == limit_tag or limit_tag is None :
-                    datastore.set_last_viewed(watch_uuid, watch['newest_history_key'])
-            
-            datastore.needs_write = True
+        for watch_uuid, watch in datastore.data['watching'].items():
+            datastore.set_last_viewed(watch_uuid, watch['newest_history_key'])
 
-            return redirect(url_for('index', tag = limit_tag))
-        except KeyError:
-            pass
-
-    # process selected
-    @app.route("/api/process-selected", methods=["POST"])
-    @login_required
-    def process_selected():
-
-        func = request.form.get('func')
-        limit_tag = request.form.get('tag')
-        uuids = request.form.get('uuids')
-
-        if uuids == '' :
-            flash("No watches selected.")
-            
-        else :
-
-            if func == 'recheck_selected' :
-
-                i = 0
-
-                running_uuids = []
-                for t in running_update_threads:
-                    running_uuids.append(t.current_uuid)
-
-                try :
-                    for uuid in uuids.split(',') :
-                        if uuid not in running_uuids and not datastore.data['watching'][uuid]['paused']:
-                            update_q.put(uuid)
-                            i += 1
-                    
-                except KeyError :
-                    pass
-
-                flash("{} watch{} {} rechecking.".format(i, "" if (i == 1) else "es", "is" if (i == 1) else "are"), "notice")
-                #flash("{} watches are rechecking.".format(i))
-            
-            # Clear selected statuses, so we do not see the 'unviewed' class
-            elif func == 'mark_selected_viewed' :
-
-                try :
-                    for uuid in uuids.split(',') :
-                        datastore.data['watching'][uuid]['last_viewed'] = datastore.data['watching'][uuid]['newest_history_key']
-               
-                except KeyError :
-                    pass
-
-                datastore.needs_write = True
-
-            # Reset selected statuses, so we see the 'unviewed' class
-            # both funcs will contain the uuid list from the processChecked javascript function
-            elif func == 'mark_selected_notviewed' or func == 'mark_all_notviewed' :
-
-                # count within limit_tag and count successes and capture unchanged
-                tagged = 0
-                marked = 0
-                unchanged = []
-                
-                try :
-                    for uuid in uuids.split(',') :
-                        # increment count with limit_tag
-                        tagged += 1
-                        dates = list(datastore.data['watching'][uuid]['history'].keys())
-                        # Convert to int, sort and back to str again
-                        dates = [int(i) for i in dates]
-                        dates.sort(reverse=True)
-                        dates = [str(i) for i in dates]
-
-                        # must be more than 1 history to mark as not viewed
-                        if len(dates) > 1 :
-                            # Save the next earliest history as the most recently viewed
-                            datastore.set_last_viewed(uuid, dates[1])
-                            # increment successes
-                            marked += 1
-                            
-                        else : 
-                            if datastore.data['watching'][uuid]['title'] :
-                                unchanged.append(datastore.data['watching'][uuid]['title'])
-                            else :
-                                unchanged.append(datastore.data['watching'][uuid]['url'])
-
-                except KeyError :
-                        pass
-
-                datastore.needs_write = True
-
-                if marked < tagged :
-                    flash("The following {} not have enough history to be remarked:".format("watch does" if len(unchanged) == 1 else "watches do"), "notice")
-                    for i in range(len(unchanged)):
-                        flash(unchanged[i], "notice")
-                
-            elif func == 'delete_selected' :
-
-                # reachable only after confirmation in javascript processChecked(func, tag) function
-                try :
-                    i = 0
-                    for uuid in uuids.split(',') :
-                        datastore.delete(uuid)
-                        i += 1
-                        
-                except KeyError :
-                    pass
-
-                datastore.needs_write = True
-                
-                flash("{0} {1} deleted.".format(i, "watch was" if (i) == 1 else "watches were")) 
-            
-            else :
-                
-                flash("Invalid parameter received.")
-
-        render_template(url_for('index'), tag = limit_tag)
-        return index()
+        flash("Cleared all statuses.")
+        return redirect(url_for('index'))
 
     @app.route("/diff/<string:uuid>", methods=['GET'])
     @login_required
@@ -883,6 +790,9 @@ def changedetection_app(config=None, datastore_o=None):
         except Exception as e:
             previous_version_file_contents = "Unable to read {}.\n".format(previous_file)
 
+
+        screenshot_url = datastore.get_screenshot(uuid)
+
         output = render_template("diff.html", watch_a=watch,
                                  newest=newest_version_file_contents,
                                  previous=previous_version_file_contents,
@@ -893,7 +803,8 @@ def changedetection_app(config=None, datastore_o=None):
                                  current_previous_version=str(previous_version),
                                  current_diff_url=watch['url'],
                                  extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
-                                 left_sticky=True)
+                                 left_sticky=True,
+                                 screenshot=screenshot_url)
 
         return output
 
@@ -953,15 +864,17 @@ def changedetection_app(config=None, datastore_o=None):
         else:
             content.append({'line': "No history found", 'classes': ''})
 
-
+        screenshot_url = datastore.get_screenshot(uuid)
         output = render_template("preview.html",
                                  content=content,
                                  extra_stylesheets=extra_stylesheets,
                                  ignored_line_numbers=ignored_line_numbers,
                                  triggered_line_numbers=trigger_line_numbers,
                                  current_diff_url=watch['url'],
+                                 screenshot=screenshot_url,
                                  watch=watch,
                                  uuid=uuid)
+        
         return output
 
     @app.route("/settings/notification-logs", methods=['GET'])
@@ -1074,6 +987,28 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
     def static_content(group, filename):
+        if group == 'screenshot':
+
+            from flask import make_response
+
+            # Could be sensitive, follow password requirements
+            if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
+                abort(403)
+
+            # These files should be in our subdirectory
+            try:
+                # set nocache, set content-type
+                watch_dir = datastore_o.datastore_path + "/" + filename
+                response = make_response(send_from_directory(filename="last-screenshot.png", directory=watch_dir, path=watch_dir + "/last-screenshot.png"))
+                response.headers['Content-type'] = 'image/png'
+                response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                response.headers['Pragma'] = 'no-cache'
+                response.headers['Expires'] = 0
+                return response
+
+            except FileNotFoundError:
+                abort(404)
+
         # These files should be in our subdirectory
         try:
             return send_from_directory("static/{}".format(group), path=filename)
@@ -1088,10 +1023,6 @@ def changedetection_app(config=None, datastore_o=None):
 
         if form.validate():
 
-            # get action parameter (add paused button value is 'add', watch button value is 'watch'
-            #action = request.form.get('action')
-            add_paused = request.form.get('add-paused')
-            
             url = request.form.get('url').strip()
             if datastore.url_exists(url):
                 flash('The URL {} already exists'.format(url), "error")
@@ -1099,26 +1030,23 @@ def changedetection_app(config=None, datastore_o=None):
 
             # @todo add_watch should throw a custom Exception for validation etc
             new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
-            
-            if add_paused :
-                datastore.data['watching'][new_uuid]['paused'] = True
-                datastore.needs_write = True
-                flash("Watch added in a paused state.")
-                                
-            else : # watch now
-                # Straight into the queue. 
-                update_q.put(new_uuid)
+            # Straight into the queue.
+            update_q.put(new_uuid)
 
-                flash("Watch added.")
+            flash("Watch added.")
             return redirect(url_for('index'))
         else:
             flash("Error")
             return redirect(url_for('index'))
 
+
     @app.route("/api/delete", methods=['GET'])
     @login_required
     def api_delete():
         uuid = request.args.get('uuid')
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
         datastore.delete(uuid)
         flash('Deleted.')
 
@@ -1172,9 +1100,7 @@ def changedetection_app(config=None, datastore_o=None):
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                     update_q.put(watch_uuid)
                     i += 1
-
         flash("{} watches are queued for rechecking.".format(i))
-
         return redirect(url_for('index', tag=tag))
 
     # @todo handle ctrl break

changedetectionio/changedetection.py

@@ -0,0 +1,114 @@
+#!/usr/bin/python3
+
+# Launch as a eventlet.wsgi server instance.
+
+import getopt
+import os
+import sys
+
+import eventlet
+import eventlet.wsgi
+from . import store, changedetection_app
+from . import __version__
+
+def main():
+    ssl_mode = False
+    host = ''
+    port = os.environ.get('PORT') or 5000
+    do_cleanup = False
+    datastore_path = None
+
+    # On Windows, create and use a default path.
+    if os.name == 'nt':
+        datastore_path = os.path.expandvars(r'%APPDATA%\changedetection.io')
+        os.makedirs(datastore_path, exist_ok=True)
+    else:
+        # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
+        datastore_path = os.path.join(os.getcwd(), "../datastore")
+
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "Ccsd:h:p:", "port")
+    except getopt.GetoptError:
+        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
+        sys.exit(2)
+
+    create_datastore_dir = False
+
+    for opt, arg in opts:
+        #        if opt == '--purge':
+        # Remove history, the actual files you need to delete manually.
+        #            for uuid, watch in datastore.data['watching'].items():
+        #                watch.update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': None})
+
+        if opt == '-s':
+            ssl_mode = True
+
+        if opt == '-h':
+            host = arg
+
+        if opt == '-p':
+            port = int(arg)
+
+        if opt == '-d':
+            datastore_path = arg
+
+        # Cleanup (remove text files that arent in the index)
+        if opt == '-c':
+            do_cleanup = True
+
+        # Create the datadir if it doesnt exist
+        if opt == '-C':
+            create_datastore_dir = True
+
+    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
+    app_config = {'datastore_path': datastore_path}
+
+    if not os.path.isdir(app_config['datastore_path']):
+        if create_datastore_dir:
+            os.mkdir(app_config['datastore_path'])
+        else:
+            print(
+                "ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists or specify a directory with the -d option.\n"
+                "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
+            sys.exit(2)
+
+    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
+    app = changedetection_app(app_config, datastore)
+
+    # Go into cleanup mode
+    if do_cleanup:
+        datastore.remove_unused_snapshots()
+
+    app.config['datastore_path'] = datastore_path
+
+
+    @app.context_processor
+    def inject_version():
+        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
+                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
+                    has_password=datastore.data['settings']['application']['password'] != False
+                    )
+
+    # Proxy sub-directory support
+    # Set environment var USE_X_SETTINGS=1 on this script
+    # And then in your proxy_pass settings
+    #
+    #         proxy_set_header Host "localhost";
+    #         proxy_set_header X-Forwarded-Prefix /app;
+
+    if os.getenv('USE_X_SETTINGS'):
+        print ("USE_X_SETTINGS is ENABLED\n")
+        from werkzeug.middleware.proxy_fix import ProxyFix
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
+
+    if ssl_mode:
+        # @todo finalise SSL config, but this should get you in the right direction if you need it.
+        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port)),
+                                               certfile='cert.pem',
+                                               keyfile='privkey.pem',
+                                               server_side=True), app)
+
+    else:
+        eventlet.wsgi.server(eventlet.listen((host, int(port))), app)
+
+

changedetectionio/content_fetcher.py

@@ -1,10 +1,12 @@
-import os
-import time
 from abc import ABC, abstractmethod
+import chardet
+import os
 from selenium import webdriver
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
 from selenium.common.exceptions import WebDriverException
+import requests
+import time
 import urllib3.exceptions
 
 
@@ -20,7 +22,7 @@ class EmptyReply(Exception):
 class Fetcher():
     error = None
     status_code = None
-    content = None # Should always be bytes.
+    content = None
     headers = None
 
     fetcher_description ="No description"
@@ -30,10 +32,24 @@ class Fetcher():
         return self.error
 
     @abstractmethod
-    def run(self, url, timeout, request_headers, request_body, request_method):
+    def run(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False):
         # Should set self.error, self.status_code and self.content
         pass
 
+    @abstractmethod
+    def quit(self):
+        return
+
+    @abstractmethod
+    def screenshot(self):
+        return
+
     @abstractmethod
     def get_last_status_code(self):
         return self.status_code
@@ -97,21 +113,27 @@ class html_webdriver(Fetcher):
         if proxy_args:
             self.proxy = SeleniumProxy(raw=proxy_args)
 
-    def run(self, url, timeout, request_headers, request_body, request_method):
+    def run(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False):
 
         # request_body, request_method unused for now, until some magic in the future happens.
 
         # check env for WEBDRIVER_URL
-        driver = webdriver.Remote(
+        self.driver = webdriver.Remote(
             command_executor=self.command_executor,
             desired_capabilities=DesiredCapabilities.CHROME,
             proxy=self.proxy)
 
         try:
-            driver.get(url)
+            self.driver.get(url)
         except WebDriverException as e:
             # Be sure we close the session window
-            driver.quit()
+            self.quit()
             raise
 
         # @todo - how to check this? is it possible?
@@ -121,32 +143,44 @@ class html_webdriver(Fetcher):
 
         # @todo - dom wait loaded?
         time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
-        self.content = driver.page_source
+        self.content = self.driver.page_source
         self.headers = {}
 
-        driver.quit()
-
+    def screenshot(self):
+        return self.driver.get_screenshot_as_png()
 
+    # Does the connection to the webdriver work? run a test connection.
     def is_ready(self):
         from selenium import webdriver
         from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
         from selenium.common.exceptions import WebDriverException
 
-        driver = webdriver.Remote(
+        self.driver = webdriver.Remote(
             command_executor=self.command_executor,
             desired_capabilities=DesiredCapabilities.CHROME)
 
         # driver.quit() seems to cause better exceptions
-        driver.quit()
-
+        self.quit()
         return True
 
+    def quit(self):
+        if self.driver:
+            try:
+                self.driver.quit()
+            except Exception as e:
+                print("Exception in chrome shutdown/quit" + str(e))
+
 # "html_requests" is listed as the default fetcher in store.py!
 class html_requests(Fetcher):
     fetcher_description = "Basic fast Plaintext/HTTP Client"
 
-    def run(self, url, timeout, request_headers, request_body, request_method):
-        import requests
+    def run(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False):
 
         r = requests.request(method=request_method,
                          data=request_body,
@@ -155,16 +189,21 @@ class html_requests(Fetcher):
                          timeout=timeout,
                          verify=False)
 
-        # https://stackoverflow.com/questions/44203397/python-requests-get-returns-improperly-decoded-text-instead-of-utf-8
-        # Return bytes here
-        html = r.text
+        # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
+        # For example - some sites don't tell us it's utf-8, but return utf-8 content
+        # This seems to not occur when using webdriver/selenium, it seems to detect the text encoding more reliably.
+        # https://github.com/psf/requests/issues/1604 good info about requests encoding detection
+        if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
+            encoding = chardet.detect(r.content)['encoding']
+            if encoding:
+                r.encoding = encoding
 
         # @todo test this
         # @todo maybe you really want to test zero-byte return pages?
-        if not r or not html or not len(html):
+        if (not ignore_status_codes and not r) or not r.content or not len(r.content):
             raise EmptyReply(url=url, status_code=r.status_code)
 
         self.status_code = r.status_code
-        self.content = html
+        self.content = r.text
         self.headers = r.headers
 

changedetectionio/diff.py

@@ -2,22 +2,31 @@
 
 import difflib
 
+
+def same_slicer(l, a, b):
+    if a == b:
+        return [l[a]]
+    else:
+        return l[a:b]
+
 # like .compare but a little different output
 def customSequenceMatcher(before, after, include_equal=False):
     cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
 
+    # @todo Line-by-line mode instead of buncghed, including `after` that is not in `before` (maybe unset?)
     for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
         if include_equal and tag == 'equal':
             g = before[alo:ahi]
             yield g
         elif tag == 'delete':
-            g = "(removed) {}".format(before[alo])
+            g = ["(removed) " + i for i in same_slicer(before, alo, ahi)]
             yield g
         elif tag == 'replace':
-            g = ["(changed) {}".format(before[alo]), "(-> into) {}".format(after[blo])]
+            g = ["(changed) " + i for i in same_slicer(before, alo, ahi)]
+            g += ["(into   ) " + i for i in same_slicer(after, blo, bhi)]
             yield g
         elif tag == 'insert':
-            g = "(added) {}".format(after[blo])
+            g = ["(added  ) " + i for i in same_slicer(after, blo, bhi)]
             yield g
 
 # only_differences - only return info about the differences, no context

changedetectionio/fetch_site_status.py

@@ -1,11 +1,10 @@
-import time
-from changedetectionio import content_fetcher
-from changedetectionio import html_tools
 import hashlib
-from inscriptis import get_text
-import urllib3
-from . import html_tools
+import os
 import re
+import time
+import urllib3
+
+from changedetectionio import content_fetcher, html_tools
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -21,11 +20,18 @@ class perform_site_check():
         timestamp = int(time.time())  # used for storage etc too
 
         changed_detected = False
+        screenshot = False # as bytes
         stripped_text_from_html = ""
 
         watch = self.datastore.data['watching'][uuid]
-        # Unset any existing notification error
 
+        # Protect against file:// access
+        if re.search(r'^file', watch['url'], re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
+            raise Exception(
+                "file:// type access is denied for security reasons."
+            )
+
+        # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
 
         extra_headers = self.datastore.get_val(uuid, 'headers')
@@ -47,6 +53,7 @@ class perform_site_check():
             url = self.datastore.get_val(uuid, 'url')
             request_body = self.datastore.get_val(uuid, 'body')
             request_method = self.datastore.get_val(uuid, 'method')
+            ignore_status_code = self.datastore.get_val(uuid, 'ignore_status_codes')
 
             # Pluggable content fetcher
             prefer_backend = watch['fetch_backend']
@@ -58,7 +65,7 @@ class perform_site_check():
 
 
             fetcher = klass()
-            fetcher.run(url, timeout, request_headers, request_body, request_method)
+            fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code)
             # Fetching complete, now filters
             # @todo move to class / maybe inside of fetcher abstract base?
 
@@ -72,8 +79,15 @@ class perform_site_check():
             is_json = 'application/json' in fetcher.headers.get('Content-Type', '')
             is_html = not is_json
             css_filter_rule = watch['css_filter']
+            subtractive_selectors = watch.get(
+                "subtractive_selectors", []
+            ) + self.datastore.data["settings"]["application"].get(
+                "global_subtractive_selectors", []
+            )
 
             has_filter_rule = css_filter_rule and len(css_filter_rule.strip())
+            has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
+
             if is_json and not has_filter_rule:
                 css_filter_rule = "json:$"
                 has_filter_rule = True
@@ -100,11 +114,16 @@ class perform_site_check():
                         else:
                             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
                             html_content = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
-
-                    # get_text() via inscriptis
-                    stripped_text_from_html = get_text(html_content)
-
-
+                    if has_subtractive_selectors:
+                        html_content = html_tools.element_removal(subtractive_selectors, html_content)
+                    # extract text
+                    stripped_text_from_html = \
+                        html_tools.html_to_text(
+                            html_content,
+                            render_anchor_tag_content=self.datastore.data["settings"][
+                                "application"].get(
+                                "render_anchor_tag_content", False)
+                        )
             # Re #340 - return the content before the 'ignore text' was applied
             text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
@@ -157,5 +176,9 @@ class perform_site_check():
                     if not watch['title'] or not len(watch['title']):
                         update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
 
+            if self.datastore.data['settings']['application'].get('real_browser_save_screenshot', True):
+                screenshot = fetcher.screenshot()
 
-        return changed_detected, update_obj, text_content_before_ignored_filter
+            fetcher.quit()
+
+        return changed_detected, update_obj, text_content_before_ignored_filter, screenshot

changedetectionio/forms.py

@@ -1,13 +1,30 @@
-from wtforms import Form, SelectField, RadioField, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \
-    Field
-
-from wtforms import widgets, SubmitField
-from wtforms.validators import ValidationError
-from wtforms.fields import html5
-from changedetectionio import content_fetcher
 import re
 
-from changedetectionio.notification import default_notification_format, valid_notification_formats, default_notification_body, default_notification_title
+from wtforms import (
+    BooleanField,
+    Field,
+    Form,
+    IntegerField,
+    PasswordField,
+    RadioField,
+    SelectField,
+    StringField,
+    SubmitField,
+    TextAreaField,
+    fields,
+    validators,
+    widgets,
+)
+from wtforms.fields import html5
+from wtforms.validators import ValidationError
+
+from changedetectionio import content_fetcher
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+    valid_notification_formats,
+)
 
 valid_method = {
     'GET',
@@ -45,8 +62,8 @@ class SaltyPasswordField(StringField):
     encrypted_password = ""
 
     def build_password(self, password):
-        import hashlib
         import base64
+        import hashlib
         import secrets
 
         # Make a new salt on every new password and store it with the password
@@ -104,9 +121,10 @@ class ValidateContentFetcherIsReady(object):
         self.message = message
 
     def __call__(self, form, field):
-        from changedetectionio import content_fetcher
         import urllib3.exceptions
 
+        from changedetectionio import content_fetcher
+
         # Better would be a radiohandler that keeps a reference to each class
         if field.data is not None:
             klass = getattr(content_fetcher, field.data)
@@ -220,44 +238,61 @@ class ValidateCSSJSONXPATHInput(object):
     @todo CSS validator ;)
     """
 
-    def __init__(self, message=None):
+    def __init__(self, message=None, allow_xpath=True, allow_json=True):
         self.message = message
+        self.allow_xpath = allow_xpath
+        self.allow_json = allow_json
 
     def __call__(self, form, field):
 
+        if isinstance(field.data, str):
+            data = [field.data]
+        else:
+            data = field.data
+
+        for line in data:
         # Nothing to see here
-        if not len(field.data.strip()):
-            return
+            if not len(line.strip()):
+                return
 
-        # Does it look like XPath?
-        if field.data.strip()[0] == '/':
-            from lxml import html, etree
-            tree = html.fromstring("<html></html>")
+            # Does it look like XPath?
+            if line.strip()[0] == '/':
+                if not self.allow_xpath:
+                    raise ValidationError("XPath not permitted in this field!")
+                from lxml import etree, html
+                tree = html.fromstring("<html></html>")
 
-            try:
-                tree.xpath(field.data.strip())
-            except etree.XPathEvalError as e:
-                message = field.gettext('\'%s\' is not a valid XPath expression. (%s)')
-                raise ValidationError(message % (field.data, str(e)))
-            except:
-                raise ValidationError("A system-error occurred when validating your XPath expression")
+                try:
+                    tree.xpath(line.strip())
+                except etree.XPathEvalError as e:
+                    message = field.gettext('\'%s\' is not a valid XPath expression. (%s)')
+                    raise ValidationError(message % (line, str(e)))
+                except:
+                    raise ValidationError("A system-error occurred when validating your XPath expression")
 
-        if 'json:' in field.data:
-            from jsonpath_ng.exceptions import JsonPathParserError, JsonPathLexerError
-            from jsonpath_ng.ext import parse
+            if 'json:' in line:
+                if not self.allow_json:
+                    raise ValidationError("JSONPath not permitted in this field!")
 
-            input = field.data.replace('json:', '')
+                from jsonpath_ng.exceptions import (
+                    JsonPathLexerError,
+                    JsonPathParserError,
+                )
+                from jsonpath_ng.ext import parse
 
-            try:
-                parse(input)
-            except (JsonPathParserError, JsonPathLexerError) as e:
-                message = field.gettext('\'%s\' is not a valid JSONPath expression. (%s)')
-                raise ValidationError(message % (input, str(e)))
-            except:
-                raise ValidationError("A system-error occurred when validating your JSONPath expression")
+                input = line.replace('json:', '')
+
+                try:
+                    parse(input)
+                except (JsonPathParserError, JsonPathLexerError) as e:
+                    message = field.gettext('\'%s\' is not a valid JSONPath expression. (%s)')
+                    raise ValidationError(message % (input, str(e)))
+                except:
+                    raise ValidationError("A system-error occurred when validating your JSONPath expression")
+
+                # Re #265 - maybe in the future fetch the page and offer a
+                # warning/notice that its possible the rule doesnt yet match anything?
 
-            # Re #265 - maybe in the future fetch the page and offer a
-            # warning/notice that its possible the rule doesnt yet match anything?
 
 class quickWatchForm(Form):
     # https://wtforms.readthedocs.io/en/2.3.x/fields/#module-wtforms.fields.html5
@@ -267,12 +302,11 @@ class quickWatchForm(Form):
 
 class commonSettingsForm(Form):
 
-    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateNotificationBodyAndTitleWhenURLisSet(), ValidateAppRiseServers()])
-    notification_title = StringField('Notification Title', default=default_notification_title, validators=[validators.Optional(), ValidateTokensList()])
-    notification_body = TextAreaField('Notification Body', default=default_notification_body, validators=[validators.Optional(), ValidateTokensList()])
-    notification_format = SelectField('Notification Format', choices=valid_notification_formats.keys(), default=default_notification_format)
-    trigger_check = BooleanField('Send test notification on save')
-    fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    notification_urls = StringListField('Notification URL list', validators=[validators.Optional(), ValidateNotificationBodyAndTitleWhenURLisSet(), ValidateAppRiseServers()])
+    notification_title = StringField('Notification title', default=default_notification_title, validators=[validators.Optional(), ValidateTokensList()])
+    notification_body = TextAreaField('Notification body', default=default_notification_body, validators=[validators.Optional(), ValidateTokensList()])
+    notification_format = SelectField('Notification format', choices=valid_notification_formats.keys(), default=default_notification_format)
+    fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
 
 class watchForm(commonSettingsForm):
@@ -282,13 +316,15 @@ class watchForm(commonSettingsForm):
 
     minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
                                                [validators.Optional(), validators.NumberRange(min=1)])
-    css_filter = StringField('CSS/JSON/XPATH Filter', [ValidateCSSJSONXPATHInput()])
+    css_filter = StringField('CSS/JSON/XPATH filter', [ValidateCSSJSONXPATHInput()])
+    subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
     title = StringField('Title')
 
-    ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
-    headers = StringDictKeyValue('Request Headers')
-    body = TextAreaField('Request Body', [validators.Optional()])
-    method = SelectField('Request Method', choices=valid_method, default=default_method)
+    ignore_text = StringListField('Ignore text', [ValidateListRegex()])
+    headers = StringDictKeyValue('Request headers')
+    body = TextAreaField('Request body', [validators.Optional()])
+    method = SelectField('Request method', choices=valid_method, default=default_method)
+    ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
     trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
 
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
@@ -308,11 +344,18 @@ class watchForm(commonSettingsForm):
         return result
 
 class globalSettingsForm(commonSettingsForm):
-
     password = SaltyPasswordField()
     minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
                                                [validators.NumberRange(min=1)])
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title')
     base_url = StringField('Base URL', validators=[validators.Optional()])
-    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
-    ignore_whitespace = BooleanField('Ignore whitespace')
+    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
+    global_ignore_text = StringListField('Ignore text', [ValidateListRegex()])
+    ignore_whitespace = BooleanField('Ignore whitespace')
+
+    render_anchor_tag_content = BooleanField('Render anchor tag content',
+                                             default=False)
+
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+    real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome')
+    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})

changedetectionio/html_tools.py

@@ -1,7 +1,13 @@
 import json
+import re
+from typing import List
+
 from bs4 import BeautifulSoup
 from jsonpath_ng.ext import parse
 import re
+from inscriptis import get_text
+from inscriptis.model.config import ParserConfig
+
 
 class JSONNotFound(ValueError):
     def __init__(self, msg):
@@ -16,11 +22,22 @@ def css_filter(css_filter, html_content):
 
     return html_block + "\n"
 
+def subtractive_css_selector(css_selector, html_content):
+    soup = BeautifulSoup(html_content, "html.parser")
+    for item in soup.select(css_selector):
+        item.decompose()
+    return str(soup)
+
+
+def element_removal(selectors: List[str], html_content):
+    """Joins individual filters into one css filter."""
+    selector = ",".join(selectors)
+    return subtractive_css_selector(selector, html_content)
+
 
 # Return str Utf-8 of matched rules
 def xpath_filter(xpath_filter, html_content):
-    from lxml import html
-    from lxml import etree
+    from lxml import etree, html
 
     tree = html.fromstring(html_content)
     html_block = ""
@@ -64,7 +81,8 @@ def _parse_json(json_data, jsonpath_filter):
         # Re 265 - Just return an empty string when filter not found
         return ''
 
-    stripped_text_from_html = json.dumps(s, indent=4)
+    # Ticket #462 - allow the original encoding through, usually it's UTF-8 or similar
+    stripped_text_from_html = json.dumps(s, indent=4, ensure_ascii=False)
 
     return stripped_text_from_html
 
@@ -151,4 +169,36 @@ def strip_ignore_text(content, wordlist, mode="content"):
     if mode == "line numbers":
         return ignored_line_numbers
 
-    return "\n".encode('utf8').join(output)
+    return "\n".encode('utf8').join(output)
+
+
+def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
+    """Converts html string to a string with just the text. If ignoring
+    rendering anchor tag content is enable, anchor tag content are also
+    included in the text
+
+    :param html_content: string with html content
+    :param render_anchor_tag_content: boolean flag indicating whether to extract
+    hyperlinks (the anchor tag content) together with text. This refers to the
+    'href' inside 'a' tags.
+    Anchor tag content is rendered in the following manner:
+    '[ text ](anchor tag content)'
+    :return: extracted text from the HTML
+    """
+    #  if anchor tag content flag is set to True define a config for
+    #  extracting this content
+    if render_anchor_tag_content:
+
+        parser_config = ParserConfig(
+            annotation_rules={"a": ["hyperlink"]}, display_links=True
+        )
+
+    # otherwise set config to None
+    else:
+        parser_config = None
+
+    # get text and annotations via inscriptis
+    text_content = get_text(html_content, config=parser_config)
+
+    return text_content
+

changedetectionio/notification.py

@@ -26,13 +26,6 @@ default_notification_title = 'ChangeDetection.io Notification - {watch_url}'
 
 def process_notification(n_object, datastore):
 
-    apobj = apprise.Apprise(debug=True)
-
-    for url in n_object['notification_urls']:
-        url = url.strip()
-        print (">> Process Notification: AppRise notifying {}".format(url))
-        apobj.add(url)
-
     # Get the notification body from datastore
     n_body = n_object.get('notification_body', default_notification_body)
     n_title = n_object.get('notification_title', default_notification_title)
@@ -54,19 +47,42 @@ def process_notification(n_object, datastore):
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
     # raise it as an exception
+    apobjs=[]
+    for url in n_object['notification_urls']:
 
-    with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
-        apobj.notify(
-        body=n_body,
-        title=n_title,
-        body_format=n_format)
+        apobj = apprise.Apprise(debug=True)
+        url = url.strip()
+        if len(url):
+            print(">> Process Notification: AppRise notifying {}".format(url))
+            with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
+                # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
+                # Because different notifications may require different pre-processing, run each sequentially :(
+                # 2000 bytes minus -
+                #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
+                #     Length of URL - Incase they specify a longer custom avatar_url
 
-        # Returns empty string if nothing found, multi-line string otherwise
-        log_value = logs.getvalue()
-        if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
-            raise Exception(log_value)
+                # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
+                k = '?' if not '?' in url else '&'
+                if not 'avatar_url' in url:
+                    url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
 
+                body = n_body[0:1800-len(n_title)-len(url)] if 'discord://' in url else n_body
+                apobj.add(url)
 
+                apobj.notify(
+                title=n_title,
+                body=body,
+                body_format=n_format)
+
+                apobj.clear()
+
+                # Incase it needs to exist in memory for a while after to process(?)
+                apobjs.append(apobj)
+
+                # Returns empty string if nothing found, multi-line string otherwise
+                log_value = logs.getvalue()
+                if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
+                    raise Exception(log_value)
 
 # Notification title + body content parameters get created here.
 def create_notification_parameters(n_object, datastore):

changedetectionio/static/images/notviewed.svg

@@ -1 +0,0 @@
-<svg id="svg2" xmlns="http://www.w3.org/2000/svg" width="9.5" height="15" viewBox="0 0 9.5 15"><path id="path3740" d="M2.2,0A2.41,2.41,0,0,0,0,1.5V2.8H2.2V0Z" transform="translate(0 0)" style="fill:#0078e7"/><path id="rect3728" d="M.3,1.7l-.2,1H2.2V.2A2.76,2.76,0,0,0,.3,1.7Z" transform="translate(0 0)" style="fill:#0078e7"/><path id="path3655" d="M9.5,2.6h0L2.3,0,2,.2,9.2,2.8v.1" transform="translate(0 0)" style="fill:#0078e7"/><path id="path3645" d="M9.2,2.8V13.3l.2-.3V2.5" transform="translate(0 0)" style="fill:#0078e7"/><path id="rect3517" d="M2,.2,9.2,2.8V13.4L2,10.8Z" transform="translate(0 0)" style="fill:#0078e7"/><path id="path3657" d="M2.1.2,9.2,2.8" transform="translate(0 0)" style="fill:#0078e7;stroke:#0078e7;stroke-miterlimit:4.660399913787842;stroke-width:0.10000000149011612px"/><path id="path3684" d="M.5,9.6.6,2.4.4,2.3S1.2,1.1,2,1L8.8,3.4v.1" transform="translate(0 0)" style="fill:#fff;fill-rule:evenodd"/><path id="path3679" d="M8.9,3.4,8.7,13,7.3,14.3.5,11.9V9.5" transform="translate(0 0)" style="fill:#fff;fill-rule:evenodd"/><path id="path3669" d="M7.5,4.2h0L.3,1.6,0,1.9,7.2,4.5v.1" transform="translate(0 0)" style="fill:#0078e7"/><path id="path3671" d="M7.2,4.5V15l.2-.3V4.2" transform="translate(0 0)" style="fill:#0078e7"/><path id="path3673" d="M0,1.9,7.2,4.5V15L0,12.4Z" transform="translate(0 0)" style="fill:#0078e7"/><path id="path3675" d="M.1,1.9,7.2,4.5" transform="translate(0 0)" style="fill:#0078e7;stroke:#0078e7;stroke-miterlimit:4.660399913787842;stroke-width:0.10000000149011612px"/></svg>

changedetectionio/static/images/pause.svg

@@ -1 +1,84 @@
-<svg id="Capa_1" data-name="Capa 1" xmlns="http://www.w3.org/2000/svg" width="15.03" height="15.03" viewBox="0 0 15.03 15.03"><path id="path2" d="M7.5,0A7.56,7.56,0,0,0,.6,4.6a7.37,7.37,0,0,0,1.5,8.1A7.52,7.52,0,0,0,10,14.6,7.53,7.53,0,0,0,10.9.8,7.73,7.73,0,0,0,7.5,0ZM6.6,10.3c0,.5-.6.5-.9.5s-.8.1-.9-.4V4.7c.1-.5.7-.3,1.1-.4a.53.53,0,0,1,.7.6Zm3.7,0c0,.5-.6.5-.9.5s-.8.1-.9-.4V4.7c.1-.5.7-.3,1.1-.4a.53.53,0,0,1,.7.6Z" transform="translate(0.01 0)" style="fill:#0078e7"/></svg>
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   version="1.1"
+   id="Capa_1"
+   x="0px"
+   y="0px"
+   viewBox="0 0 15 14.998326"
+   xml:space="preserve"
+   width="15"
+   height="14.998326"><metadata
+   id="metadata39"><rdf:RDF><cc:Work
+       rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
+         rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
+   id="defs37" />
+<path
+   id="path2"
+   style="fill:#1b98f8;fill-opacity:1;stroke-width:0.0292893"
+   d="M 7.4975161,6.5052867e-4 C 4.549072,-0.04028702 1.7055675,1.8548221 0.58868606,4.5801341 -0.57739762,7.2574642 0.02596981,10.583326 2.069916,12.671949 4.0364753,14.788409 7.2763651,15.56067 9.989207,14.57284 12.801145,13.617602 14.87442,10.855325 14.985833,7.8845744 15.172496,4.9966544 13.49856,2.1100704 10.911002,0.8209349 9.8598067,0.28073592 8.6791261,-0.00114855 7.4975161,6.5052867e-4 Z M 6.5602569,10.251923 c -0.00509,0.507593 -0.5693885,0.488472 -0.9352002,0.468629 -0.3399386,0.0018 -0.8402048,0.07132 -0.9297965,-0.374189 -0.015842,-1.8973128 -0.015872,-3.7979649 0,-5.6952784 0.1334405,-0.5224315 0.7416869,-0.3424086 1.1377562,-0.374189 0.3969969,-0.084515 0.8245634,0.1963256 0.7272405,0.6382917 0,1.7789118 0,3.5578239 0,5.3367357 z m 3.7490371,0 c -0.0051,0.507593 -0.5693888,0.488472 -0.9352005,0.468629 -0.3399386,0.0018 -0.8402048,0.07132 -0.9297965,-0.374189 -0.015842,-1.8973128 -0.015872,-3.7979649 0,-5.6952784 0.1334405,-0.5224315 0.7416869,-0.3424086 1.1377562,-0.374189 0.3969969,-0.084515 0.8245638,0.1963256 0.7272408,0.6382917 0,1.7789118 0,3.5578239 0,5.3367357 z" />
+<g
+   id="g4"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g6"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g8"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g10"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g12"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g14"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g16"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g18"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g20"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g22"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g24"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g26"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g28"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g30"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g32"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+</svg>

changedetectionio/static/images/play.svg

@@ -1 +0,0 @@
-<svg id="Capa_1" data-name="Capa 1" xmlns="http://www.w3.org/2000/svg" width="15" height="15" viewBox="0 0 15 15"><path id="path2" d="M7.5,0A7.62,7.62,0,0,0,0,7.5,7.55,7.55,0,0,0,7.5,15,7.55,7.55,0,0,0,15,7.5,7.6,7.6,0,0,0,10.9.8,9.42,9.42,0,0,0,7.5,0Z" transform="translate(0 0)" style="fill:#0078e7"/><polygon points="11.4 8 5.8 4.8 5.8 11.3 11.4 8" style="fill:#fff"/></svg>

changedetectionio/static/images/search.svg

@@ -1 +0,0 @@
-<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32"><circle cx="16" cy="16" r="16" style="fill:#0078e7"/><path d="M24,26.85l-4.93-5a8.53,8.53,0,0,1-4.71,1.41,8.63,8.63,0,1,1,7.32-4.12l5,5c.26.26,0,.9-.49,1.44l-.74.74C24.86,26.88,24.23,27.11,24,26.85Zm-3.9-12.23a5.75,5.75,0,1,0-5.74,5.79A5.76,5.76,0,0,0,20.07,14.62Z" style="fill:#fff"/></svg>

changedetectionio/static/images/sortable.svg

@@ -1 +0,0 @@
-<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" width="10.93" height="14.99" viewBox="0 0 10.93 14.99"><path d="M5.5,1,9.6,6.1H1.3Zm0,13L1.3,8.9H9.5Z" transform="translate(0.02 -0.01)" style="fill:#0078e7;stroke:#0078e7;stroke-miterlimit:10;stroke-width:1.25px"/></svg>

changedetectionio/static/images/viewed.svg

@@ -1 +0,0 @@
-<svg id="svg2" xmlns="http://www.w3.org/2000/svg" width="16.33" height="11.64" viewBox="0 0 16.33 11.64"><path id="path2416" d="M14.2,6.5l1.4,3.1-6.2.8s0,.2-.4.1a.65.65,0,0,1-.6-.6L1.8,11.1.6,4.2H.9v.1H.8L2,10.9,8.4,9.8V10h.4s0,.4.5.4l.1-.1,6-.8-1.2-3Z" transform="translate(-0.01 -0.04)" style="fill:#0078e7;stroke:#007ec0;fill-rule:evenodd"/><path id="path2400" d="M1,4.3H.9L2,10.9,8.4,9.8s-1.7-.9-6.1,1L1,4.3Z" transform="translate(-0.01 -0.04)" style="fill:#1187c3;fill-rule:evenodd"/><path id="path2402" d="M1,4.1V3.6h.1V3.2l.2-.3h.1V2.5L2.8,8.9l-.1.3v.2l-.1.1-.2.1-.1,1.1L1,4.1Z" transform="translate(-0.01 -0.04)" style="fill:#fff;stroke:#0078e7;stroke-width:0.5px;fill-rule:evenodd"/><path id="path2388" d="M2.3,10.8l.1-1.2.3-.3V9.2l.2-.3s5.5-.2,5.9.8l-.4.1s-1.7-.9-6.1,1Z" transform="translate(-0.01 -0.04)" style="fill:#fff;stroke:#0078e7;stroke-width:0.5px;fill-rule:evenodd"/><path id="path2394" d="M2.2,2.5H1.5L2.9,8.9s5.3-.2,5.9.8c0,0-.1-1.1-5.4-1.6L2.2,2.5Z" transform="translate(-0.01 -0.04)" style="fill:#fff;stroke:#0078e7;stroke-width:0.5px;fill-rule:evenodd"/><path id="path2398" d="M2,1.3,3.4,8s5,.5,5.4,1.7l-2-6.3c0-.1,0-1.1-4.8-2.1Z" transform="translate(-0.01 -0.04)" style="fill:#fff;stroke:#0078e7;stroke-width:0.5px;fill-rule:evenodd"/><path id="path2403" d="M6.9,3.3l5-2.9,2.5,5.9L8.8,9.7,6.9,3.3Z" transform="translate(-0.01 -0.04)" style="fill:#fff;stroke:#0078e7;stroke-width:0.5px;fill-rule:evenodd"/><path id="path2411" d="M8.5,10V9.8l.3-.1s.2.6.6.6v.1a.49.49,0,0,1-.5-.5l-.4.1Z" transform="translate(-0.01 -0.04)" style="fill:#1187c3;stroke:#007ec0;stroke-width:0.5px;fill-rule:evenodd"/><path id="path2415" d="M8.8,9.7s.2.6.6.5l6-.8-.3-.6h-.3c.1,0-5,.2-6,.9Z" transform="translate(-0.01 -0.04)" style="fill:#fff;stroke:#0078e7;stroke-width:0.5px;fill-rule:evenodd"/><path id="path2404" d="M15.1,8.9l-1-2.3L8.8,9.7v.1s.5-.6,6-.9V9Z" transform="translate(-0.01 -0.04)" style="fill:#fff;stroke:#0078e7;stroke-width:0.5px;fill-rule:evenodd"/></svg>

changedetectionio/static/js/notifications.js

@@ -0,0 +1,53 @@
+$(document).ready(function() {
+
+  $('#add-email-helper').click(function (e) {
+    e.preventDefault();
+    email = prompt("Destination email");
+    if(email) {
+      var n = $("#notification_urls");
+      var p=email_notification_prefix;
+      $(n).val( $.trim( $(n).val() )+"\n"+email_notification_prefix+email );
+    }
+  });
+
+  $('#send-test-notification').click(function (e) {
+    e.preventDefault();
+
+    // this can be global
+    var csrftoken = $('input[name=csrf_token]').val();
+    $.ajaxSetup({
+        beforeSend: function(xhr, settings) {
+            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken)
+            }
+        }
+    })
+
+    data = {
+        window_url : window.location.href,
+        notification_urls : $('#notification_urls').val(),
+        notification_title : $('#notification_title').val(),
+        notification_body : $('#notification_body').val(),
+        notification_format : $('#notification_format').val(),
+    }
+    for (key in data) {
+      if (!data[key].length) {
+        alert(key+" is empty, cannot send test.")
+        return;
+      }
+    }
+
+    $.ajax({
+      type: "POST",
+      url: notification_base_url,
+      data : data
+    }).done(function(data){
+      console.log(data);
+      alert('Sent');
+    }).fail(function(data){
+      console.log(data);
+      alert('Error: '+data.responseJSON.error);
+    })
+  });
+});
+

changedetectionio/static/js/tabs.js

@@ -1,13 +1,15 @@
 // Rewrite this is a plugin.. is all this JS really 'worth it?'
 
-// display correct label and messages for minutes or seconds 
-document.addEventListener("DOMContentLoaded", function(event) {
-	use_seconds_change();
-});
+
+if(!window.location.hash) {
+  var tab=document.querySelectorAll("#default-tab a");
+  tab[0].click();
+}
+
 window.addEventListener('hashchange', function() {
   var tabs = document.getElementsByClassName('active');
   while (tabs[0]) {
-    tabs[0].classList.remove('active');
+    tabs[0].classList.remove('active')
   }
   set_active_tab();
 }, false);
@@ -24,7 +26,6 @@ if (!has_errors.length) {
   focus_error_tab();
 }
 
-
 function set_active_tab() {
   var tab=document.querySelectorAll("a[href='"+location.hash+"']");
   if (tab.length) {
@@ -40,7 +41,7 @@ function focus_error_tab() {
     var tabs = document.querySelectorAll('.tabs li a'),i;
     for (i = 0; i < tabs.length; ++i) {
       var tab_name=tabs[i].hash.replace('#','');
-      var pane_errors=document.querySelectorAll('#'+tab_name+' .error');
+      var pane_errors=document.querySelectorAll('#'+tab_name+' .error')
       if (pane_errors.length) {
         document.location.hash = '#'+tab_name;
         return true;
@@ -48,3 +49,7 @@ function focus_error_tab() {
     }
     return false;
 }
+
+
+
+

changedetectionio/static/js/tbltools.js

@@ -1,406 +0,0 @@
-// table tools
-
-// must be a var for keyChar and keyCode use
-var CONSTANT_ESCAPE_KEY = 27;
-var CONSTANT_S_KEY = 83;
-var CONSTANT_s_KEY = 115;
-
-// globals
-var loading;
-var sort_column; // new window or tab is always last_changed
-var sort_order;  // new window or tab is always descending
-
-// restore scroll position on submit/reload 
-document.addEventListener("DOMContentLoaded", function(event) {
-	load_functions();
-	var scrollpos = sessionStorage.getItem('scrollpos');
-	if (scrollpos) window.scrollTo(0, scrollpos);
-});
-
-// mobile scroll position retention 
-if (/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent)) {
-	document.addEventListener("visibilitychange", function() {
-		storeScrollAndSearch();
-	});
-} else {
-	// non-mobile scroll position retention 
-	window.onbeforeunload = function(e) {
-		storeScrollAndSearch();
-	};
-}
-function storeScrollAndSearch() {
-	sessionStorage.setItem('scrollpos', window.pageYOffset);
-	sessionStorage.setItem('searchtxt', document.getElementById("txtInput").value);
-}
-
-// mobile positioning of checkbox-controls grid popup
-document.addEventListener("touchstart", touchStartHandler, false);
-var touchXY = {};
-function touchStartHandler(event) {
-	var touches = event.changedTouches;
-	touchXY = {
-		clientX : touches[0].clientX,
-		clientY : touches[0].clientY
-	};
-}
-
-// (ctl)-alt-s search hotkey
-document.onkeyup = function(e) {
-	var e = e || window.event; // for IE to cover IEs window event-object
-	if (e.altKey && (e.which == CONSTANT_S_KEY || e.which == CONSTANT_s_KEY)) {
-		document.getElementById("txtInput").focus();
-		return false;
-	}
-}
-
-// new window or tab loading
-function load_functions() {
-	// loading
-	loading = true;
-	// retain checked items
-	checkChange();
-	// retrieve saved sorting
-	getSort();
-	// sort if not default
-	sortTable(sort_column);
-	// search
-	if (isSessionStorageSupported()) {
-		// retrieve search
-		if (sessionStorage.getItem("searchtxt") != null) {
-			document.getElementById("txtInput").value = sessionStorage.getItem("searchtxt");
-			tblSearch(this);
-		}
-	}
-}
-
-// sorting
-function sortTable(n) {
-	var table, rows, switching, i, x, y, shouldSwitch, dir, switchcount = 0,
-		sortimgs, sortableimgs;
-	table = document.getElementById("watch-table");
-	switching = true;
-	//Set the sorting direction, either default 9, 1 or saved
-	if (loading) {
-		getSort();
-		dir = (sort_order == 0) ? "asc" : "desc";
-		loading = false;
-	} else {
-		dir = "asc";
-	}
-	/*Make a loop that will continue until
-	no switching has been done:*/
-	while (switching) {
-		//start by saying: no switching is done:
-		switching = false;
-		rows = table.rows;
-		/*Loop through all table rows (except the
-		first, which contains table headers):*/
-		for (i = 1; i < (rows.length - 1); i++) {
-			//start by saying there should be no switching:
-			shouldSwitch = false;
-			/*Get the two elements you want to compare,
-			one from current row and one from the next:*/
-			x = rows[i].getElementsByTagName("TD")[n];
-			y = rows[i + 1].getElementsByTagName("TD")[n];
-			x = x.innerHTML.toLowerCase();
-			y = y.innerHTML.toLowerCase();
-			if (!isNaN(x)) { // handle numeric columns
-				x = parseFloat(x);
-				y = parseFloat(y);
-			}
-			if (n == 1) { // handle play/pause column
-				x = rows[i].getElementsByTagName("TD")[n].getElementsByTagName("img")[0].src;
-				y = rows[i + 1].getElementsByTagName("TD")[n].getElementsByTagName("img")[0].src;
-			}
-			/*check if the two rows should switch place,
-			based on the direction, asc or desc:*/
-			if (dir == "asc") {
-				if (x > y) {
-					//if so, mark as a switch and break the loop:
-					shouldSwitch = true;
-					break;
-				}
-			} else if (dir == "desc") {
-				if (x < y) {
-					//if so, mark as a switch and break the loop:
-					shouldSwitch = true;
-					break;
-				}
-			}
-		}
-		if (shouldSwitch) {
-			/*If a switch has been marked, make the switch
-			and mark that a switch has been done:*/
-			rows[i].parentNode.insertBefore(rows[i + 1], rows[i]);
-			switching = true;
-			//Each time a switch is done, increase this count by 1:
-			switchcount++;
-		} else {
-			/*If no switching has been done AND the direction is "asc",
-			set the direction to "desc" and run the while loop again.*/
-			if (switchcount == 0 && dir == "asc") {
-				dir = "desc";
-				switching = true;
-			}
-		}
-	}
-	// hide all asc/desc sort arrows
-	sortimgs = document.querySelectorAll('[id^="sort-"]');
-	for (i = 0; i < sortimgs.length; i++) {
-		sortimgs[i].style.display = "none";
-	}
-	// show current asc/desc sort arrow and set sort_order var
-	if (dir == "asc") {
-		document.getElementById("sort-" + n + "a").style.display = "";
-	} else {
-		document.getElementById("sort-" + n + "d").style.display = "";
-	}
-	// show all sortable indicators
-	sortableimgs = document.querySelectorAll('[id^="sortable-"]');
-	for (i = 0; i < sortableimgs.length; i++) {
-		sortableimgs[i].style.display = "";
-	}
-	// hide sortable indicator from current column
-	document.getElementById("sortable-" + n).style.display = "none";
-	// save sorting
-	sessionStorage.setItem("sort_column", n);
-	sessionStorage.setItem("sort_order", (dir == "asc") ? 0 : 1);
-	// restripe rows
-	restripe();
-}
-
-// check/uncheck all checkboxes
-function checkAll(e) {
-	var elemID = event.srcElement.id;
-	if (!elemID) return;
-	var elem = document.getElementById(elemID);
-	var rect = elem.getBoundingClientRect();
-	var offsetLeft = document.documentElement.scrollLeft + rect.left;
-	var offsetTop;
-	if (/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent)) {
-		offsetTop = touchXY.clientY; // + rect.top;
-	}
-	else {
-		offsetTop = document.documentElement.scrollTop + rect.top;
-	}
-	var i;
-	var checkboxes = document.getElementsByName('check');
-	var checkboxFunctions = document.getElementById('checkbox-functions');
-	if (e.checked) {
-		for (i = 0; i < checkboxes.length; i++) {
-			checkboxes[i].checked = true;
-		}
-		checkboxFunctions.style.display = "";
-		checkboxFunctions.style.left = offsetLeft + 30 + "px";
-		checkboxFunctions.style.top = offsetTop + "px";
-	} else {
-		for (i = 0; i < checkboxes.length; i++) {
-			checkboxes[i].checked = false;
-		}
-		checkboxFunctions.style.display = "none";
-	}
-}
-
-// show/hide checkbox controls grid popup and check/uncheck checkall checkbox if all other checkboxes are checked/unchecked
-function checkChange(e) {
-	var elemID = event.srcElement.id;
-	if (!elemID) return;
-	var elem = document.getElementById(elemID);
-	var rect = elem.getBoundingClientRect();
-	var offsetLeft = document.documentElement.scrollLeft + rect.left;
-	var offsetTop;
-	if (/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent)) {
-		offsetTop = touchXY.clientY; // + rect.top;
-	}
-	else {
-		offsetTop = document.documentElement.scrollTop + rect.top;
-	}
-	var i;
-	var totalCheckbox = document.querySelectorAll('input[name="check"]').length;
-	var totalChecked = document.querySelectorAll('input[name="check"]:checked').length;
-	var checkboxFunctions = document.getElementById('checkbox-functions');
-	if(totalCheckbox == totalChecked) {
-		document.getElementsByName("showhide")[0].checked=true;
-	}
-	else {
-		document.getElementsByName("showhide")[0].checked=false;
-	}
-	if (totalChecked > 0) {
-		checkboxFunctions.style.display = "";
-		checkboxFunctions.style.left = offsetLeft + 30 + "px";
-		if ( offsetTop > ( window.innerHeight - checkboxFunctions.offsetHeight) ) {
-			checkboxFunctions.style.top = (window.innerHeight - checkboxFunctions.offsetHeight) + "px";
-		}
-		else {
-			checkboxFunctions.style.top = offsetTop + "px";
-		}
-	} else {
-		checkboxFunctions.style.display = "none";
-	}
-}
-
-// search watches in Title column
-function tblSearch(evt) {
-	var code = evt.charCode || evt.keyCode;
-	if (code == CONSTANT_ESCAPE_KEY) {
-		document.getElementById("txtInput").value = '';
-	}
-	var input, filter, table, tr, td, i, txtValue;
-	input = document.getElementById("txtInput");
-	filter = input.value.toUpperCase();
-	table = document.getElementById("watch-table");
-	tr = table.getElementsByTagName("tr");
-	for (i = 1; i < tr.length; i++) { // skip header
-		td = tr[i].getElementsByTagName("td")[3]; // col 3 is the hidden title/url column
-		if (td) {
-			txtValue = td.textContent || td.innerText;
-			if (txtValue.toUpperCase().indexOf(filter) > -1) {
-				tr[i].style.display = "";
-			} else {
-				tr[i].style.display = "none";
-			}
-		}
-	}
-	// restripe rows
-	restripe();
-	if (code == CONSTANT_ESCAPE_KEY) {
-		document.getElementById("watch-table-wrapper").focus();
-	}
-}
-
-// restripe after searching or sorting
-function restripe() {
-	var i, visrows = [];
-	var table = document.getElementById("watch-table");
-	var rows = table.getElementsByTagName("tr");
-
-	for (i = 1; i < rows.length; i++) { // skip header
-		if (rows[i].style.display !== "none") {
-			visrows.push(rows[i]);
-		}
-	}
-	for (i = 0; i < visrows.length; i++) {
-		var row = visrows[i];
-		if (i % 2 == 0) {
-			row.classList.remove('pure-table-odd');
-			row.classList.add('pure-table-even');
-		} else {
-			row.classList.remove('pure-table-even');
-			row.classList.add('pure-table-odd');
-		}
-		var cells = row.getElementsByTagName("td");
-		for (var j = 0; j < cells.length; j++) {
-			if (i % 2 == 0) {
-				cells[j].style.background = "#f2f2f2";
-			} else {
-				cells[j].style.background = "#ffffff";
-			}
-		}
-		// uncomment to renumber rows ascending:    var cells = row.getElementsByTagName("td");
-		// uncomment to renumber rows ascending:    cells[0].innerText = i+1;
-	}
-}
-
-// get checked or all uuids
-function getChecked(items) {
-	var i, checkedArr, uuids = '';
-
-	if (items === undefined) {
-		checkedArr = document.querySelectorAll('input[name="check"]:checked');
-	} else {
-		checkedArr = document.querySelectorAll('input[name="check"]');
-	}
-	if (checkedArr.length > 0) {
-		let output = [];
-		for (i = 0; i < checkedArr.length; i++) {
-			output.push(checkedArr[i].parentNode.parentNode.getAttribute("id"));
-		}
-		for (i = 0; i < checkedArr.length; i++) {
-			if (i < checkedArr.length - 1) {
-				uuids += output[i] + ",";
-			} else {
-				uuids += output[i];
-			}
-		}
-	}
-	return uuids;
-}
-
-// process selected watches 
-function processChecked(func, tag) {
-	var uuids, result;
-
-	if (func == 'mark_all_notviewed') {
-		uuids = getChecked('all');
-	} else {
-		uuids = getChecked();
-	}
-	// confirm if deleting
-	if (func == 'delete_selected' && uuids.length > 0) {
-		result = confirm('Deletions cannot be undone.\n\nAre you sure you want to continue?');
-		if (result == false) {
-			return;
-		}
-	}
-	// href locations
-	var currenturl = window.location;
-	var posturl = location.protocol + '//' + location.host + '/api/process-selected';
-	// posting vars
-	const XHR = new XMLHttpRequest(),
-		FD = new FormData();
-	// fill form data
-	FD.append('func', func);
-	FD.append('tag', tag);
-	FD.append('uuids', uuids);
-	// success
-	XHR.addEventListener('load', function(event) {
-		window.location = currenturl;
-	});
-	// error
-	XHR.addEventListener(' error', function(event) {
-		alert('Error posting request.');
-	});
-	// set up request
-	XHR.open('POST', posturl);
-	// send
-	XHR.send(FD);
-}
-
-function clearSearch() {
-	document.getElementById("txtInput").value = '';
-	tblSearch(CONSTANT_ESCAPE_KEY);
-}
-
-function isSessionStorageSupported() {
-	var storage = window.sessionStorage;
-	try {
-		storage.setItem('test', 'test');
-		storage.removeItem('test');
-		return true;
-	} catch (e) {
-		return false;
-	}
-}
-
-function getSort() {
-	if (isSessionStorageSupported()) {
-		// retrieve sort settings if set
-		if (sessionStorage.getItem("sort_column") != null) {
-			sort_column = sessionStorage.getItem("sort_column");
-			sort_order = sessionStorage.getItem("sort_order");
-		} else {
-			sort_column = 7; // last changed
-			sort_order = 1; // desc
-			//alert("Your web browser does not support retaining sorting and page position.");
-		}
-	}
-}
-
-function closeGridDisplay() {
-	document.getElementsByName("showhide")[0].checked = false;
-	var checkboxes = document.getElementsByName('check');
-	for (i = 0; i < checkboxes.length; i++) {
-		checkboxes[i].checked = false;
-	}
-	document.getElementById("checkbox-functions").style.display = "none";
-}

changedetectionio/static/js/watch-overview.js

@@ -0,0 +1,6 @@
+$(function () {
+  // Remove unviewed status when normally clicked
+  $('.diff-link').click(function () {
+    $(this).closest('.unviewed').removeClass('unviewed');
+  });
+});

changedetectionio/static/styles/diff.css

@@ -1,7 +1,8 @@
 #diff-ui {
   background: #fff;
   padding: 2em;
-  margin: 1em;
+  margin-left: 1em;
+  margin-right: 1em;
   border-radius: 5px;
   font-size: 11px; }
   #diff-ui table {
@@ -70,3 +71,8 @@ td#diff-col div {
 /* ignored and triggered? make it obvious error */
 .ignored.triggered {
   background-color: #ff0000; }
+
+.tab-pane-inner#screenshot {
+  text-align: center; }
+  .tab-pane-inner#screenshot img {
+    max-width: 99%; }

changedetectionio/static/styles/diff.scss

@@ -2,7 +2,8 @@
 
     background: #fff;
     padding: 2em;
-    margin: 1em;
+    margin-left: 1em;
+    margin-right: 1em;
     border-radius: 5px;
     font-size: 11px;
 
@@ -85,4 +86,11 @@ td#diff-col div {
 /* ignored and triggered? make it obvious error */
 .ignored.triggered {
   background-color: #ff0000;
+}
+
+.tab-pane-inner#screenshot {
+  text-align: center;
+  img {
+    max-width: 99%;
+  }
 }

changedetectionio/static/styles/styles.css

@@ -31,21 +31,24 @@ a.github-link {
 
 section.content {
   padding-top: 5em;
-  padding-bottom: 5em;
+  padding-bottom: 1em;
   flex-direction: column;
   display: flex;
   align-items: center;
   justify-content: center; }
 
+code {
+  background: #eee; }
+
 /* table related */
 .watch-table {
-  width: 100%; }
+  width: 100%;
+  font-size: 80%; }
   .watch-table tr.unviewed {
     font-weight: bold; }
   .watch-table .error {
     color: #a00; }
   .watch-table td {
-    font-size: 80%;
     white-space: nowrap; }
   .watch-table td.title-col {
     word-break: break-all;
@@ -80,11 +83,11 @@ section.content {
 
 body:after {
   content: "";
-  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%); }
+  background: linear-gradient(130deg, #5ad8f7, #2f50af 41.07%, #9150bf 84.05%); }
 
 body:after, body:before {
   display: block;
-  height: 600px;
+  height: 650px;
   position: absolute;
   top: 0;
   left: 0;
@@ -96,9 +99,6 @@ body::after {
 
 body::before {
   content: "";
-  background-image: url(/static/images/gradient-border.png); }
-
-body:before {
   background-size: cover; }
 
 body:after, body:before {
@@ -182,9 +182,15 @@ body:after, body:before {
 
 #notification-customisation {
   border: 1px solid #ccc;
-  padding: 1rem;
+  padding: 0.5rem;
   border-radius: 5px; }
 
+#notification-error-log {
+  border: 1px solid #ccc;
+  padding: 1rem;
+  border-radius: 5px;
+  overflow-wrap: break-word; }
+
 #token-table.pure-table td, #token-table.pure-table th {
   font-size: 80%; }
 
@@ -199,7 +205,8 @@ body:after, body:before {
   #new-watch-form .label {
     display: none; }
   #new-watch-form legend {
-    color: #fff; }
+    color: #fff;
+    font-weight: bold; }
 
 #diff-col {
   padding-left: 40px; }
@@ -243,7 +250,7 @@ footer {
 .sticky-tab {
   position: absolute;
   top: 60px;
-  font-size: 8px;
+  font-size: 65%;
   background: #fff;
   padding: 10px; }
   .sticky-tab#left-sticky {
@@ -278,6 +285,11 @@ footer {
     padding-bottom: 1em; }
     .pure-form .pure-control-group div, .pure-form .pure-group div, .pure-form .pure-controls div {
       margin: 0px; }
+    .pure-form .pure-control-group .checkbox > *, .pure-form .pure-group .checkbox > *, .pure-form .pure-controls .checkbox > * {
+      display: inline;
+      vertical-align: middle; }
+    .pure-form .pure-control-group .checkbox > label, .pure-form .pure-group .checkbox > label, .pure-form .pure-controls .checkbox > label {
+      padding-left: 5px; }
   .pure-form .error input {
     background-color: #ffebeb; }
   .pure-form ul.errors {
@@ -309,14 +321,23 @@ footer {
   #nav-menu {
     overflow-x: scroll; } }
 
-/*
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 800px) {
+  div.sticky-tab#hosted-sticky {
+    top: 60px;
+    left: 0px;
+    right: auto; }
+  section.content {
+    padding-top: 110px; }
+  div.tabs.collapsable ul li {
+    display: block;
+    border-radius: 0px; }
+  input[type='text'] {
+    width: 100%; }
+  /*
 Max width before this PARTICULAR table gets nasty
 This query will take effect for any screen smaller than 760px
 and also iPads specifically.
 */
-@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
-  input[type='text'] {
-    width: 100%; }
   .watch-table {
     /* Force table to not be like tables anymore */
     /* Force table to not be like tables anymore */
@@ -388,14 +409,22 @@ and also iPads specifically.
 .pure-form-stacked > div:first-child {
   display: block; }
 
+.login-form .inner {
+  background: #fff;
+  padding: 20px;
+  border-radius: 5px; }
+
+.tab-pane-inner {
+  padding: 0px; }
+  .tab-pane-inner:not(:target) {
+    display: none; }
+  .tab-pane-inner:target {
+    display: block; }
+
 .edit-form {
-  min-width: 70%; }
-  .edit-form .tab-pane-inner {
-    padding: 0px; }
-    .edit-form .tab-pane-inner:not(:target) {
-      display: none; }
-    .edit-form .tab-pane-inner:target {
-      display: block; }
+  min-width: 70%;
+  /* so it cant overflow */
+  max-width: 95%; }
   .edit-form .box-wrap {
     position: relative; }
   .edit-form .inner {
@@ -404,6 +433,8 @@ and also iPads specifically.
   .edit-form #actions {
     display: block;
     background: #fff; }
+  .edit-form .pure-form-message-inline {
+    padding-left: 0; }
 
 ul {
   padding-left: 1em;

changedetectionio/static/styles/styles.scss

@@ -35,16 +35,21 @@ a.github-link {
 
 section.content {
   padding-top: 5em;
-  padding-bottom: 5em;
+  padding-bottom: 1em;
   flex-direction: column;
   display: flex;
   align-items: center;
   justify-content: center;
 }
 
+code {
+  background: #eee;
+}
+
 /* table related */
 .watch-table {
   width: 100%;
+  font-size: 80%;
 
   tr.unviewed {
     font-weight: bold;
@@ -55,7 +60,6 @@ section.content {
   }
 
   td {
-    font-size: 80%;
     white-space: nowrap;
   }
 
@@ -107,12 +111,12 @@ section.content {
 
 body:after {
   content: "";
-  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)
+  background: linear-gradient(130deg, #5ad8f7, #2f50af 41.07%, #9150bf 84.05%);
 }
 
 body:after, body:before {
   display: block;
-  height: 600px;
+  height: 650px;
   position: absolute;
   top: 0;
   left: 0;
@@ -125,11 +129,8 @@ body::after {
 }
 
 body::before {
+  // background-image set in base.html so it works with reverse proxies etc
   content: "";
-  background-image: url(/static/images/gradient-border.png);
-}
-
-body:before {
   background-size: cover
 }
 
@@ -240,10 +241,16 @@ body:after, body:before {
 
 #notification-customisation {
     border: 1px solid #ccc;
-    padding: 1rem;
+    padding: 0.5rem;
     border-radius: 5px;
 }
 
+#notification-error-log {
+    border: 1px solid #ccc;
+    padding: 1rem;
+    border-radius: 5px;
+    overflow-wrap: break-word;
+}
 
 #token-table {
     &.pure-table td, &.pure-table th {
@@ -265,6 +272,7 @@ body:after, body:before {
   }
   legend {
     color: #fff;
+    font-weight: bold;
   }
 }
 
@@ -320,7 +328,7 @@ footer {
 .sticky-tab {
   position: absolute;
   top: 60px;
-  font-size: 8px;
+  font-size: 65%;
   background: #fff;
   padding: 10px;
   &#left-sticky {
@@ -367,6 +375,15 @@ footer {
         div {
             margin: 0px;
         }
+        .checkbox {
+            > * {
+              display: inline;
+              vertical-align: middle;
+            }
+            > label {
+               padding-left: 5px;
+            }
+        }
     }
   /* The input fields with errors */
   .error {
@@ -420,18 +437,35 @@ footer {
   }
 }
 
+
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 800px) {
+
+  div.sticky-tab#hosted-sticky {
+    top: 60px;
+    left: 0px;
+    right: auto;
+  }
+
+  section.content {
+    padding-top: 110px;
+  }
+
+  // Make the tabs easier to hit, they will be all nice and horizontal
+  div.tabs.collapsable ul li {
+    display: block;
+    border-radius: 0px;
+  }
+
+  input[type='text'] {
+    width: 100%;
+  }
+
 /*
 Max width before this PARTICULAR table gets nasty
 This query will take effect for any screen smaller than 760px
 and also iPads specifically.
 */
-
-@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
-
-  input[type='text'] {
-    width: 100%;
-  }
-    
   .watch-table {
     /* Force table to not be like tables anymore */
     thead, tbody, th, td, tr {
@@ -545,9 +579,16 @@ $form-edge-padding: 20px;
     display: block;
   }
 }
-.edit-form {
-  min-width: 70%;
-  .tab-pane-inner {
+
+.login-form {
+  .inner {
+    background: #fff;;
+    padding: $form-edge-padding;
+    border-radius: 5px;
+  }
+}
+
+.tab-pane-inner {
     &:not(:target) {
         display: none;
     }
@@ -556,7 +597,12 @@ $form-edge-padding: 20px;
     }
     // doesnt need padding because theres another row of buttons/activity
     padding: 0px;
-  }
+}
+
+.edit-form {
+  min-width: 70%;
+  /* so it cant overflow */
+  max-width: 95%;
   .box-wrap {
     position: relative;
   }
@@ -568,6 +614,10 @@ $form-edge-padding: 20px;
     display: block;
     background: #fff;
   }
+
+  .pure-form-message-inline {
+    padding-left: 0;
+  }
 }
 
 ul {

changedetectionio/store.py

@@ -1,15 +1,19 @@
-from os import unlink, path, mkdir
 import json
-import uuid as uuid_builder
-from threading import Lock
-from copy import deepcopy
-
 import logging
-import time
-import threading
 import os
+import threading
+import time
+import uuid as uuid_builder
+from copy import deepcopy
+from os import mkdir, path, unlink
+from threading import Lock
+
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+)
 
-from changedetectionio.notification import default_notification_format, default_notification_body, default_notification_title
 
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
@@ -46,12 +50,15 @@ class ChangeDetectionStore:
                     'extract_title_as_title': False,
                     'fetch_backend': 'html_requests',
                     'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
+                    'global_subtractive_selectors': [],
                     'ignore_whitespace': False,
+                    'render_anchor_tag_content': False,
                     'notification_urls': [], # Apprise URL list
                     # Custom notification content
                     'notification_title': default_notification_title,
                     'notification_body': default_notification_body,
                     'notification_format': default_notification_format,
+                    'real_browser_save_screenshot': True,
                 }
             }
         }
@@ -82,6 +89,7 @@ class ChangeDetectionStore:
             'notification_body': default_notification_body,
             'notification_format': default_notification_format,
             'css_filter': "",
+            'subtractive_selectors': [],
             'trigger_text': [],  # List of text or regex to wait for until a change is detected
             'fetch_backend': None,
             'extract_title_as_title': False
@@ -144,8 +152,8 @@ class ChangeDetectionStore:
             unlink(password_reset_lockfile)
 
         if not 'app_guid' in self.__data:
-            import sys
             import os
+            import sys
             if "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ:
                 self.__data['app_guid'] = "test-" + str(uuid_builder.uuid4())
             else:
@@ -201,7 +209,6 @@ class ChangeDetectionStore:
     @property
     def data(self):
         has_unviewed = False
-        unviewed_count = 0
         for uuid, v in self.__data['watching'].items():
             self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
             if int(v['newest_history_key']) <= int(v['last_viewed']):
@@ -210,7 +217,6 @@ class ChangeDetectionStore:
             else:
                 self.__data['watching'][uuid]['viewed'] = False
                 has_unviewed = True
-                unviewed_count += 1
 
             # #106 - Be sure this is None on empty string, False, None, etc
             # Default var for fetch_backend
@@ -223,7 +229,6 @@ class ChangeDetectionStore:
           self.__data['settings']['application']['base_url'] = env_base_url.strip('" ')
 
         self.__data['has_unviewed'] = has_unviewed
-        self.__data['unviewed_count'] = unviewed_count
 
         return self.__data
 
@@ -378,6 +383,22 @@ class ChangeDetectionStore:
 
         return fname
 
+    def get_screenshot(self, watch_uuid):
+        output_path = "{}/{}".format(self.datastore_path, watch_uuid)
+        fname = "{}/last-screenshot.png".format(output_path)
+        if path.isfile(fname):
+            return fname
+
+        return False
+
+    # Save as PNG, PNG is larger but better for doing visual diff in the future
+    def save_screenshot(self, watch_uuid, screenshot: bytes):
+        output_path = "{}/{}".format(self.datastore_path, watch_uuid)
+        fname = "{}/last-screenshot.png".format(output_path)
+        with open(fname, 'wb') as f:
+            f.write(screenshot)
+            f.close()
+
     def sync_to_json(self):
         logging.info("Saving JSON..")
 
@@ -397,7 +418,7 @@ class ChangeDetectionStore:
                 # system was out of memory, out of RAM etc
                 with open(self.json_store_path+".tmp", 'w') as json_file:
                     json.dump(data, json_file, indent=4)
-                os.rename(self.json_store_path+".tmp", self.json_store_path)
+                os.replace(self.json_store_path+".tmp", self.json_store_path)
             except Exception as e:
                 logging.error("Error writing JSON!! (Main JSON file save was skipped) : %s", str(e))
 
@@ -433,6 +454,7 @@ class ChangeDetectionStore:
                 index.append(self.data['watching'][uuid]['history'][str(id)])
 
         import pathlib
+
         # Only in the sub-directories
         for item in pathlib.Path(self.datastore_path).rglob("*/*txt"):
             if not str(item) in index:

changedetectionio/templates/_common_fields.jinja

@@ -1,7 +1,7 @@
 
 {% from '_helpers.jinja' import render_field %}
 
-{% macro render_common_settings_form(form, current_base_url) %}
+{% macro render_common_settings_form(form, current_base_url, emailprefix) %}
 
                         <div class="pure-control-group">
                             {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
@@ -13,13 +13,19 @@
                             <div class="pure-form-message-inline">
                               <ul>
                                 <li>Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.</li>
-                                <li><code>discord://</code> will silently fail if the total message length is more than 2000 chars.</li>
+                                <li><code>discord://</code> notifications are cut at 2,000 characters in length.</li>
                                 <li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
-                                <li>Go here for <a href="{{url_for('notification_logs')}}">Notification debug logs</a></li>
+                                <li>Go here for <a href="{{url_for('notification_logs')}}">notification debug logs</a></li>
                               </ul>
                             </div>
+                            <br/>
+                            <a id="send-test-notification" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Send test notification</a>
+{% if emailprefix %}
+                            <a id="add-email-helper" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Add email</a>
+
+{% endif %}
                         </div>
-                        <div id="notification-customisation">
+                        <div id="notification-customisation" class="pure-control-group">
                             <div class="pure-control-group">
                                 {{ render_field(form.notification_title, class="m-d") }}
                                 <span class="pure-form-message-inline">Title for all notifications</span>
@@ -34,9 +40,8 @@
                             </div>
                             <div class="pure-controls">
                             <span class="pure-form-message-inline">
-                                These tokens can be used in the notification body and title to
-                                customise the notification text.
-                            </span>
+                                These tokens can be used in the notification body and title to customise the notification text.
+
                                 <table class="pure-table" id="token-table">
                                     <thead>
                                     <tr>
@@ -88,13 +93,10 @@
                                     </tr>
                                     </tbody>
                                 </table>
-                                <span class="pure-form-message-inline">
+                                <br/>
                                 URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
                                 Your <code>BASE_URL</code> var is currently "{{current_base_url}}"
                             </span>
                             </div>
                         </div>
-                        <div class="pure-control-group">
-                            {{ render_field(form.trigger_check) }}
-                        </div>
 {% endmacro %}

changedetectionio/templates/_helpers.jinja

@@ -1,3 +1,30 @@
+{% macro render_field(field) %}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
+
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </div>
+{% endmacro %}
+
+{% macro render_checkbox_field(field) %}
+  <div class="checkbox {% if field.errors %} error {% endif %}">
+  {{ field(**kwargs)|safe }} {{ field.label }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </div>
+{% endmacro %}
+
 {% macro render_field(field) %}
   <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
   <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}

changedetectionio/templates/base.html

@@ -12,7 +12,14 @@
         <link rel="stylesheet" href="{{ m }}?ver=1000">
         {% endfor %}
     {% endif %}
+    <style>
+    body::before {
+        background-image: url({{url_for('static_content', group='images', filename='gradient-border.png')}});
+    }
+    </style>
+    <script type="text/javascript" src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 </head>
+
 <body>
 
 <div class="header">
@@ -35,16 +42,13 @@
         {% if current_user.is_authenticated or not has_password %}
             {% if not current_diff_url %}
             <li class="pure-menu-item">
-                <span class="search-box"><input type="text" id="txtInput" onkeyup="tblSearch(event)" onmouseup="clearSearch(event)" placeholder="Title..." /></span>
-            </li>
-            <li class="pure-menu-item">
-                <a href="{{ url_for('get_backup')}}" class="pure-menu-link">BACKUP</a>
+                <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
             </li>
             <li class="pure-menu-item">
                 <a href="{{ url_for('import_page')}}" class="pure-menu-link">IMPORT</a>
             </li>
             <li class="pure-menu-item">
-                <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
+                <a href="{{ url_for('get_backup')}}" class="pure-menu-link">BACKUP</a>
             </li>
             {% else %}
             <li class="pure-menu-item">

changedetectionio/templates/diff.html

@@ -1,7 +1,6 @@
 {% extends 'base.html' %}
 
 {% block content %}
-
 <div id="settings">
     <h1>Differences</h1>
     <form class="pure-form " action="" method="GET">
@@ -35,21 +34,45 @@
 <div id="diff-jump">
     <a onclick="next_diff();">Jump</a>
 </div>
+
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<div class="tabs">
+    <ul>
+        <li class="tab" id="default-tab"><a href="#text">Text</a></li>
+{% if screenshot %}
+        <li class="tab"><a href="#screenshot">Current screenshot</a></li>
+{% endif %}
+    </ul>
+</div>
+
 <div id="diff-ui">
-    <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.</div>
-    <table>
-        <tbody>
-        <tr>
-            <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
-            <td id="a" style="display: none;">{{previous}}</td>
-            <td id="b" style="display: none;">{{newest}}</td>
-            <td id="diff-col">
-                <span id="result"></span>
-            </td>
-        </tr>
-        </tbody>
-    </table>
-    Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff">github.com/kpdecker/jsdiff</a>
+     <div class="tab-pane-inner" id="text">
+         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.
+         </div>
+         <table>
+             <tbody>
+             <tr>
+                 <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
+                 <td id="a" style="display: none;">{{previous}}</td>
+                 <td id="b" style="display: none;">{{newest}}</td>
+                 <td id="diff-col">
+                     <span id="result"></span>
+                 </td>
+             </tr>
+             </tbody>
+         </table>
+         Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff">github.com/kpdecker/jsdiff</a>
+     </div>
+
+{% if screenshot %}
+     <div class="tab-pane-inner" id="screenshot">
+         <p>
+         <i>For now, only the most recent screenshot is saved and displayed.</i>
+             </p>
+
+        <img src="{{url_for('static_content', group='screenshot', filename=uuid)}}">
+     </div>
+{% endif %}
 
 </div>
 

changedetectionio/templates/edit.html

@@ -1,13 +1,19 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.jinja' import render_field %}
-{% from '_helpers.jinja' import render_button %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script>
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+{% if emailprefix %}
+    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
+{% endif %}
+</script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 
 <div class="edit-form monospaced-textarea">
 
-    <div class="tabs">
+    <div class="tabs collapsable">
         <ul>
             <li class="tab" id="default-tab"><a href="#general">General</a></li>
             <li class="tab"><a href="#request">Request</a></li>
@@ -19,6 +25,7 @@
     <div class="box-wrap inner">
         <form class="pure-form pure-form-stacked"
               action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next') ) }}" method="POST">
+             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
 
             <div class="tab-pane-inner" id="general">
                 <fieldset>
@@ -44,7 +51,7 @@
                         {% endif %}
                     </div>
                     <div class="pure-control-group">
-                        {{ render_field(form.extract_title_as_title) }}
+                        {{ render_checkbox_field(form.extract_title_as_title) }}
                     </div>
                 </fieldset>
             </div>
@@ -58,31 +65,40 @@
                         </span>
                     </div>
 
+                <hr/>
                 <fieldset class="pure-group">
-                                    <div class="pure-control-group">
-                    {{ render_field(form.method) }}
-                </div>
-                    <strong>Note: <i>Request Headers and Body settings are ONLY used by Basic fast Plaintext/HTTP Client fetch method.</i></strong>
-                    {{ render_field(form.headers, rows=5, placeholder="Example
+
+                    <span class="pure-form-message-inline">
+                        <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
+                    </span>
+                    <div class="pure-control-group">
+                        {{ render_field(form.method) }}
+                    </div>
+                    <div class="pure-control-group">
+{{ render_field(form.headers, rows=5, placeholder="Example
 Cookie: foobar
 User-Agent: wonderbra 1.0") }}
-                </fieldset>
-                <div class="pure-control-group">
-                    {{ render_field(form.body, rows=5, placeholder="Example
+                    </div>
+                    <div class="pure-control-group">
+                                        {{ render_field(form.body, rows=5, placeholder="Example
 {
    \"name\":\"John\",
    \"age\":30,
    \"car\":null
 }") }}
-                </div>
-
+                    </div>
+                    <div>
+                        {{ render_checkbox_field(form.ignore_status_codes) }}
+                    </div>
+                </fieldset>
+                <br/>
             </div>
 
             <div class="tab-pane-inner" id="notifications">
                 <strong>Note: <i>These settings override the global settings for this watch.</i></strong>
                 <fieldset>
                     <div class="field-group">
-                        {{ render_common_settings_form(form, current_base_url) }}
+                        {{ render_common_settings_form(form, current_base_url, emailprefix) }}
                     </div>
                 </fieldset>
             </div>
@@ -107,16 +123,27 @@ User-Agent: wonderbra 1.0") }}
                         <span class="pure-form-message-inline">
                     <ul>
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
-                        <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <b>"json:"</b>, <a
+                        <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <code>"json:"</code>, use <code>json:$</code> to force re-formatting if required,  <a
                                 href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
-                        <li>XPATH - Limit text to this XPath rule, simply start with a forward-slash, example  <b>//*[contains(@class, 'sametext')]</b>, <a
+                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash, example  <code>//*[contains(@class, 'sametext')]</code>, <a
                                 href="http://xpather.com/" target="new">test your XPath here</a></li>
                     </ul>
                     Please be sure that you thoroughly understand how to write CSS or JSONPath, XPath selector rules before filing an issue on GitHub! <a
                                 href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
                 </span>
                     </div>
-
+                    <fieldset class="pure-group">
+                      {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
+footer
+nav
+.stockticker") }}
+                      <span class="pure-form-message-inline">
+                        <ul>
+                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
+                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
+                        </ul>
+                      </span>
+                    </fieldset>
                 </fieldset>
                 <fieldset class="pure-group">
                     {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
@@ -125,7 +152,7 @@ User-Agent: wonderbra 1.0") }}
                     <span class="pure-form-message-inline">
                         <ul>
                             <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
-                            <li>Regular Expression support, wrap the line in forward slash <b>/regex/</b></li>
+                            <li>Regular Expression support, wrap the line in forward slash <code>/regex/</code></li>
                             <li>Changing this will affect the comparison checksum which may trigger an alert</li>
                             <li>Use the preview/show current tab to see ignores</li>
                         </ul>
@@ -141,8 +168,8 @@ User-Agent: wonderbra 1.0") }}
                     <ul>
                         <li>Text to wait for before triggering a change/notification, all text and regex are tested <i>case-insensitive</i>.</li>
                         <li>Trigger text is processed from the result-text that comes out of any CSS/JSON Filters for this watch</li>
-                        <li>Each line is process separately (think of each line as "OR")</li>
-                        <li>Note: Wrap in forward slash / to use regex  example: <span style="font-family: monospace; background: #eee">/foo\d/</span></li>
+                        <li>Each line is processed separately (think of each line as "OR")</li>
+                        <li>Note: Wrap in forward slash / to use regex  example: <code>/foo\d/</code></li>
                     </ul>
                         </span>
                     </div>

changedetectionio/templates/import.html

@@ -4,6 +4,7 @@
 <div class="edit-form">
      <div class="inner">
         <form class="pure-form pure-form-aligned" action="{{url_for('import_page')}}" method="POST">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
             <fieldset class="pure-group">
               <legend>
                 Enter one URL per line, and optionally add tags for each URL after a space, delineated by comma (,):

changedetectionio/templates/login.html

@@ -1,15 +1,15 @@
 {% extends 'base.html' %}
 
 {% block content %}
-<div class="edit-form">
-
+<div class="login-form">
  <div class="inner">
     <form class="pure-form pure-form-stacked" action="{{url_for('login')}}" method="POST">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <div class="pure-control-group">
                 <label for="password">Password</label>
                 <input type="password" id="password" required="" name="password" value=""
-                       size="15"/>
+                       size="15" autofocus />
                 <input type="hidden" id="email" name="email" value="defaultuser@changedetection.io" />
             </div>
             <div class="pure-control-group">

changedetectionio/templates/notification-log.html

@@ -5,7 +5,7 @@
      <div class="inner">
 
          <h4 style="margin-top: 0px;">The following issues were detected when sending notifications</h4>
-                <div id="notification-customisation">
+                <div id="notification-error-log">
                 <ul style="font-size: 80%; margin:0px; padding: 0 0 0 7px">
                 {% for log in logs|reverse %}
                     <li>{{log}}</li>

changedetectionio/templates/preview.html

@@ -6,18 +6,40 @@
     <h1>Current - {{watch.last_checked|format_timestamp_timeago}}</h1>
 </div>
 
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<div class="tabs">
+    <ul>
+        <li class="tab" id="default-tab"><a href="#text">Text</a></li>
+{% if screenshot %}
+        <li class="tab"><a href="#screenshot">Current screenshot</a></li>
+{% endif %}
+    </ul>
+</div>
+
 <div id="diff-ui">
-    <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
-    <table>
-        <tbody>
-        <tr>
-            <td id="diff-col">
+    <div class="tab-pane-inner" id="text">
+        <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
+        <table>
+            <tbody>
+            <tr>
+                <td id="diff-col">
                     {% for row in content %}
                     <div class="{{row.classes}}">{{row.line}}</div>
                     {% endfor %}
-            </td>
-        </tr>
-        </tbody>
-    </table>
+                </td>
+            </tr>
+            </tbody>
+        </table>
+    </div>
+
+{% if screenshot %}
+     <div class="tab-pane-inner" id="screenshot">
+         <p>
+         <i>For now, only the most recent screenshot is saved and displayed.</i>
+             </p>
+
+        <img src="{{url_for('static_content', group='screenshot', filename=uuid)}}">
+     </div>
+{% endif %}
 </div>
 {% endblock %}

changedetectionio/templates/scrub.html

@@ -4,6 +4,7 @@
 <div class="edit-form">
     <div class="box-wrap inner">
     <form class="pure-form pure-form-stacked" action="{{url_for('scrub_page')}}" method="POST">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <div class="pure-control-group">
                 This will remove all version snapshots/data, but keep your list of URLs. <br/>

changedetectionio/templates/settings.html

@@ -1,14 +1,20 @@
 {% extends 'base.html' %}
 
 {% block content %}
-{% from '_helpers.jinja' import render_field %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
-
+<script>
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+{% if emailprefix %}
+    const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
+{% endif %}
+</script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='settings.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 
 <div class="edit-form">
-    <div class="tabs">
+    <div class="tabs collapsable">
         <ul>
             <li class="tab" id="default-tab"><a href="#general">General</a></li>
             <li class="tab"><a href="#notifications">Notifications</a></li>
@@ -18,6 +24,7 @@
     </div>
     <div class="box-wrap inner">
         <form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
             <div class="tab-pane-inner" id="general">
                 <fieldset>
                     <div class="pure-control-group">
@@ -27,8 +34,7 @@
                     <div class="pure-control-group">
                         {% if not hide_remove_pass %}
                             {% if current_user.is_authenticated %}
-                            <a href="{{url_for('settings_page', removepassword='yes')}}"
-                               class="pure-button pure-button-primary">Remove password</a>
+                                {{ render_button(form.removepassword_button) }}
                             {% else %}
                             {{ render_field(form.password) }}
                             <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
@@ -47,20 +53,24 @@
                     </div>
 
                     <div class="pure-control-group">
-                        {{ render_field(form.extract_title_as_title) }}
+                        {{ render_checkbox_field(form.extract_title_as_title) }}
                         <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
                     </div>
+
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.real_browser_save_screenshot) }}
+                        <span class="pure-form-message-inline">When using a Chrome browser, a screenshot from the last check will be available on the Diff page</span>
+                    </div>
+
                 </fieldset>
             </div>
 
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div class="field-group">
-                        {{ render_common_settings_form(form, current_base_url) }}
+                        {{ render_common_settings_form(form, current_base_url, emailprefix) }}
                     </div>
                 </fieldset>
-                <a href="{{url_for('notification_logs')}}">Notification debug logs</a>
-
             </div>
 
             <div class="tab-pane-inner" id="fetching">
@@ -77,13 +87,30 @@
             <div class="tab-pane-inner" id="filters">
 
                     <fieldset class="pure-group">
-                    {{ render_field(form.ignore_whitespace) }}
+                    {{ render_checkbox_field(form.ignore_whitespace) }}
                     <span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br/>
-                    <i>Note:</i> Changing this will change the status of your existing watches, possibily trigger alerts etc.
+                    <i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc.
                     </span>
                     </fieldset>
-
-
+                <fieldset class="pure-group">
+                    {{ render_checkbox_field(form.render_anchor_tag_content) }}
+                    <span class="pure-form-message-inline">Render anchor tag content, default disabled, when enabled renders links as <code>(link text)[https://somesite.com]</code>
+                        <br/>
+                    <i>Note:</i> Changing this could affect the content of your existing watches, possibly trigger alerts etc.
+                    </span>
+                    </fieldset>
+                    <fieldset class="pure-group">
+                      {{ render_field(form.global_subtractive_selectors, rows=5, placeholder="header
+footer
+nav
+.stockticker") }}
+                      <span class="pure-form-message-inline">
+                        <ul>
+                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
+                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
+                        </ul>
+                      </span>
+                    </fieldset>
                     <fieldset class="pure-group">
                     {{ render_field(form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
 /some.regex\d{2}/ for case-INsensitive regex
@@ -93,7 +120,7 @@
                         <ul>
                             <li>Note: This is applied globally in addition to the per-watch rules.</li>
                             <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
-                            <li>Regular Expression support, wrap the line in forward slash <b>/regex/</b></li>
+                            <li>Regular Expression support, wrap the line in forward slash <code>/regex/</code></li>
                             <li>Changing this will affect the comparison checksum which may trigger an alert</li>
                             <li>Use the preview/show current tab to see ignores</li>
                         </ul>
@@ -103,11 +130,9 @@
 
             <div id="actions">
                 <div class="pure-control-group">
-                    <button type="submit" class="pure-button pure-button-primary">Save</button>
-                                           <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
-                        <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete
-                            History
-                            Snapshot Data</a>
+                    {{ render_button(form.save_button) }}
+                    <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
+                    <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
                 </div>
             </div>
         </form>

changedetectionio/templates/watch-overview.html

@@ -1,147 +1,106 @@
 {% extends 'base.html' %}
 {% block content %}
 {% from '_helpers.jinja' import render_simple_field %}
-
-<script src="{{url_for('static_content', group='js', filename='tbltools.js')}}"></script>
-
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 <div class="box">
-	<form class="pure-form" action="{{ url_for('api_watch_add') }}" method="POST" id="new-watch-form">
-		<fieldset>
-			<legend>Add a new change detection watch</legend>
-				{{ render_simple_field(form.url, placeholder="https://...", required=true) }}
-				{{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="tag") }}
-			<div id="watch-actions"><button type="submit" class="pure-button pure-button-primary" name="action" value="watch">Watch</button>&nbsp;
-			<span id="add-paused"><label><input type="checkbox" name="add-paused">&nbsp;Add Paused</label></span></div>
-	   </fieldset>
-		<!-- add extra stuff, like do a http POST and send headers -->
-		<!-- user/pass r = requests.get('https://api.github.com/user', auth=('user', 'pass')) -->
-	</form>
-    <div id="watch-table-wrapper" tabindex="-1">
-		<div id="categories">
-			<a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
-			{% for tag in tags %}
-				{% if tag != "" %}
-					<a href="{{url_for('index', tag=tag) }}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
-				{% endif %}
-			{% endfor %}
-		</div>
-			
-		<div id="controls-top">
-			<div id="checkbox-functions" style="display: none;">
-				<ul id="post-list-buttons-top-grid">
-					<li id="grid-item-1">
-						<a href="javascript:processChecked('recheck_selected', '{{ active_tag }}');" class="pure-button button-tag " title="Recheck Selected{%if active_tag%} in &quot;{{active_tag}}&quot;{%endif%}">Recheck&nbsp;</a>
-					</li>
-					<li id="grid-item-2">
-						<a href="javascript:processChecked('mark_selected_notviewed', '{{ active_tag }}');" class="pure-button button-tag " title="Mark Selected as Unviewed{%if active_tag%} in &quot;{{active_tag}}&quot;{%endif%}">Unviewed</a>
-					</li>
-					<li id="grid-item-3">
-						<a href="javascript:processChecked('mark_selected_viewed', '{{ active_tag }}');" class="pure-button button-tag " title="Mark Selected as Viewed{%if active_tag%} in &quot;{{active_tag}}&quot;{%endif%}">Viewed&nbsp;&nbsp;</a>
-					</li>
-					<li id="grid-item-4">
-						<a href="javascript:processChecked('delete_selected', '{{ active_tag }}');" class="pure-button button-tag danger " title="Delete Selected{%if active_tag%} in &quot;{{active_tag}}&quot;{%endif%}">Delete&nbsp;&nbsp;</a>
-					</li>
-					<li id="grid-item-5">
-					   <a href="javascript:closeGridDisplay();" class="pure-button button-tag ">Cancel&nbsp;&nbsp;</a>
-					</li>
-				</ul>
-			</div>
-		</div>
 
-		<div>
-			<table class="pure-table pure-table-striped watch-table" id="watch-table">
-				<thead>
-				<tr id="header">
-					<th class="inline chkbox-header"><input id="chk-all" type="checkbox" name="showhide" onchange="checkAll(this)" title="Check/Uncheck All">&nbsp;&nbsp;#</th>
-					<th class="pause-resume-header" onclick="sortTable(1)">
-						<span class="clickable"
-																				 title="Sort by Pause/Resume"><a
-							href="{{url_for('index', pause='pause-all', tag=active_tag)}}"><img
-							src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause"
-							title="Pause All {%if active_tag%}in &quot;{{active_tag}}&quot; {%endif%}"/></a>&nbsp;<a
-							href="{{url_for('index', pause='resume-all', tag=active_tag)}}"><img
-							src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="Resume"
-							title="Resume All {%if active_tag%}in &quot;{{active_tag}}&quot; {%endif%}"/></a>&nbsp;&nbsp;<span
-							id="sortable-1"><img
-							src="{{url_for('static_content', group='images', filename='sortable.svg')}}"
-							alt="sort"/></span><span class="sortarrow"><span id="sort-1a"
-																			 style="display:none;">&#9650;</span><span
-							id="sort-1d" style="display:none;">&#9660;</span></span></span></th>
-					<th onclick="sortTable(3)"><span class="clickable" title="Sort by Title">Title&nbsp;&nbsp;<span id="sortable-3"><img src="{{url_for('static_content', group='images', filename='sortable.svg')}}" alt="sort" /></span><span class="sortarrow"><span id="sort-3a" style="display:none;">&#9650;</span><span id="sort-3d" style="display:none;">&#9660;</span></span></span></th>
-					<th onclick="sortTable(5)"><span class="clickable" title="Sort by Last Checked">Checked&nbsp;&nbsp;<span id="sortable-5"><img src="{{url_for('static_content', group='images', filename='sortable.svg')}}" alt="sort" /></span><span class="sortarrow"><span id="sort-5a" style="display:none;">&#9650;</span><span id="sort-5d" style="display:none;">&#9660;</span></span></span></th>
-					<th onclick="sortTable(7)"><span class="clickable" title="Sort by Last Changed">Changed&nbsp;&nbsp;<span id="sortable-7" style="display:none;"><img src="{{url_for('static_content', group='images', filename='sortable.svg')}}" alt="sort" /></span><span class="sortarrow"><span id="sort-7a" style="display:none;">&#9650;</span><span id="sort-7d">&#9660;</span></span></span></th>
-					<th>Actions</th>
-				</tr>
-				</thead>
-				<tbody>
-					{% for watch in watches %}
-						<tr id="{{ watch.uuid }}"
-							class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
-																								  
-							{% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
-							{% if watch.last_notification_error is defined and watch.last_notification_error != False %}error{% endif %}
-							{% if watch.paused is defined and watch.paused != False %}paused{% endif %}
-							{% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
-							<td class="inline chkbox"><input id="chk-{{ loop.index }}" type="checkbox" name="check" onchange="checkChange(this);">&nbsp;&nbsp;{{ loop.index }}</td>
-							<td class="inline pause-resume">
-							{% if watch.paused %}
-								<a href="{{url_for('index', pause=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="Resume" title="Resume"/></a>
-							{% else %}
-								<a href="{{url_for('index', pause=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause" title="Pause"/></a>
-							{% endif %}
-							</td>
-							<td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
-								<a class="external inline-hover-img" target="_blank" rel="noopener" href="{{ watch.url }}"></a>
-								{%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
+    <form class="pure-form" action="{{ url_for('api_watch_add') }}" method="POST" id="new-watch-form">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+        <fieldset>
+            <legend>Add a new change detection watch</legend>
+                {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
+                {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="tag") }}
+            <button type="submit" class="pure-button pure-button-primary">Watch</button>
+        </fieldset>
+        <!-- add extra stuff, like do a http POST and send headers -->
+        <!-- user/pass r = requests.get('https://api.github.com/user', auth=('user', 'pass')) -->
+    </form>
+    <div>
+        <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
+        {% for tag in tags %}
+            {% if tag != "" %}
+                <a href="{{url_for('index', tag=tag) }}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
+            {% endif %}
+        {% endfor %}
+    </div>
 
-								{% if watch.last_error is defined and watch.last_error != False %}
-								<div class="fetch-error">{{ watch.last_error }}</div>
-								{% endif %}
-								{% if watch.last_notification_error is defined and watch.last_notification_error != False %}
-								<div class="fetch-error notification-error">{{ watch.last_notification_error }}</div>
-								{% endif %}
-								{% if not active_tag %}
-								<span class="watch-tag-list">{{ watch.tag}}</span>
-								{% endif %}
-							</td>
-							<td class="last-checked">{{watch|format_last_checked_time}}</td>
-							<td class="last-changed">{% if watch.history|length >= 2 and watch.last_changed %}
-								{{watch.last_changed|format_timestamp_timeago}}
-								{% else %}
-								Not yet
-								{% endif %}
-							</td>
-							<td>
-								<a href="{{ url_for('api_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
-								   class="pure-button button-small pure-button-primary">Recheck</a>
-								<a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
-								{% if watch.history|length >= 2 %}
-								<a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
-								{% else %}
-									{% if watch.history|length == 1 %}
-										<a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
-									{% endif %}
-								{% endif %}
-							</td>
-						</tr>
-					{% endfor %}
-				</tbody>
-			</table>
-			<ul id="post-list-buttons">
-				{% if has_unviewed %}
-				<li>
-					<a href="{{url_for('mark_all_viewed', tag=request.args.get('tag')) }}" class="pure-button button-tag ">Mark all viewed</a>
-				</li>
-				{% endif %}
-				<li>
-				   <a href="{{ url_for('api_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
-					all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
-				</li>
-				<li>
-					<a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>
-				</li>
-			</ul>
-		</div>
-	</div>
+    <div id="watch-table-wrapper">
+        <table class="pure-table pure-table-striped watch-table">
+            <thead>
+            <tr>
+                <th>#</th>
+                <th></th>
+                <th></th>
+                <th>Last Checked</th>
+                <th>Last Changed</th>
+                <th></th>
+            </tr>
+            </thead>
+            <tbody>
+
+
+            {% for watch in watches %}
+            <tr id="{{ watch.uuid }}"
+                class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
+                {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
+                {% if watch.last_notification_error is defined and watch.last_notification_error != False %}error{% endif %}
+                {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
+                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
+                <td class="inline">{{ loop.index }}</td>
+                <td class="inline paused-state state-{{watch.paused}}"><a href="{{url_for('index', pause=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause" title="Pause"/></a></td>
+
+                <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
+                    <a class="external" target="_blank" rel="noopener" href="{{ watch.url }}"></a>
+                    {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
+
+                    {% if watch.last_error is defined and watch.last_error != False %}
+                    <div class="fetch-error">{{ watch.last_error }}</div>
+                    {% endif %}
+                    {% if watch.last_notification_error is defined and watch.last_notification_error != False %}
+                    <div class="fetch-error notification-error">{{ watch.last_notification_error }}</div>
+                    {% endif %}
+                    {% if not active_tag %}
+                    <span class="watch-tag-list">{{ watch.tag}}</span>
+                    {% endif %}
+                </td>
+                <td class="last-checked">{{watch|format_last_checked_time}}</td>
+                <td class="last-changed">{% if watch.history|length >= 2 and watch.last_changed %}
+                    {{watch.last_changed|format_timestamp_timeago}}
+                    {% else %}
+                    Not yet
+                    {% endif %}
+                </td>
+                <td>
+                    <a href="{{ url_for('api_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
+                       class="pure-button button-small pure-button-primary">Recheck</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
+                    {% if watch.history|length >= 2 %}
+                    <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary diff-link">Diff</a>
+                    {% else %}
+                        {% if watch.history|length == 1 %}
+                            <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
+                        {% endif %}
+                    {% endif %}
+                </td>
+            </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+        <ul id="post-list-buttons">
+            {% if has_unviewed %}
+            <li>
+                <a href="{{url_for('mark_all_viewed', tag=request.args.get('tag')) }}" class="pure-button button-tag ">Mark all viewed</a>
+            </li>
+            {% endif %}
+            <li>
+               <a href="{{ url_for('api_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
+                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
+            </li>
+            <li>
+                <a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>
+            </li>
+        </ul>
+    </div>
 </div>
 {% endblock %}

changedetectionio/tests/conftest.py

@@ -42,6 +42,9 @@ def app(request):
     cleanup(app_config['datastore_path'])
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
     app = changedetection_app(app_config, datastore)
+
+    # Disable CSRF while running tests
+    app.config['WTF_CSRF_ENABLED'] = False
     app.config['STOP_THREADS'] = True
 
     def teardown():

changedetectionio/tests/test_access_control.py

@@ -4,8 +4,8 @@ from flask import url_for
 def test_check_access_control(app, client):
     # Still doesnt work, but this is closer.
 
-    with app.test_client() as c:
-        # Check we dont have any password protection enabled yet.
+    with app.test_client(use_cookies=True) as c:
+        # Check we don't have any password protection enabled yet.
         res = c.get(url_for("settings_page"))
         assert b"Remove password" not in res.data
 
@@ -46,15 +46,20 @@ def test_check_access_control(app, client):
         assert b"BACKUP" in res.data
         assert b"IMPORT" in res.data
         assert b"LOG OUT" in res.data
+        assert b"minutes_between_check" in res.data
+        assert b"fetch_backend" in res.data
 
-        # Now remove the password so other tests function, @todo this should happen before each test automatically
-        res = c.get(url_for("settings_page", removepassword="yes"),
-              follow_redirects=True)
-        assert b"Password protection removed." in res.data
-
-        res = c.get(url_for("index"))
-        assert b"LOG OUT" not in res.data
-
+        res = c.post(
+            url_for("settings_page"),
+            data={
+                "minutes_between_check": 180,
+                "tag": "",
+                "headers": "",
+                "fetch_backend": "html_webdriver",
+                "removepassword_button": "Remove password"
+            },
+            follow_redirects=True,
+        )
 
 # There was a bug where saving the settings form would submit a blank password
 def test_check_access_control_no_blank_password(app, client):
@@ -71,8 +76,7 @@ def test_check_access_control_no_blank_password(app, client):
             data={"password": "",
                   "minutes_between_check": 180,
                   'fetch_backend': "html_requests"},
-
-        follow_redirects=True
+            follow_redirects=True
         )
 
         assert b"Password protection enabled." not in res.data
@@ -91,7 +95,8 @@ def test_check_access_no_remote_access_to_remove_password(app, client):
         # Enable password check.
         res = c.post(
             url_for("settings_page"),
-            data={"password": "password", "minutes_between_check": 180,
+            data={"password": "password",
+                  "minutes_between_check": 180,
                   'fetch_backend': "html_requests"},
             follow_redirects=True
         )
@@ -99,8 +104,17 @@ def test_check_access_no_remote_access_to_remove_password(app, client):
         assert b"Password protection enabled." in res.data
         assert b"Login" in res.data
 
-        res = c.get(url_for("settings_page", removepassword="yes"),
-              follow_redirects=True)
+        res = c.post(
+            url_for("settings_page"),
+            data={
+                "minutes_between_check": 180,
+                "tag": "",
+                "headers": "",
+                "fetch_backend": "html_webdriver",
+                "removepassword_button": "Remove password"
+            },
+            follow_redirects=True,
+        )
         assert b"Password protection removed." not in res.data
 
         res = c.get(url_for("index"),

changedetectionio/tests/test_api.py

@@ -26,7 +26,8 @@ def test_snapshot_api_detects_change(client, live_server):
     time.sleep(1)
 
     # Add our URL to the import page
-    test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
+    test_url = url_for('test_endpoint', content_type="text/plain",
+                       _external=True)
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},

changedetectionio/tests/test_backend.py

@@ -25,6 +25,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
         data={"urls": url_for('test_endpoint', _external=True)},
         follow_redirects=True
     )
+
     assert b"1 Imported" in res.data
 
     time.sleep(sleep_time_for_fetch_thread)
@@ -69,6 +70,11 @@ def test_check_basic_change_detection_functionality(client, live_server):
     res = client.get(url_for("rss"))
     expected_url = url_for('test_endpoint', _external=True)
     assert b'<rss' in res.data
+
+    # re #16 should have the diff in here too
+    assert b'(into   ) which has this one new line' in res.data
+    assert b'CDATA' in res.data
+    
     assert expected_url.encode('utf-8') in res.data
 
     # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times

changedetectionio/tests/test_element_removal.py

@@ -0,0 +1,168 @@
+#!/usr/bin/python3
+
+import time
+
+from flask import url_for
+
+from ..html_tools import *
+from .util import live_server_setup
+
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def set_original_response():
+    test_return_data = """<html>
+    <header>
+    <h2>Header</h2>
+    </header>
+    <nav>
+    <ul>
+      <li><a href="#">A</a></li>
+      <li><a href="#">B</a></li>
+      <li><a href="#">C</a></li>
+    </ul>
+    </nav>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+    <div id="changetext">Some text that will change</div>
+     </body>
+    <footer>
+    <p>Footer</p>
+    </footer>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_modified_response():
+    test_return_data = """<html>
+    <header>
+    <h2>Header changed</h2>
+    </header>
+    <nav>
+    <ul>
+      <li><a href="#">A changed</a></li>
+      <li><a href="#">B</a></li>
+      <li><a href="#">C</a></li>
+    </ul>
+    </nav>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+    <div id="changetext">Some text that changes</div>
+     </body>
+    <footer>
+    <p>Footer changed</p>
+    </footer>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_element_removal_output():
+    from changedetectionio import fetch_site_status
+    from inscriptis import get_text
+
+    # Check text with sub-parts renders correctly
+    content = """<html>
+    <header>
+    <h2>Header</h2>
+    </header>
+    <nav>
+    <ul>
+      <li><a href="#">A</a></li>
+    </ul>
+    </nav>
+       <body>
+     Some initial text</br>
+     <p>across multiple lines</p>
+     <div id="changetext">Some text that changes</div>
+     </body>
+    <footer>
+    <p>Footer</p>
+    </footer>
+     </html>
+    """
+    html_blob = element_removal(
+        ["header", "footer", "nav", "#changetext"], html_content=content
+    )
+    text = get_text(html_blob)
+    assert (
+        text
+        == """Some initial text
+
+across multiple lines
+"""
+    )
+
+
+def test_element_removal_full(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for("test_endpoint", _external=True)
+    res = client.post(
+        url_for("import_page"), data={"urls": test_url}, follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Goto the edit page, add the filter data
+    # Not sure why \r needs to be added - absent of the #changetext this is not necessary
+    subtractive_selectors_data = "header\r\nfooter\r\nnav\r\n#changetext"
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "subtractive_selectors": subtractive_selectors_data,
+            "url": test_url,
+            "tag": "",
+            "headers": "",
+            "fetch_backend": "html_requests",
+        },
+        follow_redirects=True,
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(subtractive_selectors_data.encode("utf-8")) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # No change yet - first check
+    res = client.get(url_for("index"))
+    assert b"unviewed" not in res.data
+
+    #  Make a change to header/footer/nav
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # There should not be an unviewed change, as changes should be removed
+    res = client.get(url_for("index"))
+    assert b"unviewed" not in res.data

changedetectionio/tests/test_encoding.py

@@ -0,0 +1,87 @@
+#!/usr/bin/python3
+# coding=utf-8
+
+import time
+from flask import url_for
+from .util import live_server_setup
+import pytest
+
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def set_html_response():
+    test_return_data = """
+<html><body><span class="nav_second_img_text">
+                  &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;铸大国重器，挺制造脊梁，致力能源未来，赋能美好生活。
+                                  </span>
+</body></html>
+    """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+
+# In the case the server does not issue a charset= or doesnt have content_type header set
+def test_check_encoding_detection(client, live_server):
+    set_html_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="text/html", _external=True)
+    client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(2)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should see the proper string
+    assert "铸大国重".encode('utf-8') in res.data
+    # Should not see the failed encoding
+    assert b'\xc2\xa7' not in res.data
+
+
+# In the case the server does not issue a charset= or doesnt have content_type header set
+def test_check_encoding_detection_missing_content_type_header(client, live_server):
+    set_html_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(2)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should see the proper string
+    assert "铸大国重".encode('utf-8') in res.data
+    # Should not see the failed encoding
+    assert b'\xc2\xa7' not in res.data

changedetectionio/tests/test_errorhandling.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
 import time
+
 from flask import url_for
 from . util import live_server_setup
 
@@ -17,7 +18,9 @@ def test_error_handler(client, live_server):
     time.sleep(1)
 
     # Add our URL to the import page
-    test_url = url_for('test_endpoint_403_error', _external=True)
+    test_url = url_for('test_endpoint',
+                       status_code=403,
+                       _external=True)
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},

changedetectionio/tests/test_html_to_text.py

@@ -0,0 +1,38 @@
+#!/usr/bin/python3
+"""Test suite for the method to extract text from an html string"""
+from ..html_tools import html_to_text
+
+
+def test_html_to_text_func():
+    test_html = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     <a href="/first_link"> More Text </a>
+     </br>
+     So let's see what happens.  </br>
+     <a href="second_link.com"> Even More Text </a>
+     </body>
+     </html>
+    """
+
+    # extract text, with 'render_anchor_tag_content' set to False
+    text_content = html_to_text(test_html, render_anchor_tag_content=False)
+
+    no_links_text = \
+        "Some initial text\n\nWhich is across multiple " \
+        "lines\n\nMore Text So let's see what happens. Even More Text"
+
+    # check that no links are in the extracted text
+    assert text_content == no_links_text
+
+    # extract text, with 'render_anchor_tag_content' set to True
+    text_content = html_to_text(test_html, render_anchor_tag_content=True)
+
+    links_text = \
+        "Some initial text\n\nWhich is across multiple lines\n\n[ More Text " \
+        "](/first_link) So let's see what happens. [ Even More Text ]" \
+        "(second_link.com)"
+
+    # check that links are present in the extracted text
+    assert text_content == links_text

changedetectionio/tests/test_ignorehyperlinks.py

@@ -0,0 +1,219 @@
+#!/usr/bin/python3
+"""Test suite for the render/not render anchor tag content functionality"""
+
+import time
+from flask import url_for
+from .util import live_server_setup
+
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <a href="/original_link"> Some More Text </a>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+# Should be the same as set_original_ignore_response() but with a different
+# link
+def set_modified_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <a href="/modified_link"> Some More Text </a>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+def test_render_anchor_tag_content_true(client, live_server):
+    """Testing that the link changes are detected when
+    render_anchor_tag_content setting is set to true"""
+    sleep_time_for_fetch_thread = 3
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # set original html text
+    set_original_ignore_response()
+
+    # Goto the settings page, choose not to ignore links
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "minutes_between_check": 180,
+            "render_anchor_tag_content": "true",
+            "fetch_backend": "html_requests",
+        },
+        follow_redirects=True,
+    )
+    assert b"Settings updated." in res.data
+
+    # Add our URL to the import page
+    test_url = url_for("test_endpoint", _external=True)
+    res = client.post(
+        url_for("import_page"), data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # set a new html text with a modified link
+    set_modified_ignore_response()
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # check that the anchor tag content is rendered
+    res = client.get(url_for("preview_page", uuid="first"))
+    assert '(/modified_link)' in res.data.decode()
+
+    # since the link has changed, and we chose to render anchor tag content,
+    # we should detect a change (new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b"unviewed" in res.data
+    assert b"/test-endpoint" in res.data
+
+    # Cleanup everything
+    res = client.get(url_for("api_delete", uuid="all"),
+                     follow_redirects=True)
+    assert b'Deleted' in res.data
+
+
+def test_render_anchor_tag_content_false(client, live_server):
+    """Testing that anchor tag content changes are ignored when
+    render_anchor_tag_content setting is set to false"""
+    sleep_time_for_fetch_thread = 3
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # set the original html text
+    set_original_ignore_response()
+
+    # Goto the settings page, choose to ignore hyperlinks
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "minutes_between_check": 180,
+            "render_anchor_tag_content": "false",
+            "fetch_backend": "html_requests",
+        },
+        follow_redirects=True,
+    )
+    assert b"Settings updated." in res.data
+
+    # Add our URL to the import page
+    test_url = url_for("test_endpoint", _external=True)
+    res = client.post(
+        url_for("import_page"), data={"urls": test_url}, follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # set a new html text, with a modified link
+    set_modified_ignore_response()
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # check that the anchor tag content is not rendered
+    res = client.get(url_for("preview_page", uuid="first"))
+    assert '(/modified_link)' not in res.data.decode()
+
+    # even though the link has changed, we shouldn't detect a change since
+    # we selected to not render anchor tag content (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b"unviewed" not in res.data
+    assert b"/test-endpoint" in res.data
+
+    # Cleanup everything
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+
+def test_render_anchor_tag_content_default(client, live_server):
+    """Testing that anchor tag content changes are ignored when the
+    render_anchor_tag_content setting is not explicitly selected"""
+    sleep_time_for_fetch_thread = 3
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # set the original html text
+    set_original_ignore_response()
+
+    # Goto the settings page, not passing the render_anchor_tag_content setting
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "minutes_between_check": 180,
+            "fetch_backend": "html_requests",
+        },
+        follow_redirects=True,
+    )
+    assert b"Settings updated." in res.data
+
+    # Add our URL to the import page
+    test_url = url_for("test_endpoint", _external=True)
+    res = client.post(
+        url_for("import_page"), data={"urls": test_url}, follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # set a new html text, with a modified link
+    set_modified_ignore_response()
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # check that the anchor tag content is not rendered
+    res = client.get(url_for("preview_page", uuid="first"))
+    assert '(/modified_link)' not in res.data.decode()
+
+    # even though the link has changed, we shouldn't detect a change since
+    # we did not select the setting and the default behaviour is to not
+    # render anchor tag content (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b"unviewed" not in res.data
+    assert b"/test-endpoint" in res.data
+
+    # Cleanup everything
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetectionio/tests/test_ignorestatuscode.py

@@ -0,0 +1,190 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_some_changed_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines, and a new thing too.</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_normal_page_check_works_with_ignore_status_code(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    set_original_response()
+
+    # Goto the settings page, add our ignore text
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "minutes_between_check": 180,
+            "ignore_status_codes": "y",
+            'fetch_backend': "html_requests"
+        },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    set_some_changed_response()
+    time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+    assert b'/test-endpoint' in res.data
+
+
+# Tests the whole stack works with staus codes ignored
+def test_403_page_check_works_with_ignore_status_code(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', status_code=403, _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, check our ignore option
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+    #  Make a change
+    set_some_changed_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should have 'unviewed' still
+    # Because it should be looking at only that 'sametext' id
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+
+# Tests the whole stack works with staus codes ignored
+def test_403_page_check_fails_without_ignore_status_code(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', status_code=403, _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, check our ignore option
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+    #  Make a change
+    set_some_changed_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should have 'unviewed' still
+    # Because it should be looking at only that 'sametext' id
+    res = client.get(url_for("index"))
+    assert b'Status Code 403' in res.data

changedetectionio/tests/test_jsonpath_selector.py

@@ -1,4 +1,5 @@
 #!/usr/bin/python3
+# coding=utf-8
 
 import time
 from flask import url_for
@@ -142,7 +143,7 @@ def set_modified_response():
         }
       ],
       "boss": {
-        "name": "Foobar"
+        "name": "Örnsköldsvik"
       },
       "available": false
     }
@@ -246,8 +247,10 @@ def test_check_json_filter(client, live_server):
 
     # Should not see this, because its not in the JSONPath we entered
     res = client.get(url_for("diff_history_page", uuid="first"))
+
     # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
-    assert b'Foobar' in res.data
+    # And #462 - check we see the proper utf-8 string there
+    assert "Örnsköldsvik".encode('utf-8') in res.data
 
 
 def test_check_json_filter_bool_val(client, live_server):

changedetectionio/tests/test_notification.py

@@ -2,15 +2,17 @@ import os
 import time
 import re
 from flask import url_for
-from . util import set_original_response, set_modified_response, live_server_setup
+from . util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup
 import logging
 from changedetectionio.notification import default_notification_body, default_notification_title
 
+def test_setup(live_server):
+    live_server_setup(live_server)
+
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_check_notification(client, live_server):
 
-    live_server_setup(live_server)
     set_original_response()
 
     # Give the endpoint time to spin up
@@ -49,84 +51,76 @@ def test_check_notification(client, live_server):
     notification_url = url.replace('http', 'json')
 
     print (">>>> Notification URL: "+notification_url)
+
+    notification_form_data = {"notification_urls": notification_url,
+                              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
+                              "notification_body": "BASE URL: {base_url}\n"
+                                                   "Watch URL: {watch_url}\n"
+                                                   "Watch UUID: {watch_uuid}\n"
+                                                   "Watch title: {watch_title}\n"
+                                                   "Watch tag: {watch_tag}\n"
+                                                   "Preview: {preview_url}\n"
+                                                   "Diff URL: {diff_url}\n"
+                                                   "Snapshot: {current_snapshot}\n"
+                                                   "Diff: {diff}\n"
+                                                   "Diff Full: {diff_full}\n"
+                                                   ":-)",
+                              "notification_format": "Text"}
+
+    notification_form_data.update({
+        "url": test_url,
+        "tag": "my tag",
+        "title": "my title",
+        "headers": "",
+        "fetch_backend": "html_requests"})
+
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"notification_urls": notification_url,
-              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
-              "notification_body": "BASE URL: {base_url}\n"
-                                   "Watch URL: {watch_url}\n"
-                                   "Watch UUID: {watch_uuid}\n"
-                                   "Watch title: {watch_title}\n"
-                                   "Watch tag: {watch_tag}\n"
-                                   "Preview: {preview_url}\n"
-                                   "Diff URL: {diff_url}\n"
-                                   "Snapshot: {current_snapshot}\n"
-                                   "Diff: {diff}\n"
-                                   "Diff Full: {diff_full}\n"
-                                   ":-)",
-              "notification_format": "Text",
-              "url": test_url,
-              "tag": "my tag",
-              "title": "my title",
-              "headers": "",
-              "fetch_backend": "html_requests",
-              "trigger_check": "y"},
+        data=notification_form_data,
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    assert b"Test notification queued" in res.data
+
 
     # Hit the edit page, be sure that we saved it
+    # Re #242 - wasnt saving?
     res = client.get(
         url_for("edit_page", uuid="first"))
     assert bytes(notification_url.encode('utf-8')) in res.data
-
-    # Re #242 - wasnt saving?
     assert bytes("New ChangeDetection.io Notification".encode('utf-8')) in res.data
 
 
 
-    # Because we hit 'send test notification on save'
+    ## Now recheck, and it should have sent the notification
     time.sleep(3)
+    set_modified_response()
 
     notification_submission = None
 
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(3)
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
-        # Did we see the URL that had a change, in the notification?
-
-    assert test_url in notification_submission
-
     os.unlink("test-datastore/notification.txt")
 
-    set_modified_response()
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(3)
-
-    # Did the front end see it?
-    res = client.get(
-        url_for("index"))
-
-    assert bytes("just now".encode('utf-8')) in res.data
-
-    notification_submission=None
-    # Verify what was sent as a notification
-    with open("test-datastore/notification.txt", "r") as f:
-        notification_submission = f.read()
-        # Did we see the URL that had a change, in the notification?
-
-    assert test_url in notification_submission
-
+    # Did we see the URL that had a change, in the notification?
     # Diff was correctly executed
+    assert test_url in notification_submission
+    assert ':-)' in notification_submission
     assert "Diff Full: Some initial text" in notification_submission
     assert "Diff: (changed) Which is across multiple lines" in notification_submission
-    assert "(-> into) which has this one new line" in notification_submission
-
+    assert "(into   ) which has this one new line" in notification_submission
+    # Re #342 - check for accidental python byte encoding of non-utf8/string
+    assert "b'" not in notification_submission
+    assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE)
+    assert "Watch title: my title" in notification_submission
+    assert "Watch tag: my tag" in notification_submission
+    assert "diff/" in notification_submission
+    assert "preview/" in notification_submission
+    assert ":-)" in notification_submission
+    assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
 
     if env_base_url:
         # Re #65 - did we see our BASE_URl ?
@@ -135,50 +129,17 @@ def test_check_notification(client, live_server):
     else:
         logging.debug(">>> Skipping BASE_URL check")
 
-    ##  Now configure something clever, we go into custom config (non-default) mode, this is returned by the endpoint
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(";jasdhflkjadshf kjhsdfkjl ahslkjf haslkjd hfaklsj hf\njl;asdhfkasj stuff we will detect\n")
 
-    res = client.post(
-        url_for("settings_page"),
-        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
-              "notification_urls": "json://foobar.com", #Re #143 should not see that it sent without [test checkbox]
-              "minutes_between_check": 180,
-              "fetch_backend": "html_requests",
-              },
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
-    # Re #143 - should not see this if we didnt hit the test box
-    assert b"Test notification queued" not in res.data
 
-    # Trigger a check
+    # This should insert the {current_snapshot}
+    set_more_modified_response()
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
     time.sleep(3)
-
-    # Did the front end see it?
-    res = client.get(
-        url_for("index"))
-
-    assert bytes("just now".encode('utf-8')) in res.data
-
+    # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
-        print ("Notification submission was:", notification_submission)
-        # Re #342 - check for accidental python byte encoding of non-utf8/string
-        assert "b'" not in notification_submission
+    assert "Ohh yeah awesome" in notification_submission
 
-        assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE)
-        assert "Watch title: my title" in notification_submission
-        assert "Watch tag: my tag" in notification_submission
-        assert "diff/" in notification_submission
-        assert "preview/" in notification_submission
-        assert ":-)" in notification_submission
-        assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
-        # This should insert the {current_snapshot}
-        assert "stuff we will detect" in notification_submission
 
     # Prove that "content constantly being marked as Changed with no Updating causes notification" is not a thing
     # https://github.com/dgtlmoon/changedetection.io/discussions/192
@@ -186,33 +147,39 @@ def test_check_notification(client, live_server):
 
     # Trigger a check
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
-    time.sleep(3)
+    time.sleep(1)
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
-    time.sleep(3)
+    time.sleep(1)
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
-    time.sleep(3)
+    time.sleep(1)
     assert os.path.exists("test-datastore/notification.txt") == False
 
-
-    # Now adding a wrong token should give us an error
-    res = client.post(
-        url_for("settings_page"),
-        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
-              "notification_body": "Rubbish: {rubbish}\n",
-              "notification_format": "Text",
-              "notification_urls": "json://foobar.com",
-              "minutes_between_check": 180,
-              "fetch_backend": "html_requests"
-              },
+    # cleanup for the next
+    client.get(
+        url_for("api_delete", uuid="first"),
         follow_redirects=True
     )
 
-    assert bytes("is not a valid token".encode('utf-8')) in res.data
+
+def test_notification_validation(client, live_server):
+    #live_server_setup(live_server)
+    time.sleep(3)
+    # re #242 - when you edited an existing new entry, it would not correctly show the notification settings
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("api_watch_add"),
+        data={"url": test_url, "tag": 'nice one'},
+        follow_redirects=True
+    )
+    with open("xxx.bin", "wb") as f:
+        f.write(res.data)
+    assert b"Watch added" in res.data
 
     # Re #360 some validation
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"notification_urls": notification_url,
+        data={"notification_urls": 'json://localhost/foobar',
               "notification_title": "",
               "notification_body": "",
               "notification_format": "Text",
@@ -220,8 +187,28 @@ def test_check_notification(client, live_server):
               "tag": "my tag",
               "title": "my title",
               "headers": "",
-              "fetch_backend": "html_requests",
-              "trigger_check": "y"},
+              "fetch_backend": "html_requests"},
         follow_redirects=True
     )
     assert b"Notification Body and Title is required when a Notification URL is used" in res.data
+
+    # Now adding a wrong token should give us an error
+    res = client.post(
+        url_for("settings_page"),
+        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
+              "notification_body": "Rubbish: {rubbish}\n",
+              "notification_format": "Text",
+              "notification_urls": "json://localhost/foobar",
+              "time_between_check": {'seconds': 180},
+              "fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+
+    assert bytes("is not a valid token".encode('utf-8')) in res.data
+
+    # cleanup for the next
+    client.get(
+        url_for("api_delete", uuid="first"),
+        follow_redirects=True
+    )

changedetectionio/tests/test_request.py

@@ -77,14 +77,6 @@ def test_body_in_request(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_body', _external=True)
 
-    # Add the test URL twice, we will check
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},
@@ -94,19 +86,6 @@ def test_body_in_request(client, live_server):
 
     body_value = 'Test Body Value'
 
-    # Attempt to add a body with a GET method
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-              "url": test_url,
-              "tag": "",
-              "method": "GET",
-              "fetch_backend": "html_requests",
-              "body": "invalid"},
-        follow_redirects=True
-    )
-    assert b"Body must be empty when Request Method is set to GET" in res.data
-
     # Add a properly formatted body with a proper method
     res = client.post(
         url_for("edit_page", uuid="first"),
@@ -120,8 +99,7 @@ def test_body_in_request(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    # Give the thread time to pick up the first version
-    time.sleep(5)
+    time.sleep(3)
 
     # The service should echo back the body
     res = client.get(
@@ -129,9 +107,20 @@ def test_body_in_request(client, live_server):
         follow_redirects=True
     )
 
-    # Check if body returned contains the specified data
+    # If this gets stuck something is wrong, something should always be there
+    assert b"No history found" not in res.data
+    # We should see what we sent in the reply
     assert str.encode(body_value) in res.data
 
+    ####### data sanity checks
+    # Add the test URL twice, we will check
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
     watches_with_body = 0
     with open('test-datastore/url-watches.json') as f:
         app_struct = json.load(f)
@@ -142,6 +131,20 @@ def test_body_in_request(client, live_server):
     # Should be only one with body set
     assert watches_with_body==1
 
+    # Attempt to add a body with a GET method
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": test_url,
+              "tag": "",
+              "method": "GET",
+              "fetch_backend": "html_requests",
+              "body": "invalid"},
+        follow_redirects=True
+    )
+    assert b"Body must be empty when Request Method is set to GET" in res.data
+
+
 def test_method_in_request(client, live_server):
     # Add our URL to the import page
     test_url = url_for('test_method', _external=True)

changedetectionio/tests/test_security.py

@@ -0,0 +1,36 @@
+from flask import url_for
+from . util import set_original_response, set_modified_response, live_server_setup
+import time
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def test_file_access(client, live_server):
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": 'https://localhost'},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    # Attempt to add a body with a GET method
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": 'file:///etc/passwd',
+              "tag": "",
+              "method": "GET",
+              "fetch_backend": "html_requests",
+              "body": ""},
+        follow_redirects=True
+    )
+    time.sleep(3)
+
+    res = client.get(
+        url_for("index", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b'denied for security reasons' in res.data

changedetectionio/tests/unit/test-content/after-2.txt

@@ -0,0 +1,3 @@
+After twenty years, as cursed as I may be
+ok
+and insure that I'm one of those computer nerds.

changedetectionio/tests/unit/test-content/after.txt

@@ -2,5 +2,6 @@ After twenty years, as cursed as I may be
 for having learned computerese,
 I continue to examine bits, bytes and words
 xok
+next-x-ok
 and insure that I'm one of those computer nerds.
 and something new

changedetectionio/tests/unit/test_notification_diff.py

@@ -12,12 +12,19 @@ from changedetectionio import diff
 class TestDiffBuilder(unittest.TestCase):
 
     def test_expected_diff_output(self):
-        base_dir=os.path.dirname(__file__)
-        output = diff.render_diff(base_dir+"/test-content/before.txt", base_dir+"/test-content/after.txt")
+        base_dir = os.path.dirname(__file__)
+        output = diff.render_diff(previous_file=base_dir + "/test-content/before.txt", newest_file=base_dir + "/test-content/after.txt")
         output = output.split("\n")
-        self.assertIn("(changed) ok", output)
-        self.assertIn("(-> into) xok", output)
-        self.assertIn("(added) and something new", output)
+        self.assertIn('(changed) ok', output)
+        self.assertIn('(into   ) xok', output)
+        self.assertIn('(into   ) next-x-ok', output)
+        self.assertIn('(added  ) and something new', output)
+
+
+        output = diff.render_diff(previous_file=base_dir + "/test-content/before.txt", newest_file=base_dir + "/test-content/after-2.txt")
+        output = output.split("\n")
+        self.assertIn('(removed) for having learned computerese,', output)
+        self.assertIn('(removed) I continue to examine bits, bytes and words', output)
 
         # @todo test blocks of changed, blocks of added, blocks of removed
 

changedetectionio/tests/util.py

@@ -35,24 +35,40 @@ def set_modified_response():
 
     return None
 
+def set_more_modified_response():
+    test_return_data = """<html>
+    <head><title>modified head title</title></head>
+    <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     Ohh yeah awesome<br/>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
 
 def live_server_setup(live_server):
 
-
     @live_server.app.route('/test-endpoint')
     def test_endpoint():
         ctype = request.args.get('content_type')
+        status_code = request.args.get('status_code')
 
-        # Tried using a global var here but didn't seem to work, so reading from a file instead.
-        with open("test-datastore/endpoint-content.txt", "r") as f:
-            resp = make_response(f.read())
-            resp.headers['Content-Type'] = ctype if ctype else 'text/html'
-            return resp
-
-    @live_server.app.route('/test-403')
-    def test_endpoint_403_error():
-        resp = make_response('', 403)
-        return resp
+        try:
+            # Tried using a global var here but didn't seem to work, so reading from a file instead.
+            with open("test-datastore/endpoint-content.txt", "r") as f:
+                resp = make_response(f.read(), status_code)
+                resp.headers['Content-Type'] = ctype if ctype else 'text/html'
+                return resp
+        except FileNotFoundError:
+            return make_response('', status_code)
 
     # Just return the headers in the request
     @live_server.app.route('/test-headers')
@@ -84,7 +100,7 @@ def live_server_setup(live_server):
             if data != None:
                 f.write(data)
 
-        print("\n>> Test notification endpoint was hit.\n")
+        print("\n>> Test notification endpoint was hit.\n", data)
         return "Text was set"
 
 

changedetectionio/update_worker.py

@@ -38,20 +38,18 @@ class update_worker(threading.Thread):
 
                     changed_detected = False
                     contents = ""
+                    screenshot = False
                     update_obj= {}
                     now = time.time()
 
                     try:
-
-                        changed_detected, update_obj, contents = update_handler.run(uuid)
+                        changed_detected, update_obj, contents, screenshot = update_handler.run(uuid)
 
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
                         # We then convert/.decode('utf-8') for the notification etc
                         if not isinstance(contents, (bytes, bytearray)):
                             raise Exception("Error - returned data from the fetch handler SHOULD be bytes")
-
-
                     except PermissionError as e:
                         self.app.logger.error("File permission error updating", uuid, str(e))
                     except content_fetcher.EmptyReply as e:
@@ -143,6 +141,9 @@ class update_worker(threading.Thread):
                         # Always record that we atleast tried
                         self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
                                                                            'last_checked': round(time.time())})
+                        # Always save the screenshot if it's available
+                        if screenshot:
+                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
 
                 self.current_uuid = None  # Done
                 self.q.task_done()

docker-compose.yml

@@ -1,9 +1,9 @@
 version: '2'
 services:
-    changedetection.io:
+    changedetection:
       image: ghcr.io/dgtlmoon/changedetection.io
-      container_name: changedetection.io
-      hostname: changedetection.io
+      container_name: changedetection
+      hostname: changedetection
       volumes:
         - changedetection-data:/datastore
 

requirements.txt

@@ -1,9 +1,9 @@
 flask~= 2.0
-
+flask_wtf
 eventlet>=0.31.0
 validators
 timeago ~=1.0
-inscriptis ~= 1.2
+inscriptis ~= 2.2
 feedgen ~= 0.9
 flask-login ~= 0.5
 pytz
@@ -17,7 +17,7 @@ wtforms ~= 2.3.3
 jsonpath-ng ~= 1.5.3
 
 # Notification library
-apprise ~= 0.9.6
+apprise ~= 0.9.7
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
@@ -34,5 +34,9 @@ lxml
 
 # 3.141 was missing socksVersion, 3.150 was not in pypi, so we try 4.1.0
 selenium ~= 4.1.0
-pytest ~=6.2
-pytest-flask ~=1.2
+
+# https://stackoverflow.com/questions/71652965/importerror-cannot-import-name-safe-str-cmp-from-werkzeug-security/71653849#71653849
+# ImportError: cannot import name 'safe_str_cmp' from 'werkzeug.security'
+# need to revisit flask login versions
+werkzeug ~= 2.0.0
+

setup.py

@@ -32,11 +32,11 @@ setup(
     long_description_content_type='text/markdown',
     keywords='website change monitor for changes notification change detection '
              'alerts tracking website tracker change alert website and monitoring',
-    zip_safe=False,
-    entry_points={"console_scripts": ["changedetection.io=changedetection:main"]},
+    entry_points={"console_scripts": ["changedetection.io=changedetectionio.changedetection:main"]},
+    zip_safe=True,
+    scripts=["changedetection.py"],
     author='dgtlmoon',
     url='https://changedetection.io',
-    scripts=['changedetection.py'],
     packages=['changedetectionio'],
     include_package_data=True,
     install_requires=install_requires,