0.40.2

.github/workflows/containers.yml

@@ -50,7 +50,6 @@ jobs:
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
 
       - name: Create release metadata
         run: |
@@ -99,6 +98,7 @@ jobs:
           platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
+          provenance: false
 
       # A new tagged release is required, which builds :tag and :latest
       - name: Build and push :tag
@@ -117,6 +117,7 @@ jobs:
           platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
+          provenance: false
 
       - name: Image digest
         run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}

.github/workflows/pypi.yml

@@ -19,12 +19,6 @@ jobs:
         with:
           python-version: 3.9
 
-#      - name: Install dependencies
-#        run: |
-#          python -m pip install --upgrade pip
-#          pip install flake8 pytest
-#          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-#          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
 
       - name: Test that pip builds without error
         run: |

.github/workflows/test-container-build.yml

@@ -10,11 +10,13 @@ on:
     paths:
       - requirements.txt
       - Dockerfile
+      - .github/workflows/*
 
   pull_request:
     paths:
       - requirements.txt
       - Dockerfile
+      - .github/workflows/*
 
   # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
   # @todo: some kind of path filter for requirements.txt and Dockerfile

.github/workflows/test-only.yml

@@ -8,32 +8,70 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.9
+
+      # Mainly just for link/flake8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install flake8 pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+          python-version: '3.10'
 
       - name: Lint with flake8
         run: |
+          pip3 install flake8
           # stop the build if there are Python syntax errors or undefined names
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
-      - name: Unit tests
+      - name: Spin up ancillary testable services
         run: |
-          python3 -m unittest changedetectionio.tests.unit.test_notification_diff
+          
+          docker network create changedet-network
 
-      - name: Test with pytest
+          # Selenium+browserless
+          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome-debug:3.141.59
+          docker run --network changedet-network -d --hostname browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
+
+      - name: Build changedetection.io container for testing
+        run: |         
+          # Build a changedetection.io container and start testing inside
+          docker build . -t test-changedetectionio
+
+      - name: Test built container with pytest
         run: |
-          # Each test is totally isolated and performs its own cleanup/reset
-          cd changedetectionio; ./run_all_tests.sh
+          
+          # Unit tests
+          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
+          
+          # All tests
+          docker run --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
 
+      - name: Test built container selenium+browserless/playwright
+        run: |
+          
+          # Selenium fetch
+          docker run -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
+          
+          # Playwright/Browserless fetch
+          docker run -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
 
+      - name: Test proxy interaction
+        run: |
+          cd changedetectionio
+          ./run_proxy_tests.sh
+          cd ..
+
+      - name: Test changedetection.io container starts+runs basically without error
+        run: |
+          docker run -p 5556:5000 -d test-changedetectionio
+          sleep 3
+          # Should return 0 (no error) when grep finds it
+          curl -s http://localhost:5556 |grep -q checkbox-uuid
+          
+          # and IPv6
+          curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+          
+
+#export WEBDRIVER_URL=http://localhost:4444/wd/hub
+#pytest tests/fetchers/test_content.py
+#pytest tests/test_errorhandling.py

CONTRIBUTING.md

@@ -7,9 +7,3 @@ Otherwise, it's always best to PR into the `dev` branch.
 Please be sure that all new functionality has a matching test!
 
 Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notification.py` for example
-
-```
-pip3 install -r requirements-dev
-```
-
-this is from https://github.com/dgtlmoon/changedetection.io/blob/master/requirements-dev.txt

Dockerfile

@@ -1,7 +1,7 @@
 # pip dependencies install stage
-FROM python:3.8-slim as builder
+FROM python:3.10-slim as builder
 
-# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
+# See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -29,22 +29,16 @@ RUN pip install --target=/dependencies playwright~=1.27.1 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.8-slim
+FROM python:3.10-slim
 
-# Actual packages needed at runtime, usually due to the notification (apprise) backend
-# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
-ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
-
-# Re #93, #73, excluding rustc (adds another 430Mb~)
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    g++ \
-    gcc \
-    libc-dev \
-    libffi-dev \
-    libjpeg-dev \
-    libssl-dev \
-    libxslt-dev \
-    zlib1g-dev
+    libssl1.1 \
+    libxslt1.1 \
+    # For pdftohtml
+    poppler-utils \
+    zlib1g \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
 
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1

MANIFEST.in

@@ -1,9 +1,10 @@
 recursive-include changedetectionio/api *
-recursive-include changedetectionio/templates *
-recursive-include changedetectionio/static *
+recursive-include changedetectionio/blueprint *
 recursive-include changedetectionio/model *
-recursive-include changedetectionio/tests *
 recursive-include changedetectionio/res *
+recursive-include changedetectionio/static *
+recursive-include changedetectionio/templates *
+recursive-include changedetectionio/tests *
 prune changedetectionio/static/package-lock.json
 prune changedetectionio/static/styles/node_modules
 prune changedetectionio/static/styles/package-lock.json

README.md

@@ -1,6 +1,8 @@
 ## Web Site Change Detection, Monitoring and Notification.
 
-_Live your data-life pro-actively, Detect website changes and perform meaningful actions, trigger notifications via Discord, Email, Slack, Telegram, API calls and many more._
+**_Detect website content changes and perform meaningful actions - trigger notifications via Discord, Email, Slack, Telegram, API calls and many more._**
+
+_Live your data-life pro-actively._ 
 
 
 [<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start?src=github)
@@ -9,22 +11,41 @@ _Live your data-life pro-actively, Detect website changes and perform meaningful
 
 ![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 
-[**Don't have time? Let us host it for you! try our $6.99/month subscription - use our proxies and support!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
+[**Don't have time? Let us host it for you! try our $8.99/month subscription - use our proxies and support!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
 
 - Chrome browser included.
 - Super fast, no registration needed setup.
-- Start watching and receiving change notifications instantly.
+- Get started watching and receiving website change notifications straight away.
 
 
-Easily see what changed, examine by word, line, or individual character.
+### Target specific parts of the webpage using the Visual Selector tool.
 
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
+Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://lemonade.changedetection.io/start?src=github)
+
+### Easily see what changed, examine by word, line, or individual character.
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://lemonade.changedetection.io/start?src=github)
 
 
-#### Example use cases
+### Perform interactive browser steps
+
+Fill in text boxes, click buttons and more, setup your changedetection scenario. 
+
+Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
+
+[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Website change detection with interactive browser steps, login, cookies etc" />](https://lemonade.changedetection.io/start?src=github)
+
+After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
+Requires Playwright to be enabled.
+
+
+### Example use cases
 
 - Products and services have a change in pricing
 - _Out of stock notification_ and _Back In stock notification_
+- Monitor and track PDF file changes, know when a PDF file has text changes.
 - Governmental department updates (changes are often only on their websites)
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
@@ -50,6 +71,7 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 - Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
 - Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
 - Switch between fast non-JS and Chrome JS based "fetchers"
+- Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)
 - Easily specify how often a site should be checked
 - Execute JS before extracting text (Good for logging in, see examples in the UI!)
 - Override Request Headers, Specify `POST` or `GET` and other methods
@@ -59,27 +81,8 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 
 We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
 
-## Screenshots
-
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
-### Target specific parts of the webpage using the Visual Selector tool.
-
-Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
-
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
-
-### Perform interactive browser steps
-
-Fill in text boxes, click buttons and more, setup your changedetection scenario. 
-
-Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
-
-<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Website change detection with interactive browser steps, login, cookies etc" />
-
-After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
-Requires Playwright to be enabled.
-
 ## Installation
 
 ### Docker
@@ -160,7 +163,7 @@ Just some examples
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
 
-Now you can also customise your notification content!
+Now you can also customise your notification content and use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2 templating</a> for their title and body!
 
 ## JSON API Monitoring
 
@@ -188,11 +191,29 @@ When you enable a `json:` or `jq:` filter, you can even automatically extract an
 <html>
 ...
 <script type="application/ld+json">
-  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","price": 23.50 }
+
+{
+   "@context":"http://schema.org/",
+   "@type":"Product",
+   "offers":{
+      "@type":"Offer",
+      "availability":"http://schema.org/InStock",
+      "price":"3949.99",
+      "priceCurrency":"USD",
+      "url":"https://www.newegg.com/p/3D5-000D-001T1"
+   },
+   "description":"Cobratype King Cobra Hero Desktop Gaming PC",
+   "name":"Cobratype King Cobra Hero Desktop Gaming PC",
+   "sku":"3D5-000D-001T1",
+   "itemCondition":"NewCondition"
+}
 </script>
 ```  
 
-`json:$.price` or `jq:.price` would give `23.50`, or you can extract the whole structure
+`json:$..price` or `jq:..price` would give `3949.99`, or you can extract the whole structure (use a JSONpath test website to validate with)
+
+The application also supports notifying you that it can follow this information automatically
+
 
 ## Proxy Configuration
 
@@ -202,6 +223,9 @@ See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configura
 
 Raspberry Pi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! See the wiki for [details](https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver)
 
+## API Support
+
+Supports managing the website watch list [via our API](https://changedetection.io/docs/api_v1/index.html)
 
 ## Support us
 
@@ -226,5 +250,5 @@ I offer commercial support, this software is depended on by network security, ae
 [test-shield]: https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master
 
 [license-shield]: https://img.shields.io/github/license/dgtlmoon/changedetection.io.svg?style=for-the-badge
-[release-link]: https://github.com/dgtlmoon.com/changedetection.io/releases
+[release-link]: https://github.com/dgtlmoon/changedetection.io/releases
 [docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io

changedetection.py

@@ -7,7 +7,7 @@
 
 from changedetectionio import changedetection
 import multiprocessing
-import signal
+import sys
 import os
 
 def sigchld_handler(_signo, _stack_frame):
@@ -35,6 +35,9 @@ if __name__ == '__main__':
     try:
         while True:
             time.sleep(1)
+            if not parse_process.is_alive():
+                # Process died/crashed for some reason, exit with error set
+                sys.exit(1)
 
     except KeyboardInterrupt:
         #parse_process.terminate() not needed, because this process will issue it to the sub-process anyway

changedetectionio/__init__.py

@@ -1,5 +1,15 @@
 #!/usr/bin/python3
 
+from changedetectionio import queuedWatchMetaData
+from copy import deepcopy
+from distutils.util import strtobool
+from feedgen.feed import FeedGenerator
+from flask_compress import Compress as FlaskCompress
+from flask_login import current_user
+from flask_restful import abort, Api
+from flask_wtf import CSRFProtect
+from functools import wraps
+from threading import Event
 import datetime
 import flask_login
 import logging
@@ -10,11 +20,6 @@ import threading
 import time
 import timeago
 
-from copy import deepcopy
-from distutils.util import strtobool
-from feedgen.feed import FeedGenerator
-from threading import Event
-
 from flask import (
     Flask,
     abort,
@@ -27,14 +32,11 @@ from flask import (
     session,
     url_for,
 )
-from flask_login import login_required
-from flask_restful import abort, Api
-from flask_wtf import CSRFProtect
 
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.39.22.1'
+__version__ = '0.40.2'
 
 datastore = None
 
@@ -52,6 +54,9 @@ app = Flask(__name__,
             static_folder="static",
             template_folder="templates")
 
+# Super handy for compressing large BrowserSteps responses and others
+FlaskCompress(app)
+
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
 
@@ -59,8 +64,6 @@ app.config.exit = Event()
 
 app.config['NEW_VERSION_AVAILABLE'] = False
 
-app.config['LOGIN_DISABLED'] = False
-
 #app.config["EXPLAIN_TEMPLATE_LOADING"] = True
 
 # Disables caching of the templates
@@ -68,7 +71,6 @@ app.config['TEMPLATES_AUTO_RELOAD'] = True
 app.jinja_env.add_extension('jinja2.ext.loopcontrols')
 csrf = CSRFProtect()
 csrf.init_app(app)
-
 notification_debug_log=[]
 
 watch_api = Api(app, decorators=[csrf.exempt])
@@ -90,6 +92,12 @@ def init_app_secret(datastore_path):
 
     return secret
 
+
+@app.template_global()
+def get_darkmode_state():
+    css_dark_mode = request.cookies.get('css_dark_mode', 'false')
+    return 'true' if css_dark_mode and strtobool(css_dark_mode) else 'false'
+
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
 # running or something similar.
 @app.template_filter('format_last_checked_time')
@@ -137,7 +145,6 @@ class User(flask_login.UserMixin):
 
     # Compare given password against JSON store or Env var
     def check_password(self, password):
-
         import base64
         import hashlib
 
@@ -145,11 +152,9 @@ class User(flask_login.UserMixin):
         raw_salt_pass = os.getenv("SALTED_PASS", False)
 
         if not raw_salt_pass:
-            raw_salt_pass = datastore.data['settings']['application']['password']
+            raw_salt_pass = datastore.data['settings']['application'].get('password')
 
         raw_salt_pass = base64.b64decode(raw_salt_pass)
-
-
         salt_from_storage = raw_salt_pass[:32]  # 32 is the length of the salt
 
         # Use the exact same setup you used to generate the key, but this time put in the password to check
@@ -159,21 +164,44 @@ class User(flask_login.UserMixin):
             salt_from_storage,
             100000
         )
-        new_key =  salt_from_storage + new_key
+        new_key = salt_from_storage + new_key
 
         return new_key == raw_salt_pass
 
     pass
 
+
+def login_optionally_required(func):
+    @wraps(func)
+    def decorated_view(*args, **kwargs):
+
+        has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False)
+
+        # Permitted
+        if request.endpoint == 'static_content' and request.view_args['group'] == 'styles':
+            return func(*args, **kwargs)
+        # Permitted
+        elif request.endpoint == 'diff_history_page' and datastore.data['settings']['application'].get('shared_diff_access'):
+            return func(*args, **kwargs)
+
+        elif request.method in flask_login.config.EXEMPT_METHODS:
+            return func(*args, **kwargs)
+        elif app.config.get('LOGIN_DISABLED'):
+            return func(*args, **kwargs)
+        elif has_password_enabled and not current_user.is_authenticated:
+            return app.login_manager.unauthorized()
+
+        return func(*args, **kwargs)
+
+    return decorated_view
+
 def changedetection_app(config=None, datastore_o=None):
     global datastore
     datastore = datastore_o
 
     # so far just for read-only via tests, but this will be moved eventually to be the main source
     # (instead of the global var)
-    app.config['DATASTORE']=datastore_o
-
-    #app.config.update(config or {})
+    app.config['DATASTORE'] = datastore_o
 
     login_manager = flask_login.LoginManager(app)
     login_manager.login_view = 'login'
@@ -197,12 +225,12 @@ def changedetection_app(config=None, datastore_o=None):
     watch_api.add_resource(api_v1.SystemInfo, '/api/v1/systeminfo',
                            resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
 
-
-
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
     #    CORS(app)
 
+
+
     @login_manager.user_loader
     def user_loader(email):
         user = User()
@@ -211,7 +239,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     @login_manager.unauthorized_handler
     def unauthorized_handler():
-        # @todo validate its a URL of this host and use that
+        flash("You must be logged in, please log in.", 'error')
         return redirect(url_for('login', next=url_for('index')))
 
     @app.route('/logout')
@@ -224,10 +252,6 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route('/login', methods=['GET', 'POST'])
     def login():
 
-        if not datastore.data['settings']['application']['password'] and not os.getenv("SALTED_PASS", False):
-            flash("Login not required, no password enabled.", "notice")
-            return redirect(url_for('index'))
-
         if request.method == 'GET':
             if flask_login.current_user.is_authenticated:
                 flash("Already logged in")
@@ -262,27 +286,22 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('login'))
 
     @app.before_request
-    def do_something_whenever_a_request_comes_in():
-
-        # Disable password login if there is not one set
-        # (No password in settings or env var)
-        app.config['LOGIN_DISABLED'] = datastore.data['settings']['application']['password'] == False and os.getenv("SALTED_PASS", False) == False
-
+    def before_request_handle_cookie_x_settings():
         # Set the auth cookie path if we're running as X-settings/X-Forwarded-Prefix
         if os.getenv('USE_X_SETTINGS') and 'X-Forwarded-Prefix' in request.headers:
             app.config['REMEMBER_COOKIE_PATH'] = request.headers['X-Forwarded-Prefix']
             app.config['SESSION_COOKIE_PATH'] = request.headers['X-Forwarded-Prefix']
 
-        # For the RSS path, allow access via a token
-        if request.path == '/rss' and request.args.get('token'):
-            app_rss_token = datastore.data['settings']['application']['rss_access_token']
-            rss_url_token = request.args.get('token')
-            if app_rss_token == rss_url_token:
-                app.config['LOGIN_DISABLED'] = True
+        return None
 
     @app.route("/rss", methods=['GET'])
-    @login_required
     def rss():
+        # Always requires token set
+        app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
+        rss_url_token = request.args.get('token')
+        if rss_url_token != app_rss_token:
+            return "Access denied, bad token", 403
+
         from . import diff
         limit_tag = request.args.get('tag')
 
@@ -356,7 +375,7 @@ def changedetection_app(config=None, datastore_o=None):
         return response
 
     @app.route("/", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def index():
         from changedetectionio import forms
 
@@ -369,9 +388,9 @@ def changedetection_app(config=None, datastore_o=None):
         if op:
             uuid = request.args.get('uuid')
             if op == 'pause':
-                datastore.data['watching'][uuid]['paused'] ^= True
+                datastore.data['watching'][uuid].toggle_pause()
             elif op == 'mute':
-                datastore.data['watching'][uuid]['notification_muted'] ^= True
+                datastore.data['watching'][uuid].toggle_mute()
 
             datastore.needs_write = True
             return redirect(url_for('index', tag = limit_tag))
@@ -395,19 +414,22 @@ def changedetection_app(config=None, datastore_o=None):
                 sorted_watches.append(watch)
 
         existing_tags = datastore.get_all_tags()
-
         form = forms.quickWatchForm(request.form)
-        output = render_template("watch-overview.html",
-                                 form=form,
-                                 watches=sorted_watches,
-                                 tags=existing_tags,
+        output = render_template(
+            "watch-overview.html",
+                                 # Don't link to hosting when we're on the hosting environment
                                  active_tag=limit_tag,
                                  app_rss_token=datastore.data['settings']['application']['rss_access_token'],
-                                 has_unviewed=datastore.has_unviewed,
-                                 # Don't link to hosting when we're on the hosting environment
-                                 hosted_sticky=os.getenv("SALTED_PASS", False) == False,
+                                 form=form,
                                  guid=datastore.data['app_guid'],
-                                 queued_uuids=[uuid for p,uuid in update_q.queue])
+                                 has_proxies=datastore.proxy_list,
+                                 has_unviewed=datastore.has_unviewed,
+                                 hosted_sticky=os.getenv("SALTED_PASS", False) == False,
+                                 queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
+                                 system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
+                                 tags=existing_tags,
+                                 watches=sorted_watches
+                                 )
 
 
         if session.get('share-link'):
@@ -417,7 +439,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     # AJAX endpoint for sending a test
     @app.route("/notification/send-test", methods=['POST'])
-    @login_required
+    @login_optionally_required
     def ajax_callback_send_notification_test():
 
         import apprise
@@ -450,7 +472,7 @@ def changedetection_app(config=None, datastore_o=None):
 
 
     @app.route("/clear_history/<string:uuid>", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def clear_watch_history(uuid):
         try:
             datastore.clear_watch_history(uuid)
@@ -462,7 +484,7 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/clear_history", methods=['GET', 'POST'])
-    @login_required
+    @login_optionally_required
     def clear_all_history():
 
         if request.method == 'POST':
@@ -519,7 +541,7 @@ def changedetection_app(config=None, datastore_o=None):
 
 
     @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
-    @login_required
+    @login_optionally_required
     # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists
     # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ?
 
@@ -559,6 +581,8 @@ def changedetection_app(config=None, datastore_o=None):
                                data=default,
                                )
 
+        form.fetch_backend.choices.append(("system", 'System settings default'))
+
         # form.browser_steps[0] can be assumed that we 'goto url' first
 
         if datastore.proxy_list is None:
@@ -571,6 +595,7 @@ def changedetection_app(config=None, datastore_o=None):
 
 
         if request.method == 'POST' and form.validate():
+
             extra_update_obj = {}
 
             if request.args.get('unpause_on_save'):
@@ -587,25 +612,12 @@ def changedetection_app(config=None, datastore_o=None):
                     using_default_check_time = False
                     break
 
-            # Use the default if its the same as system wide
-            if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']:
-                extra_update_obj['fetch_backend'] = None
 
 
              # Ignore text
             form_ignore_text = form.ignore_text.data
             datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
 
-            # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
-            if form_ignore_text:
-                if len(datastore.data['watching'][uuid].history):
-                    extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
-
-            # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
-            if form.include_filters.data != datastore.data['watching'][uuid].get('include_filters', []):
-                if len(datastore.data['watching'][uuid].history):
-                    extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
-
             # Be sure proxy value is None
             if datastore.proxy_list is not None and form.data['proxy'] == '':
                 extra_update_obj['proxy'] = None
@@ -623,7 +635,7 @@ def changedetection_app(config=None, datastore_o=None):
             datastore.needs_write_urgent = True
 
             # Queue the watch for immediate recheck, with a higher priority
-            update_q.put((1, uuid))
+            update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
 
             # Diff page [edit] link should go back to diff page
             if request.args.get("next") and request.args.get("next") == 'diff':
@@ -637,8 +649,6 @@ def changedetection_app(config=None, datastore_o=None):
 
             visualselector_data_is_ready = datastore.visualselector_data_is_ready(uuid)
 
-            # Only works reliably with Playwright
-            visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and default['fetch_backend'] == 'html_webdriver'
 
             # JQ is difficult to install on windows and must be manually added (outside requirements.txt)
             jq_support = True
@@ -649,8 +659,13 @@ def changedetection_app(config=None, datastore_o=None):
 
             watch = datastore.data['watching'].get(uuid)
             system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
-            is_html_webdriver = True if watch.get('fetch_backend') == 'html_webdriver' or (
-                    watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
+
+            is_html_webdriver = False
+            if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver':
+                is_html_webdriver = True
+
+            # Only works reliably with Playwright
+            visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver
 
             output = render_template("edit.html",
                                      browser_steps_config=browser_step_ui_config,
@@ -672,7 +687,7 @@ def changedetection_app(config=None, datastore_o=None):
         return output
 
     @app.route("/settings", methods=['GET', "POST"])
-    @login_required
+    @login_optionally_required
     def settings_page():
         from changedetectionio import content_fetcher, forms
 
@@ -752,7 +767,7 @@ def changedetection_app(config=None, datastore_o=None):
         return output
 
     @app.route("/import", methods=['GET', "POST"])
-    @login_required
+    @login_optionally_required
     def import_page():
         remaining_urls = []
         if request.method == 'POST':
@@ -764,7 +779,7 @@ def changedetection_app(config=None, datastore_o=None):
                 importer = import_url_list()
                 importer.run(data=request.values.get('urls'), flash=flash, datastore=datastore)
                 for uuid in importer.new_uuids:
-                    update_q.put((1, uuid))
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
 
                 if len(importer.remaining_data) == 0:
                     return redirect(url_for('index'))
@@ -777,7 +792,7 @@ def changedetection_app(config=None, datastore_o=None):
                 d_importer = import_distill_io_json()
                 d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)
                 for uuid in d_importer.new_uuids:
-                    update_q.put((1, uuid))
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
 
 
 
@@ -790,7 +805,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     # Clear all statuses, so we do not see the 'unviewed' class
     @app.route("/form/mark-all-viewed", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def mark_all_viewed():
 
         # Save the current newest history as the most recently viewed
@@ -799,10 +814,12 @@ def changedetection_app(config=None, datastore_o=None):
 
         return redirect(url_for('index'))
 
-    @app.route("/diff/<string:uuid>", methods=['GET'])
-    @login_required
+    @app.route("/diff/<string:uuid>", methods=['GET', 'POST'])
+    @login_optionally_required
     def diff_history_page(uuid):
 
+        from changedetectionio import forms
+
         # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
@@ -814,6 +831,28 @@ def changedetection_app(config=None, datastore_o=None):
             flash("No history found for the specified link, bad link?", "error")
             return redirect(url_for('index'))
 
+        # For submission of requesting an extract
+        extract_form = forms.extractDataForm(request.form)
+        if request.method == 'POST':
+            if not extract_form.validate():
+                flash("An error occurred, please see below.", "error")
+
+            else:
+                extract_regex = request.form.get('extract_regex').strip()
+                output = watch.extract_regex_from_all_history(extract_regex)
+                if output:
+                    watch_dir = os.path.join(datastore_o.datastore_path, uuid)
+                    response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True))
+                    response.headers['Content-type'] = 'text/csv'
+                    response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                    response.headers['Pragma'] = 'no-cache'
+                    response.headers['Expires'] = 0
+                    return response
+
+
+                flash('Nothing matches that RegEx', 'error')
+                redirect(url_for('diff_history_page', uuid=uuid)+'#extract')
+
         history = watch.history
         dates = list(history.keys())
 
@@ -852,32 +891,39 @@ def changedetection_app(config=None, datastore_o=None):
 
         system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
 
-        is_html_webdriver = True if watch.get('fetch_backend') == 'html_webdriver' or (
-                    watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
+        is_html_webdriver = False
+        if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver':
+            is_html_webdriver = True
+
+        password_enabled_and_share_is_off = False
+        if datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False):
+            password_enabled_and_share_is_off = not datastore.data['settings']['application'].get('shared_diff_access')
 
         output = render_template("diff.html",
-                                 watch_a=watch,
-                                 newest=newest_version_file_contents,
-                                 previous=previous_version_file_contents,
-                                 extra_stylesheets=extra_stylesheets,
-                                 versions=dates[:-1], # All except current/last
-                                 uuid=uuid,
-                                 newest_version_timestamp=dates[-1],
-                                 current_previous_version=str(previous_version),
                                  current_diff_url=watch['url'],
+                                 current_previous_version=str(previous_version),
+                                 extra_stylesheets=extra_stylesheets,
                                  extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
-                                 left_sticky=True,
-                                 screenshot=screenshot_url,
+                                 extract_form=extract_form,
                                  is_html_webdriver=is_html_webdriver,
                                  last_error=watch['last_error'],
+                                 last_error_screenshot=watch.get_error_snapshot(),
                                  last_error_text=watch.get_error_text(),
-                                 last_error_screenshot=watch.get_error_snapshot()
+                                 left_sticky=True,
+                                 newest=newest_version_file_contents,
+                                 newest_version_timestamp=dates[-1],
+                                 password_enabled_and_share_is_off=password_enabled_and_share_is_off,
+                                 previous=previous_version_file_contents,
+                                 screenshot=screenshot_url,
+                                 uuid=uuid,
+                                 versions=dates[:-1], # All except current/last
+                                 watch_a=watch
                                  )
 
         return output
 
     @app.route("/preview/<string:uuid>", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def preview_page(uuid):
         content = []
         ignored_line_numbers = []
@@ -967,7 +1013,7 @@ def changedetection_app(config=None, datastore_o=None):
         return output
 
     @app.route("/settings/notification-logs", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def notification_logs():
         global notification_debug_log
         output = render_template("notification-log.html",
@@ -975,13 +1021,9 @@ def changedetection_app(config=None, datastore_o=None):
 
         return output
 
-    @app.route("/favicon.ico", methods=['GET'])
-    def favicon():
-        return send_from_directory("static/images", path="favicon.ico")
-
     # We're good but backups are even better!
     @app.route("/backup", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def get_backup():
 
         import zipfile
@@ -1101,13 +1143,14 @@ def changedetection_app(config=None, datastore_o=None):
             abort(404)
 
     @app.route("/form/add/quickwatch", methods=['POST'])
-    @login_required
+    @login_optionally_required
     def form_quick_watch_add():
         from changedetectionio import forms
         form = forms.quickWatchForm(request.form)
 
         if not form.validate():
-            flash("Error")
+            for widget, l in form.errors.items():
+                flash(','.join(l), 'error')
             return redirect(url_for('index'))
 
         url = request.form.get('url').strip()
@@ -1118,22 +1161,21 @@ def changedetection_app(config=None, datastore_o=None):
         add_paused = request.form.get('edit_and_watch_submit_button') != None
         new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip(), extras={'paused': add_paused})
 
-
-        if not add_paused and new_uuid:
-            # Straight into the queue.
-            update_q.put((1, new_uuid))
-            flash("Watch added.")
-
-        if add_paused:
-            flash('Watch added in Paused state, saving will unpause.')
-            return redirect(url_for('edit_page', uuid=new_uuid, unpause_on_save=1))
+        if new_uuid:
+            if add_paused:
+                flash('Watch added in Paused state, saving will unpause.')
+                return redirect(url_for('edit_page', uuid=new_uuid, unpause_on_save=1))
+            else:
+                # Straight into the queue.
+                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
+                flash("Watch added.")
 
         return redirect(url_for('index'))
 
 
 
     @app.route("/api/delete", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def form_delete():
         uuid = request.args.get('uuid')
 
@@ -1150,7 +1192,7 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/api/clone", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def form_clone():
         uuid = request.args.get('uuid')
         # More for testing, possible to return the first/only
@@ -1158,15 +1200,16 @@ def changedetection_app(config=None, datastore_o=None):
             uuid = list(datastore.data['watching'].keys()).pop()
 
         new_uuid = datastore.clone(uuid)
-        update_q.put((5, new_uuid))
-        flash('Cloned.')
+        if new_uuid:
+            update_q.put(queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
+            flash('Cloned.')
 
         return redirect(url_for('index'))
 
     @app.route("/api/checknow", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def form_watch_checknow():
-
+        # Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True})))
         tag = request.args.get('tag')
         uuid = request.args.get('uuid')
         i = 0
@@ -1175,11 +1218,9 @@ def changedetection_app(config=None, datastore_o=None):
         for t in running_update_threads:
             running_uuids.append(t.current_uuid)
 
-        # @todo check thread is running and skip
-
         if uuid:
             if uuid not in running_uuids:
-                update_q.put((1, uuid))
+                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
             i = 1
 
         elif tag != None:
@@ -1187,20 +1228,20 @@ def changedetection_app(config=None, datastore_o=None):
             for watch_uuid, watch in datastore.data['watching'].items():
                 if (tag != None and tag in watch['tag']):
                     if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                        update_q.put((1, watch_uuid))
+                        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
                         i += 1
 
         else:
             # No tag, no uuid, add everything.
             for watch_uuid, watch in datastore.data['watching'].items():
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                    update_q.put((1, watch_uuid))
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
                     i += 1
-        flash("{} watches are queued for rechecking.".format(i))
+        flash("{} watches queued for rechecking.".format(i))
         return redirect(url_for('index', tag=tag))
 
     @app.route("/form/checkbox-operations", methods=['POST'])
-    @login_required
+    @login_optionally_required
     def form_watch_list_checkbox_operations():
         op = request.form['op']
         uuids = request.form.getlist('uuids')
@@ -1217,7 +1258,6 @@ def changedetection_app(config=None, datastore_o=None):
                 uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
                     datastore.data['watching'][uuid.strip()]['paused'] = True
-
             flash("{} watches paused".format(len(uuids)))
 
         elif (op == 'unpause'):
@@ -1241,6 +1281,14 @@ def changedetection_app(config=None, datastore_o=None):
                     datastore.data['watching'][uuid.strip()]['notification_muted'] = False
             flash("{} watches un-muted".format(len(uuids)))
 
+        elif (op == 'recheck'):
+            for uuid in uuids:
+                uuid = uuid.strip()
+                if datastore.data['watching'].get(uuid):
+                    # Recheck and require a full reprocessing
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
+            flash("{} watches queued for rechecking".format(len(uuids)))
+
         elif (op == 'notification-default'):
             from changedetectionio.notification import (
                 default_notification_format_for_watch
@@ -1257,7 +1305,7 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/api/share-url", methods=['GET'])
-    @login_required
+    @login_optionally_required
     def form_share_put_watch():
         """Given a watch UUID, upload the info and return a share-link
            the share-link can be imported/added"""
@@ -1313,6 +1361,10 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.blueprint.browser_steps as browser_steps
     app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
 
+    import changedetectionio.blueprint.price_data_follower as price_data_follower
+    app.register_blueprint(price_data_follower.construct_blueprint(datastore, update_q), url_prefix='/price_data_follower')
+
+
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
     threading.Thread(target=notification_runner).start()
@@ -1418,7 +1470,11 @@ def ticker_thread_check_time_launch_checks():
         watch_uuid_list = []
         while True:
             try:
-                watch_uuid_list = datastore.data['watching'].keys()
+                # Get a list of watches sorted by last_checked, [1] because it gets passed a tuple
+                # This is so we examine the most over-due first
+                for k in sorted(datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked',0)):
+                    watch_uuid_list.append(k[0])
+
             except RuntimeError as e:
                 # RuntimeError: dictionary changed size during iteration
                 time.sleep(0.1)
@@ -1458,7 +1514,7 @@ def ticker_thread_check_time_launch_checks():
             seconds_since_last_recheck = now - watch['last_checked']
 
             if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds:
-                if not uuid in running_uuids and uuid not in [q_uuid for p,q_uuid in update_q.queue]:
+                if not uuid in running_uuids and uuid not in [q_uuid.item['uuid'] for q_uuid in update_q.queue]:
 
                     # Proxies can be set to have a limit on seconds between which they can be called
                     watch_proxy = datastore.get_preferred_proxy_for_watch(uuid=uuid)
@@ -1489,8 +1545,9 @@ def ticker_thread_check_time_launch_checks():
                             priority,
                             watch.jitter_seconds,
                             now - watch['last_checked']))
+
                     # Into the queue with you
-                    update_q.put((priority, uuid))
+                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=priority, item={'uuid': uuid, 'skip_when_checksum_same': True}))
 
                     # Reset for next time
                     watch.jitter_seconds = 0

changedetectionio/api/api_schema.py

@@ -0,0 +1,117 @@
+# Responsible for building the storage dict into a set of rules ("JSON Schema") acceptable via the API
+# Probably other ways to solve this when the backend switches to some ORM
+
+def build_time_between_check_json_schema():
+    # Setup time between check schema
+    schema_properties_time_between_check = {
+        "type": "object",
+        "additionalProperties": False,
+        "properties": {}
+    }
+    for p in ['weeks', 'days', 'hours', 'minutes', 'seconds']:
+        schema_properties_time_between_check['properties'][p] = {
+            "anyOf": [
+                {
+                    "type": "integer"
+                },
+                {
+                    "type": "null"
+                }
+            ]
+        }
+
+    return schema_properties_time_between_check
+
+def build_watch_json_schema(d):
+    # Base JSON schema
+    schema = {
+        'type': 'object',
+        'properties': {},
+    }
+
+    for k, v in d.items():
+        # @todo 'integer' is not covered here because its almost always for internal usage
+
+        if isinstance(v, type(None)):
+            schema['properties'][k] = {
+                "anyOf": [
+                    {"type": "null"},
+                ]
+            }
+        elif isinstance(v, list):
+            schema['properties'][k] = {
+                "anyOf": [
+                    {"type": "array",
+                     # Always is an array of strings, like text or regex or something
+                     "items": {
+                         "type": "string",
+                         "maxLength": 5000
+                     }
+                     },
+                ]
+            }
+        elif isinstance(v, bool):
+            schema['properties'][k] = {
+                "anyOf": [
+                    {"type": "boolean"},
+                ]
+            }
+        elif isinstance(v, str):
+            schema['properties'][k] = {
+                "anyOf": [
+                    {"type": "string",
+                     "maxLength": 5000},
+                ]
+            }
+
+    # Can also be a string (or None by default above)
+    for v in ['body',
+              'notification_body',
+              'notification_format',
+              'notification_title',
+              'proxy',
+              'tag',
+              'title',
+              'webdriver_js_execute_code'
+              ]:
+        schema['properties'][v]['anyOf'].append({'type': 'string', "maxLength": 5000})
+
+    # None or Boolean
+    schema['properties']['track_ldjson_price_data']['anyOf'].append({'type': 'boolean'})
+
+    schema['properties']['method'] = {"type": "string",
+                                      "enum": ["GET", "POST", "DELETE", "PUT"]
+                                      }
+
+    schema['properties']['fetch_backend']['anyOf'].append({"type": "string",
+                                                           "enum": ["html_requests", "html_webdriver"]
+                                                           })
+
+
+
+    # All headers must be key/value type dict
+    schema['properties']['headers'] = {
+        "type": "object",
+        "patternProperties": {
+            # Should always be a string:string type value
+            ".*": {"type": "string"},
+        }
+    }
+
+    from changedetectionio.notification import valid_notification_formats
+
+    schema['properties']['notification_format'] = {'type': 'string',
+                                                   'enum': list(valid_notification_formats.keys())
+                                                   }
+
+    # Stuff that shouldn't be available but is just state-storage
+    for v in ['previous_md5', 'last_error', 'has_ldjson_price_data', 'previous_md5_before_filters', 'uuid']:
+        del schema['properties'][v]
+
+    schema['properties']['webdriver_delay']['anyOf'].append({'type': 'integer'})
+
+    schema['properties']['time_between_check'] = build_time_between_check_json_schema()
+
+    # headers ?
+    return schema
+

changedetectionio/api/api_v1.py

@@ -1,11 +1,24 @@
+from flask_expects_json import expects_json
+from changedetectionio import queuedWatchMetaData
 from flask_restful import abort, Resource
 from flask import request, make_response
 import validators
 from . import auth
+import copy
 
+# See docs/README.md for rebuilding the docs/apidoc information
 
+from . import api_schema
 
-# https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+# Build a JSON Schema atleast partially based on our Watch model
+from changedetectionio.model.Watch import base_config as watch_base_config
+schema = api_schema.build_watch_json_schema(watch_base_config)
+
+schema_create_watch = copy.deepcopy(schema)
+schema_create_watch['required'] = ['url']
+
+schema_update_watch = copy.deepcopy(schema)
+schema_update_watch['additionalProperties'] = False
 
 class Watch(Resource):
     def __init__(self, **kwargs):
@@ -15,16 +28,45 @@ class Watch(Resource):
 
     # Get information about a single watch, excluding the history list (can be large)
     # curl http://localhost:4000/api/v1/watch/<string:uuid>
+    # @todo - version2 - ?muted and ?paused should be able to be called together, return the watch struct not "OK"
     # ?recheck=true
     @auth.check_token
     def get(self, uuid):
+        """
+        @api {get} /api/v1/watch/:uuid Single watch information
+        @apiDescription Retrieve watch information and set muted/paused status
+        @apiExample {curl} Example usage:
+            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=unmuted"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?paused=unpaused"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+        @apiName Watch
+        @apiGroup Watch
+        @apiParam {uuid} uuid Watch unique ID.
+        @apiQuery {Boolean} [recheck] Recheck this watch `recheck=1`
+        @apiQuery {String} [paused] =`paused` or =`unpaused` , Sets the PAUSED state
+        @apiQuery {String} [muted] =`muted` or =`unmuted` , Sets the MUTE NOTIFICATIONS state
+        @apiSuccess (200) {String} OK When paused/muted/recheck operation OR full JSON object of the watch
+        @apiSuccess (200) {JSON} WatchJSON JSON Full JSON object of the watch
+        """
         from copy import deepcopy
         watch = deepcopy(self.datastore.data['watching'].get(uuid))
         if not watch:
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
 
         if request.args.get('recheck'):
-            self.update_q.put((1, uuid))
+            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
+            return "OK", 200
+        if request.args.get('paused', '') == 'paused':
+            self.datastore.data['watching'].get(uuid).pause()
+            return "OK", 200
+        elif request.args.get('paused', '') == 'unpaused':
+            self.datastore.data['watching'].get(uuid).unpause()
+            return "OK", 200
+        if request.args.get('muted', '') == 'muted':
+            self.datastore.data['watching'].get(uuid).mute()
+            return "OK", 200
+        elif request.args.get('muted', '') == 'unmuted':
+            self.datastore.data['watching'].get(uuid).unmute()
             return "OK", 200
 
         # Return without history, get that via another API call
@@ -33,12 +75,53 @@ class Watch(Resource):
 
     @auth.check_token
     def delete(self, uuid):
+        """
+        @api {delete} /api/v1/watch/:uuid Delete watch information
+        @apiExample {curl} Example usage:
+            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+        @apiParam {uuid} uuid Watch unique ID.
+        @apiName Delete
+        @apiGroup Watch
+        @apiSuccess (200) {String} OK Was deleted
+        """
         if not self.datastore.data['watching'].get(uuid):
             abort(400, message='No watch exists with the UUID of {}'.format(uuid))
 
         self.datastore.delete(uuid)
         return 'OK', 204
 
+    # Update an existing
+    @auth.check_token
+    @expects_json(schema_update_watch)
+    def put(self, uuid):
+        """
+        @api {put} /api/v1/watch/:uuid Update watch information
+        @apiExample {curl} Example usage:
+            Create a watch (POST)
+            curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
+            Update (PUT)
+            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}'
+
+        @apiDescription Updates an existing watch using JSON, accepts the same structure as at https://github.com/dgtlmoon/changedetection.io/blob/fab7d325f764d6912bef671f1d78bf217689c537/changedetectionio/model/Watch.py#L15
+        @apiParam {uuid} uuid Watch unique ID.
+        @apiName Update
+        @apiGroup Watch
+        @apiSuccess (200) {String} OK Was updated
+        @apiSuccess (500) {String} ERR Some other error
+        """
+        watch = self.datastore.data['watching'].get(uuid)
+        if not watch:
+            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
+
+        if request.json.get('proxy'):
+            plist = self.datastore.proxy_list
+            if not request.json.get('proxy') in plist:
+                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
+
+        watch.update(request.json)
+
+        return "OK", 200
+
 
 class WatchHistory(Resource):
     def __init__(self, **kwargs):
@@ -89,36 +172,74 @@ class CreateWatch(Resource):
         self.update_q = kwargs['update_q']
 
     @auth.check_token
+    @expects_json(schema_create_watch)
     def post(self):
-        # curl http://localhost:4000/api/v1/watch -H "Content-Type: application/json" -d '{"url": "https://my-nice.com", "tag": "one, two" }'
+        """
+        @api {post} /api/v1/watch Create a watch
+        @apiDescription requires `url`, Creates a watch, also accepts accepts the same structure as at https://github.com/dgtlmoon/changedetection.io/blob/fab7d325f764d6912bef671f1d78bf217689c537/changedetectionio/model/Watch.py#L15
+        @apiExample {curl} Example usage:
+            curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
+        @apiName Create
+        @apiGroup CreateWatch
+        @apiSuccess (200) {String} OK Was created
+        @apiSuccess (500) {String} ERR Some other error
+        """
+
+        #
         json_data = request.get_json()
-        tag = json_data['tag'].strip() if json_data.get('tag') else ''
+        url = json_data['url'].strip()
+
 
         if not validators.url(json_data['url'].strip()):
             return "Invalid or unsupported URL", 400
 
-        extras = {'title': json_data['title'].strip()} if json_data.get('title') else {}
+        if json_data.get('proxy'):
+            plist = self.datastore.proxy_list
+            if not json_data.get('proxy') in plist:
+                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
 
-        new_uuid = self.datastore.add_watch(url=json_data['url'].strip(), tag=tag, extras=extras)
-        self.update_q.put((1, new_uuid))
-        return {'uuid': new_uuid}, 201
+        extras = copy.deepcopy(json_data)
+        del extras['url']
+
+        new_uuid = self.datastore.add_watch(url=url, extras=extras)
+        if new_uuid:
+            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
+            return {'uuid': new_uuid}, 201
+        else:
+            return "Invalid or unsupported URL", 400
 
-    # Return concise list of available watches and some very basic info
-    # curl http://localhost:4000/api/v1/watch|python -mjson.tool
-    # ?recheck_all=1 to recheck all
     @auth.check_token
     def get(self):
+        """
+        @api {get} /api/v1/watch
+        @apiDescription Return concise list of available watches and some very basic info
+        @apiExample {curl} Example usage:
+            curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            recheck_all=1 to recheck all
+        @apiParam {String} [recheck_all]       Optional Set to =1 to force recheck of all watches
+        @apiParam {String} [tag]               Optional name of tag to limit results
+        @apiName ListWatches
+        @apiGroup CreateWatch
+
+        :return:
+        """
         list = {}
-        for k, v in self.datastore.data['watching'].items():
-            list[k] = {'url': v['url'],
-                       'title': v['title'],
-                       'last_checked': v['last_checked'],
-                       'last_changed': v.last_changed,
-                       'last_error': v['last_error']}
+
+        tag_limit = request.args.get('tag', None)
+        for k, watch in self.datastore.data['watching'].items():
+            if tag_limit:
+                if not tag_limit.lower() in watch.all_tags:
+                    continue
+
+            list[k] = {'url': watch['url'],
+                       'title': watch['title'],
+                       'last_checked': watch['last_checked'],
+                       'last_changed': watch.last_changed,
+                       'last_error': watch['last_error']}
 
         if request.args.get('recheck_all'):
             for uuid in self.datastore.data['watching'].keys():
-                self.update_q.put((1, uuid))
+                self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
             return {'status': "OK"}, 200
 
         return list, 200

changedetectionio/blueprint/browser_steps/__init__.py

@@ -21,57 +21,50 @@
 # OR
 # - use multiprocessing to bump this over to its own process and add some transport layer (queue/pipes)
 
-
-
-
 from distutils.util import strtobool
 from flask import Blueprint, request, make_response
-from flask_login import login_required
 import os
 import logging
 from changedetectionio.store import ChangeDetectionStore
-
+from changedetectionio import login_optionally_required
 browsersteps_live_ui_o = {}
 browsersteps_playwright_browser_interface = None
-browsersteps_playwright_browser_interface_start_time = None
 browsersteps_playwright_browser_interface_browser = None
+browsersteps_playwright_browser_interface_context = None
 browsersteps_playwright_browser_interface_end_time = None
-
+browsersteps_playwright_browser_interface_start_time = None
 
 def cleanup_playwright_session():
-    print("Cleaning up old playwright session because time was up")
-    global browsersteps_playwright_browser_interface
+
     global browsersteps_live_ui_o
-    global browsersteps_playwright_browser_interface_browser
     global browsersteps_playwright_browser_interface
-    global browsersteps_playwright_browser_interface_start_time
+    global browsersteps_playwright_browser_interface_browser
+    global browsersteps_playwright_browser_interface_context
     global browsersteps_playwright_browser_interface_end_time
-
-    import psutil
-
-    current_process = psutil.Process()
-    children = current_process.children(recursive=True)
-    for child in children:
-        print (child)
-        print('Child pid is {}'.format(child.pid))
-
-    # .stop() hangs sometimes if its called when there are no children to process
-    # but how do we know this is our child? dunno
-    if children:
-        browsersteps_playwright_browser_interface.stop()
+    global browsersteps_playwright_browser_interface_start_time
 
     browsersteps_live_ui_o = {}
     browsersteps_playwright_browser_interface = None
-    browsersteps_playwright_browser_interface_start_time = None
     browsersteps_playwright_browser_interface_browser = None
     browsersteps_playwright_browser_interface_end_time = None
+    browsersteps_playwright_browser_interface_start_time = None
+
+    print("Cleaning up old playwright session because time was up, calling .goodbye()")
+    try:
+        browsersteps_playwright_browser_interface_context.goodbye()
+    except Exception as e:
+        print ("Got exception in shutdown, probably OK")
+        print (str(e))
+
+    browsersteps_playwright_browser_interface_context = None
+
     print ("Cleaning up old playwright session because time was up - done")
 
 def construct_blueprint(datastore: ChangeDetectionStore):
 
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
 
-    @login_required
+    @login_optionally_required
     @browser_steps_blueprint.route("/browsersteps_update", methods=['GET', 'POST'])
     def browsersteps_ui_update():
         import base64
@@ -99,12 +92,8 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if browsersteps_playwright_browser_interface_end_time:
             remaining = browsersteps_playwright_browser_interface_end_time-time.time()
             if browsersteps_playwright_browser_interface_end_time and remaining <= 0:
-
-
                 cleanup_playwright_session()
-
-                return make_response('Browser session expired, please reload the Browser Steps interface', 500)
-
+                return make_response('Browser session expired, please reload the Browser Steps interface', 401)
 
         # Actions - step/apply/etc, do the thing and return state
         if request.method == 'POST':
@@ -131,21 +120,11 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 this_session.call_action(action_name=step_operation,
                                          selector=step_selector,
                                          optional_value=step_optional_value)
-            except playwright._impl._api_types.TimeoutError as e:
-                print("Element wasnt found :-(", step_operation)
-                return make_response("Element was not found on page", 401)
-
-            except playwright._impl._api_types.Error as e:
-                # Browser/playwright level error
-                print("Browser error - got playwright._impl._api_types.Error, try reloading the session/browser")
-                print (str(e))
 
+            except Exception as e:
+                print("Exception when calling step operation", step_operation, str(e))
                 # Try to find something of value to give back to the user
-                for l in str(e).splitlines():
-                    if 'DOMException' in l:
-                        return make_response(l, 401)
-
-                return make_response('Browser session ran out of time :( Please reload this page.', 401)
+                return make_response(str(e).splitlines()[0], 401)
 
             # Get visual selector ready/update its data (also use the current filter info from the page?)
             # When the last 'apply' button was pressed
@@ -162,10 +141,11 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             if not browsersteps_playwright_browser_interface:
                 print("Starting connection with playwright")
                 logging.debug("browser_steps.py connecting")
-                from playwright.sync_api import sync_playwright
-
-                browsersteps_playwright_browser_interface = sync_playwright().start()
 
+                global browsersteps_playwright_browser_interface_context
+                from . import nonContext
+                browsersteps_playwright_browser_interface_context = nonContext.c_sync_playwright()
+                browsersteps_playwright_browser_interface = browsersteps_playwright_browser_interface_context.start()
 
                 time.sleep(1)
                 # At 20 minutes, some other variable is closing it
@@ -205,21 +185,45 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             cleanup_playwright_session()
             return make_response('Browser session ran out of time :( Please reload this page.', 401)
 
-        try:
-            state = this_session.get_current_state()
-        except playwright._impl._api_types.Error as e:
-            return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
+        response = None
 
-        p = {'screenshot': "data:image/png;base64,{}".format(
-            base64.b64encode(state[0]).decode('ascii')),
-            'xpath_data': state[1],
-            'session_age_start': this_session.age_start,
-            'browser_time_remaining': round(remaining)
-        }
+        if request.method == 'POST':
+            # Screenshots and other info only needed on requesting a step (POST)
+            try:
+                state = this_session.get_current_state()
+            except playwright._impl._api_types.Error as e:
+                return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
 
+            # Use send_file() which is way faster than read/write loop on bytes
+            import json
+            from tempfile import mkstemp
+            from flask import send_file
+            tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
 
-        # @todo BSON/binary JSON, faster xfer, OR pick it off the disk
-        return p
+            output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
+                base64.b64encode(state[0]).decode('ascii')),
+                'xpath_data': state[1],
+                'session_age_start': this_session.age_start,
+                'browser_time_remaining': round(remaining)
+            })
+
+            with os.fdopen(tmp_fd, 'w') as f:
+                f.write(output)
+
+            response = make_response(send_file(path_or_file=tmp_file,
+                                               mimetype='application/json; charset=UTF-8',
+                                               etag=True))
+            # No longer needed
+            os.unlink(tmp_file)
+
+        elif request.method == 'GET':
+            # Just enough to get the session rolling, it will call for goto-site via POST next
+            response = make_response({
+                'session_age_start': this_session.age_start,
+                'browser_time_remaining': round(remaining)
+            })
+
+        return response
 
     return browser_steps_blueprint
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -22,6 +22,7 @@ browser_step_ui_config = {'Choose one': '0 0',
                           'Click element': '1 0',
                           'Click element containing text': '0 1',
                           'Enter text in field': '1 1',
+                          'Execute JS': '0 1',
 #                          'Extract text and use as filter': '1 0',
                           'Goto site': '0 0',
                           'Press Enter': '0 0',
@@ -74,22 +75,20 @@ class steppable_browser_interface():
     def action_goto_url(self, url, optional_value):
         # self.page.set_viewport_size({"width": 1280, "height": 5000})
         now = time.time()
-        response = self.page.goto(url, timeout=0, wait_until='domcontentloaded')
-        print("Time to goto URL", time.time() - now)
+        response = self.page.goto(url, timeout=0, wait_until='commit')
 
         # Wait_until = commit
         # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
         # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
         # This seemed to solve nearly all 'TimeoutErrors'
-        extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))
-        self.page.wait_for_timeout(extra_wait * 1000)
+        print("Time to goto URL ", time.time() - now)
 
     def action_click_element_containing_text(self, selector=None, value=''):
         if not len(value.strip()):
             return
         elem = self.page.get_by_text(value)
         if elem.count():
-            elem.first.click(delay=randint(200, 500))
+            elem.first.click(delay=randint(200, 500), timeout=3000)
 
     def action_enter_text_in_field(self, selector, value):
         if not len(selector.strip()):
@@ -97,6 +96,9 @@ class steppable_browser_interface():
 
         self.page.fill(selector, value, timeout=10 * 1000)
 
+    def action_execute_js(self, selector, value):
+        self.page.evaluate(value)
+
     def action_click_element(self, selector, value):
         print("Clicking element")
         if not len(selector.strip()):
@@ -142,10 +144,10 @@ class steppable_browser_interface():
         self.page.keyboard.press("PageDown", delay=randint(200, 500))
 
     def action_check_checkbox(self, selector, value):
-        self.page.locator(selector).check()
+        self.page.locator(selector).check(timeout=1000)
 
     def action_uncheck_checkbox(self, selector, value):
-        self.page.locator(selector).uncheck()
+        self.page.locator(selector, timeout=1000).uncheck(timeout=1000)
 
 
 # Responsible for maintaining a live 'context' with browserless
@@ -207,7 +209,7 @@ class browsersteps_live_ui(steppable_browser_interface):
         # Listen for all console events and handle errors
         self.page.on("console", lambda msg: print(f"Browser steps console - {msg.type}: {msg.text} {msg.args}"))
 
-        print("time to browser setup", time.time() - now)
+        print("Time to browser setup", time.time() - now)
         self.page.wait_for_timeout(1 * 1000)
 
     def mark_as_closed(self):
@@ -232,7 +234,7 @@ class browsersteps_live_ui(steppable_browser_interface):
         self.page.evaluate("var include_filters=''")
         # Go find the interactive elements
         # @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers?
-        elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4'
+        elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
         xpath_element_js = xpath_element_js.replace('%ELEMENTS%', elements)
         xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
         # So the JS will find the smallest one first

changedetectionio/blueprint/browser_steps/nonContext.py

@@ -0,0 +1,18 @@
+from playwright.sync_api import PlaywrightContextManager
+import asyncio
+
+# So playwright wants to run as a context manager, but we do something horrible and hacky
+# we are holding the session open for as long as possible, then shutting it down, and opening a new one
+# So it means we don't get to use PlaywrightContextManager' __enter__ __exit__
+# To work around this, make goodbye() act the same as the __exit__()
+#
+# But actually I think this is because the context is opened correctly with __enter__() but we timeout the connection
+# then theres some lock condition where we cant destroy it without it hanging
+
+class c_PlaywrightContextManager(PlaywrightContextManager):
+
+    def goodbye(self) -> None:
+        self.__exit__()
+
+def c_sync_playwright() -> PlaywrightContextManager:
+    return c_PlaywrightContextManager()

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -0,0 +1,33 @@
+
+from distutils.util import strtobool
+from flask import Blueprint, flash, redirect, url_for
+from flask_login import login_required
+from changedetectionio.store import ChangeDetectionStore
+from changedetectionio import queuedWatchMetaData
+from queue import PriorityQueue
+
+PRICE_DATA_TRACK_ACCEPT = 'accepted'
+PRICE_DATA_TRACK_REJECT = 'rejected'
+
+def construct_blueprint(datastore: ChangeDetectionStore, update_q: PriorityQueue):
+
+    price_data_follower_blueprint = Blueprint('price_data_follower', __name__)
+
+    @login_required
+    @price_data_follower_blueprint.route("/<string:uuid>/accept", methods=['GET'])
+    def accept(uuid):
+        datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_ACCEPT
+        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
+        return redirect(url_for("form_watch_checknow", uuid=uuid))
+
+
+    @login_required
+    @price_data_follower_blueprint.route("/<string:uuid>/reject", methods=['GET'])
+    def reject(uuid):
+        datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_REJECT
+        return redirect(url_for("index"))
+
+
+    return price_data_follower_blueprint
+
+

changedetectionio/changedetection.py

@@ -3,11 +3,14 @@
 # Launch as a eventlet.wsgi server instance.
 
 from distutils.util import strtobool
+from json.decoder import JSONDecodeError
+
 import eventlet
 import eventlet.wsgi
 import getopt
 import os
 import signal
+import socket
 import sys
 
 from . import store, changedetection_app, content_fetcher
@@ -28,11 +31,13 @@ def sigterm_handler(_signo, _stack_frame):
 def main():
     global datastore
     global app
-    ssl_mode = False
-    host = ''
-    port = os.environ.get('PORT') or 5000
-    do_cleanup = False
+
     datastore_path = None
+    do_cleanup = False
+    host = ''
+    ipv6_enabled = False
+    port = os.environ.get('PORT') or 5000
+    ssl_mode = False
 
     # On Windows, create and use a default path.
     if os.name == 'nt':
@@ -43,7 +48,7 @@ def main():
         datastore_path = os.path.join(os.getcwd(), "../datastore")
 
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "Ccsd:h:p:", "port")
+        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:", "port")
     except getopt.GetoptError:
         print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
         sys.exit(2)
@@ -63,6 +68,10 @@ def main():
         if opt == '-d':
             datastore_path = arg
 
+        if opt == '-6':
+            print ("Enabling IPv6 listen support")
+            ipv6_enabled = True
+
         # Cleanup (remove text files that arent in the index)
         if opt == '-c':
             do_cleanup = True
@@ -83,8 +92,14 @@ def main():
                 "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
             sys.exit(2)
 
+    try:
+        datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
+    except JSONDecodeError as e:
+        # Dont' start if the JSON DB looks corrupt
+        print ("ERROR: JSON DB or Proxy List JSON at '{}' appears to be corrupt, aborting".format(app_config['datastore_path']))
+        print(str(e))
+        return
 
-    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
     app = changedetection_app(app_config, datastore)
 
     signal.signal(signal.SIGTERM, sigterm_handler)
@@ -124,13 +139,15 @@ def main():
         from werkzeug.middleware.proxy_fix import ProxyFix
         app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
 
+    s_type = socket.AF_INET6 if ipv6_enabled else socket.AF_INET
+
     if ssl_mode:
         # @todo finalise SSL config, but this should get you in the right direction if you need it.
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port)),
+        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), s_type),
                                                certfile='cert.pem',
                                                keyfile='privkey.pem',
                                                server_side=True), app)
 
     else:
-        eventlet.wsgi.server(eventlet.listen((host, int(port))), app)
+        eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app)
 

changedetectionio/content_fetcher.py

@@ -1,3 +1,4 @@
+import hashlib
 from abc import abstractmethod
 import chardet
 import json
@@ -23,6 +24,9 @@ class Non200ErrorCodeReceived(Exception):
             self.page_text = html_tools.html_to_text(page_html)
         return
 
+class checksumFromPreviousCheckWasTheSame(Exception):
+    def __init__(self):
+        return
 
 class JSActionExceptions(Exception):
     def __init__(self, status_code, url, screenshot, message=''):
@@ -39,7 +43,7 @@ class BrowserStepsStepTimout(Exception):
 
 
 class PageUnloadable(Exception):
-    def __init__(self, status_code, url, screenshot=False, message=False):
+    def __init__(self, status_code, url, message, screenshot=False):
         # Set this so we can use it in other parts of the app
         self.status_code = status_code
         self.url = url
@@ -113,7 +117,8 @@ class Fetcher():
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None):
+            current_include_filters=None,
+            is_binary=False):
         # Should set self.error, self.status_code and self.content
         pass
 
@@ -238,10 +243,15 @@ class base_html_playwright(Fetcher):
         if proxy_override:
             self.proxy = {'server': proxy_override}
 
-    def screenshot_step(self, step_n=''):
+        if self.proxy:
+            # Playwright needs separate username and password values
+            from urllib.parse import urlparse
+            parsed = urlparse(self.proxy.get('server'))
+            if parsed.username:
+                self.proxy['username'] = parsed.username
+                self.proxy['password'] = parsed.password
 
-        # There's a bug where we need to do it twice or it doesnt take the whole page, dont know why.
-        self.page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024})
+    def screenshot_step(self, step_n=''):
         screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=85)
 
         if self.browser_steps_screenshot_path is not None:
@@ -264,7 +274,8 @@ class base_html_playwright(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None):
+            current_include_filters=None,
+            is_binary=False):
 
         from playwright.sync_api import sync_playwright
         import playwright._impl._api_types
@@ -286,6 +297,8 @@ class base_html_playwright(Fetcher):
                 proxy=self.proxy,
                 # This is needed to enable JavaScript execution on GitHub and others
                 bypass_csp=True,
+                # Should be `allow` or `block` - sites like YouTube can transmit large amounts of data via Service Workers
+                service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'),
                 # Should never be needed
                 accept_downloads=False
             )
@@ -294,24 +307,34 @@ class base_html_playwright(Fetcher):
             if len(request_headers):
                 context.set_extra_http_headers(request_headers)
 
-            try:
                 self.page.set_default_navigation_timeout(90000)
                 self.page.set_default_timeout(90000)
 
                 # Listen for all console events and handle errors
                 self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
 
-                # Bug - never set viewport size BEFORE page.goto
-
-
-                # Waits for the next navigation. Using Python context manager
-                # prevents a race condition between clicking and waiting for a navigation.
-                with self.page.expect_navigation():
-                    response = self.page.goto(url, wait_until='load')
+            # Goto page
+            try:
                 # Wait_until = commit
                 # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
                 # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
                 # This seemed to solve nearly all 'TimeoutErrors'
+                response = self.page.goto(url, wait_until='commit')
+            except playwright._impl._api_types.Error as e:
+                # Retry once - https://github.com/browserless/chrome/issues/2485
+                # Sometimes errors related to invalid cert's and other can be random
+                print ("Content Fetcher > retrying request got error - ", str(e))
+                time.sleep(1)
+                response = self.page.goto(url, wait_until='commit')
+
+            except Exception as e:
+                print ("Content Fetcher > Other exception when page.goto", str(e))
+                context.close()
+                browser.close()
+                raise PageUnloadable(url=url, status_code=None, message=str(e))
+
+            # Execute any browser steps
+            try:
                 extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
                 self.page.wait_for_timeout(extra_wait * 1000)
 
@@ -324,40 +347,30 @@ class base_html_playwright(Fetcher):
                 # This can be ok, we will try to grab what we could retrieve
                 pass
             except Exception as e:
-                print ("other exception when page.goto")
-                print (str(e))
+                print ("Content Fetcher > Other exception when executing custom JS code", str(e))
                 context.close()
                 browser.close()
-                raise PageUnloadable(url=url, status_code=None)
-
+                raise PageUnloadable(url=url, status_code=None, message=str(e))
 
             if response is None:
                 context.close()
                 browser.close()
-                print ("response object was none")
+                print ("Content Fetcher > Response object was none")
                 raise EmptyReply(url=url, status_code=None)
 
-            # Bug 2(?) Set the viewport size AFTER loading the page
-            self.page.set_viewport_size({"width": 1280, "height": 1024})
-
             # Run Browser Steps here
             self.iterate_browser_steps()
 
             extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
             time.sleep(extra_wait)
 
-
             self.content = self.page.content()
             self.status_code = response.status
-
             if len(self.page.content().strip()) == 0:
                 context.close()
                 browser.close()
-                print ("Content was empty")
-                raise EmptyReply(url=url, status_code=None)
-
-            # Bug 2(?) Set the viewport size AFTER loading the page
-            self.page.set_viewport_size({"width": 1280, "height": 1024})
+                print ("Content Fetcher > Content was empty")
+                raise EmptyReply(url=url, status_code=response.status)
 
             self.status_code = response.status
             self.content = self.page.content()
@@ -379,8 +392,6 @@ class base_html_playwright(Fetcher):
             # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
             # acceptable screenshot quality here
             try:
-                # Quality set to 1 because it's not used, just used as a work-around for a bug, no need to change this.
-                self.page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024}, quality=1)
                 # The actual screenshot
                 self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
             except Exception as e:
@@ -440,7 +451,8 @@ class base_html_webdriver(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None):
+            current_include_filters=None,
+            is_binary=False):
 
         from selenium import webdriver
         from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
@@ -498,7 +510,7 @@ class base_html_webdriver(Fetcher):
             try:
                 self.driver.quit()
             except Exception as e:
-                print("Exception in chrome shutdown/quit" + str(e))
+                print("Content Fetcher > Exception in chrome shutdown/quit" + str(e))
 
 
 # "html_requests" is listed as the default fetcher in store.py!
@@ -515,7 +527,8 @@ class html_requests(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None):
+            current_include_filters=None,
+            is_binary=False):
 
         # Make requests use a more modern looking user-agent
         if not 'User-Agent' in request_headers:
@@ -545,10 +558,12 @@ class html_requests(Fetcher):
         # For example - some sites don't tell us it's utf-8, but return utf-8 content
         # This seems to not occur when using webdriver/selenium, it seems to detect the text encoding more reliably.
         # https://github.com/psf/requests/issues/1604 good info about requests encoding detection
-        if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
-            encoding = chardet.detect(r.content)['encoding']
-            if encoding:
-                r.encoding = encoding
+        if not is_binary:
+            # Don't run this for PDF (and requests identified as binary) takes a _long_ time
+            if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
+                encoding = chardet.detect(r.content)['encoding']
+                if encoding:
+                    r.encoding = encoding
 
         if not r.content or not len(r.content):
             raise EmptyReply(url=url, status_code=r.status_code)
@@ -560,8 +575,14 @@ class html_requests(Fetcher):
             raise Non200ErrorCodeReceived(url=url, status_code=r.status_code, page_html=r.text)
 
         self.status_code = r.status_code
-        self.content = r.text
+        if is_binary:
+            # Binary files just return their checksum until we add something smarter
+            self.content = hashlib.md5(r.content).hexdigest()
+        else:
+            self.content = r.text
+
         self.headers = r.headers
+        self.raw_content = r.content
 
 
 # Decide which is the 'real' HTML webdriver, this is more a system wide config

changedetectionio/dev-docker/Dockerfile

@@ -1,14 +0,0 @@
-FROM python:3.8-slim
-
-# https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
-ENV PYTHONUNBUFFERED=1
-
-WORKDIR /app
-
-RUN [ ! -d "/datastore" ] && mkdir /datastore
-
-COPY sleep.py /
-CMD [ "python", "/sleep.py" ]
-
-
-

changedetectionio/dev-docker/sleep.py

@@ -1,7 +0,0 @@
-import time
-
-print ("Sleep loop, you should run your script from the console")
-
-while True: 
-    # Wait for 5 seconds
-    time.sleep(2)

changedetectionio/fetch_site_status.py

@@ -1,11 +1,13 @@
 import hashlib
+import json
 import logging
 import os
 import re
-import time
 import urllib3
 
 from changedetectionio import content_fetcher, html_tools
+from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
+from copy import deepcopy
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -14,6 +16,10 @@ class FilterNotFoundInResponse(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
 
+class PDFToHTMLToolNotFound(ValueError):
+    def __init__(self, msg):
+        ValueError.__init__(self, msg)
+
 
 # Some common stuff here that can be moved to a base class
 # (set_proxy_from_list)
@@ -38,8 +44,7 @@ class perform_site_check():
 
         return regex
 
-    def run(self, uuid):
-        from copy import deepcopy
+    def run(self, uuid, skip_when_checksum_same=True):
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
@@ -86,7 +91,10 @@ class perform_site_check():
             is_source = True
 
         # Pluggable content fetcher
-        prefer_backend = watch.get('fetch_backend')
+        prefer_backend = watch.get_fetch_backend
+        if not prefer_backend or prefer_backend == 'system':
+            prefer_backend = self.datastore.data['settings']['application']['fetch_backend']
+
         if hasattr(content_fetcher, prefer_backend):
             klass = getattr(content_fetcher, prefer_backend)
         else:
@@ -116,12 +124,26 @@ class perform_site_check():
         if watch.get('webdriver_js_execute_code') is not None and watch.get('webdriver_js_execute_code').strip():
             fetcher.webdriver_js_execute_code = watch.get('webdriver_js_execute_code')
 
-        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'))
+        # requests for PDF's, images etc should be passwd the is_binary flag
+        is_binary = watch.is_pdf
+
+        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'), is_binary=is_binary)
         fetcher.quit()
 
         self.screenshot = fetcher.screenshot
         self.xpath_data = fetcher.xpath_data
 
+        # Track the content type
+        update_obj['content_type'] = fetcher.headers.get('Content-Type', '')
+
+        # Watches added automatically in the queue manager will skip if its the same checksum as the previous run
+        # Saves a lot of CPU
+        update_obj['previous_md5_before_filters'] = hashlib.md5(fetcher.content.encode('utf-8')).hexdigest()
+        if skip_when_checksum_same:
+            if update_obj['previous_md5_before_filters'] == watch.get('previous_md5_before_filters'):
+                raise content_fetcher.checksumFromPreviousCheckWasTheSame()
+
+
         # Fetching complete, now filters
         # @todo move to class / maybe inside of fetcher abstract base?
 
@@ -140,7 +162,32 @@ class perform_site_check():
             is_html = False
             is_json = False
 
-        include_filters_rule = watch.get('include_filters', [])
+        if watch.is_pdf or 'application/pdf' in fetcher.headers.get('Content-Type', '').lower():
+            from shutil import which
+            tool = os.getenv("PDF_TO_HTML_TOOL", "pdftohtml")
+            if not which(tool):
+                raise PDFToHTMLToolNotFound("Command-line `{}` tool was not found in system PATH, was it installed?".format(tool))
+
+            import subprocess
+            proc = subprocess.Popen(
+                [tool, '-stdout', '-', '-s', 'out.pdf', '-i'],
+                stdout=subprocess.PIPE,
+                stdin=subprocess.PIPE)
+            proc.stdin.write(fetcher.raw_content)
+            proc.stdin.close()
+            fetcher.content = proc.stdout.read().decode('utf-8')
+            proc.wait(timeout=60)
+
+            # Add a little metadata so we know if the file changes (like if an image changes, but the text is the same
+            # @todo may cause problems with non-UTF8?
+            metadata = "<p>Added by changedetection.io: Document checksum - {} Filesize - {} bytes</p>".format(
+                hashlib.md5(fetcher.raw_content).hexdigest().upper(),
+                len(fetcher.content))
+
+            fetcher.content = fetcher.content.replace('</body>', metadata + '</body>')
+
+
+        include_filters_rule = deepcopy(watch.get('include_filters', []))
         # include_filters_rule = watch['include_filters']
         subtractive_selectors = watch.get(
             "subtractive_selectors", []
@@ -148,6 +195,10 @@ class perform_site_check():
             "global_subtractive_selectors", []
         )
 
+        # Inject a virtual LD+JSON price tracker rule
+        if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
+            include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
+
         has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip())
         has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
 
@@ -155,6 +206,14 @@ class perform_site_check():
             include_filters_rule.append("json:$")
             has_filter_rule = True
 
+        if is_json:
+            # Sort the JSON so we dont get false alerts when the content is just re-ordered
+            try:
+                fetcher.content = json.dumps(json.loads(fetcher.content), sort_keys=True)
+            except Exception as e:
+                # Might have just been a snippet, or otherwise bad JSON, continue
+                pass
+
         if has_filter_rule:
             json_filter_prefixes = ['json:', 'jq:']
             for filter in include_filters_rule:
@@ -162,6 +221,8 @@ class perform_site_check():
                     stripped_text_from_html += html_tools.extract_json_as_string(content=fetcher.content, json_filter=filter)
                     is_html = False
 
+
+
         if is_html or is_source:
 
             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
@@ -173,9 +234,13 @@ class perform_site_check():
                 # Don't run get_text or xpath/css filters on plaintext
                 stripped_text_from_html = html_content
             else:
+                # Does it have some ld+json price data? used for easier monitoring
+                update_obj['has_ldjson_price_data'] = html_tools.has_ldjson_product_info(fetcher.content)
+
                 # Then we assume HTML
                 if has_filter_rule:
                     html_content = ""
+
                     for filter_rule in include_filters_rule:
                         # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
                         if filter_rule[0] == '/' or filter_rule.startswith('xpath:'):

changedetectionio/forms.py

@@ -138,7 +138,7 @@ class ValidateContentFetcherIsReady(object):
         from changedetectionio import content_fetcher
 
         # Better would be a radiohandler that keeps a reference to each class
-        if field.data is not None:
+        if field.data is not None and field.data != 'system':
             klass = getattr(content_fetcher, field.data)
             some_object = klass()
             try:
@@ -193,7 +193,7 @@ class ValidateAppRiseServers(object):
                 message = field.gettext('\'%s\' is not a valid AppRise URL.' % (server_url))
                 raise ValidationError(message)
 
-class ValidateTokensList(object):
+class ValidateJinja2Template(object):
     """
     Validates that a {token} is from a valid set
     """
@@ -202,14 +202,27 @@ class ValidateTokensList(object):
 
     def __call__(self, form, field):
         from changedetectionio import notification
-        regex = re.compile('{.*?}')
-        for p in re.findall(regex, field.data):
-            if not p.strip('{}') in notification.valid_tokens:
-                message = field.gettext('Token \'%s\' is not a valid token.')
-                raise ValidationError(message % (p))
-            
+
+        from jinja2 import Environment, BaseLoader, TemplateSyntaxError
+        from jinja2.meta import find_undeclared_variables
+
+
+        try:
+            jinja2_env = Environment(loader=BaseLoader)
+            jinja2_env.globals.update(notification.valid_tokens)
+            rendered = jinja2_env.from_string(field.data).render()
+        except TemplateSyntaxError as e:
+            raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
+
+        ast = jinja2_env.parse(field.data)
+        undefined = ", ".join(find_undeclared_variables(ast))
+        if undefined:
+            raise ValidationError(
+                f"The following tokens used in the notification are not valid: {undefined}"
+            )
+
 class validateURL(object):
-    
+
     """
        Flask wtform validators wont work with basic auth
     """
@@ -219,12 +232,18 @@ class validateURL(object):
 
     def __call__(self, form, field):
         import validators
+
         try:
             validators.url(field.data.strip())
         except validators.ValidationFailure:
             message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
             raise ValidationError(message)
 
+        from .model.Watch import is_safe_url
+        if not is_safe_url(field.data):
+            raise ValidationError('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX')
+
+
 class ValidateListRegex(object):
     """
     Validates that anything that looks like a regex passes as a regex
@@ -331,13 +350,14 @@ class quickWatchForm(Form):
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
 
 
+
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
-    notification_urls = StringListField('Notification URL list', validators=[validators.Optional(), ValidateAppRiseServers()])
-    notification_title = StringField('Notification title', validators=[validators.Optional(), ValidateTokensList()])
-    notification_body = TextAreaField('Notification body', validators=[validators.Optional(), ValidateTokensList()])
+    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
+    notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
+    notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
-    fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
     webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1,
                                                                                                                                     message="Should contain one or more seconds")])
@@ -400,9 +420,25 @@ class watchForm(commonSettingsForm):
             self.body.errors.append('Body must be empty when Request Method is set to GET')
             result = False
 
+        # Attempt to validate jinja2 templates in the URL
+        from jinja2 import Environment
+        # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
+        try:
+            ready_url = str(jinja2_env.from_string(self.url.data).render())
+        except Exception as e:
+            self.url.errors.append('Invalid template syntax')
+            result = False
         return result
 
 
+class SingleExtraProxy(Form):
+
+    # maybe better to set some <script>var..
+    proxy_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
+    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "http://user:pass@...:3128", "size":50})
+    # @todo do the validation here instead
+
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
     time_between_check = FormField(TimeBetweenCheckForm)
@@ -410,21 +446,30 @@ class globalSettingsRequestForm(Form):
     jitter_seconds = IntegerField('Random jitter seconds ± check',
                                   render_kw={"style": "width: 5em;"},
                                   validators=[validators.NumberRange(min=0, message="Should contain zero or more seconds")])
+    extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5)
+
+    def validate_extra_proxies(self, extra_validators=None):
+        for e in self.data['extra_proxies']:
+            if e.get('proxy_name') or e.get('proxy_url'):
+                if not e.get('proxy_name','').strip() or not e.get('proxy_url','').strip():
+                    self.extra_proxies.errors.append('Both a name, and a Proxy URL is required.')
+                    return False
+
 
 # datastore.data['settings']['application']..
 class globalSettingsApplicationForm(commonSettingsForm):
 
-    base_url = StringField('Base URL', validators=[validators.Optional()])
-    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
-    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
-    ignore_whitespace = BooleanField('Ignore whitespace')
-    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
-    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
-    render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
-    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
+    base_url = StringField('Base URL', validators=[validators.Optional()])
+    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
+    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
+    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
+    ignore_whitespace = BooleanField('Ignore whitespace')
     password = SaltyPasswordField()
-
+    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
+    render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
+    shared_diff_access = BooleanField('Allow access to view diff page when password is enabled', default=False, validators=[validators.Optional()])
     filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
                                                                   render_kw={"style": "width: 5em;"},
                                                                   validators=[validators.NumberRange(min=0,
@@ -439,3 +484,8 @@ class globalSettingsForm(Form):
     requests = FormField(globalSettingsRequestForm)
     application = FormField(globalSettingsApplicationForm)
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+
+
+class extractDataForm(Form):
+    extract_regex = StringField('RegEx to extract', validators=[validators.Length(min=1, message="Needs a RegEx")])
+    extract_submit_button = SubmitField('Extract as CSV', render_kw={"class": "pure-button pure-button-primary"})

changedetectionio/html_tools.py

@@ -10,6 +10,10 @@ import re
 # HTML added to be sure each result matching a filter (.example) gets converted to a new line by Inscriptis
 TEXT_FILTER_LIST_LINE_SUFFIX = "<br/>"
 
+# 'price' , 'lowPrice', 'highPrice' are usually under here
+# all of those may or may not appear on different websites
+LD_JSON_PRODUCT_OFFER_SELECTOR = "json:$..offers"
+
 class JSONNotFound(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
@@ -127,8 +131,10 @@ def _get_stripped_text_from_json_match(match):
 
     return stripped_text_from_html
 
-def extract_json_as_string(content, json_filter):
-
+# content - json
+# json_filter - ie json:$..price
+# ensure_is_ldjson_info_type - str "product", optional, "@type == product" (I dont know how to do that as a json selector)
+def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None):
     stripped_text_from_html = False
 
     # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded <script type=ldjson>
@@ -139,7 +145,12 @@ def extract_json_as_string(content, json_filter):
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         s = []
         soup = BeautifulSoup(content, 'html.parser')
-        bs_result = soup.findAll('script')
+
+        if ensure_is_ldjson_info_type:
+            bs_result = soup.findAll('script', {"type": "application/ld+json"})
+        else:
+            bs_result = soup.findAll('script')
+
 
         if not bs_result:
             raise JSONNotFound("No parsable JSON found in this document")
@@ -156,7 +167,14 @@ def extract_json_as_string(content, json_filter):
                 continue
             else:
                 stripped_text_from_html = _parse_json(json_data, json_filter)
-                if stripped_text_from_html:
+                if ensure_is_ldjson_info_type:
+                    # Could sometimes be list, string or something else random
+                    if isinstance(json_data, dict):
+                        # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
+                        # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
+                        if json_data.get('@type', False) and json_data.get('@type','').lower() == ensure_is_ldjson_info_type.lower() and stripped_text_from_html:
+                            break
+                elif stripped_text_from_html:
                     break
 
     if not stripped_text_from_html:
@@ -243,6 +261,18 @@ def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
 
     return text_content
 
+
+# Does LD+JSON exist with a @type=='product' and a .price set anywhere?
+def has_ldjson_product_info(content):
+    try:
+        pricing_data = extract_json_as_string(content=content, json_filter=LD_JSON_PRODUCT_OFFER_SELECTOR, ensure_is_ldjson_info_type="product")
+    except JSONNotFound as e:
+        # Totally fine
+        return False
+    x=bool(pricing_data)
+    return x
+
+
 def workarounds_for_obfuscations(content):
     """
     Some sites are using sneaky tactics to make prices and other information un-renderable by Inscriptis

changedetectionio/model/App.py

@@ -15,11 +15,12 @@ class model(dict):
                 'headers': {
                 },
                 'requests': {
-                    'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")),  # Default 45 seconds
-                    'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
+                    'extra_proxies': [], # Configurable extra proxies via the UI
                     'jitter_seconds': 0,
+                    'proxy': None, # Preferred proxy connection
+                    'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
+                    'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")),  # Default 45 seconds
                     'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")),  # Number of threads, lower is better for slow connections
-                    'proxy': None # Preferred proxy connection
                 },
                 'application': {
                     'api_access_token_enabled': True,
@@ -39,6 +40,7 @@ class model(dict):
                     'notification_body': default_notification_body,
                     'notification_format': default_notification_format,
                     'schema_version' : 0,
+                    'shared_diff_access': False,
                     'webdriver_delay': None  # Extra delay in seconds before extracting text
                 }
             }

changedetectionio/model/Watch.py

@@ -1,9 +1,14 @@
 from distutils.util import strtobool
 import logging
 import os
+import re
 import time
 import uuid
 
+# Allowable protocols, protects against javascript: etc
+# file:// is further checked by ALLOW_FILE_URI
+SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):'
+
 minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
 mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
 
@@ -11,57 +16,74 @@ from changedetectionio.notification import (
     default_notification_format_for_watch
 )
 
+base_config = {
+    'body': None,
+    'check_unique_lines': False,  # On change-detected, compare against all history if its something new
+    'check_count': 0,
+    'consecutive_filter_failures': 0,  # Every time the CSS/xPath filter cannot be located, reset when all is fine.
+    'extract_text': [],  # Extract text by regex after filters
+    'extract_title_as_title': False,
+    'fetch_backend': 'system',
+    'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
+    'has_ldjson_price_data': None,
+    'track_ldjson_price_data': None,
+    'headers': {},  # Extra headers to send
+    'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
+    'include_filters': [],
+    'last_checked': 0,
+    'last_error': False,
+    'last_viewed': 0,  # history key value of the last viewed via the [diff] link
+    'method': 'GET',
+    # Custom notification content
+    'notification_body': None,
+    'notification_format': default_notification_format_for_watch,
+    'notification_muted': False,
+    'notification_title': None,
+    'notification_screenshot': False,  # Include the latest screenshot if available and supported by the apprise URL
+    'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
+    'paused': False,
+    'previous_md5': False,
+    'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
+    'proxy': None,  # Preferred proxy connection
+    'subtractive_selectors': [],
+    'tag': None,
+    'text_should_not_be_present': [],  # Text that should not present
+    # Re #110, so then if this is set to None, we know to use the default value instead
+    # Requires setting to None on submit if it's the same as the default
+    # Should be all None by default, so we use the system default in this case.
+    'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
+    'title': None,
+    'trigger_text': [],  # List of text or regex to wait for until a change is detected
+    'url': '',
+    'uuid': str(uuid.uuid4()),
+    'webdriver_delay': None,
+    'webdriver_js_execute_code': None,  # Run before change-detection
+}
+
+
+def is_safe_url(test_url):
+    # See https://github.com/dgtlmoon/changedetection.io/issues/1358
+
+    # Remove 'source:' prefix so we dont get 'source:javascript:' etc
+    # 'source:' is a valid way to tell us to return the source
+
+    r = re.compile(re.escape('source:'), re.IGNORECASE)
+    test_url = r.sub('', test_url)
+
+    pattern = re.compile(os.getenv('SAFE_PROTOCOL_REGEX', SAFE_PROTOCOL_REGEX), re.IGNORECASE)
+    if not pattern.match(test_url.strip()):
+        return False
+
+    return True
 
 class model(dict):
     __newest_history_key = None
-    __history_n=0
-    __base_config = {
-            #'history': {},  # Dict of timestamp and output stripped filename (removed)
-            #'newest_history_key': 0, (removed, taken from history.txt index)
-            'body': None,
-            'check_unique_lines': False, # On change-detected, compare against all history if its something new
-            'check_count': 0,
-            'consecutive_filter_failures': 0, # Every time the CSS/xPath filter cannot be located, reset when all is fine.
-            'extract_text': [],  # Extract text by regex after filters
-            'extract_title_as_title': False,
-            'fetch_backend': None,
-            'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
-            'headers': {},  # Extra headers to send
-            'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
-            'include_filters': [],
-            'last_checked': 0,
-            'last_error': False,
-            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
-            'method': 'GET',
-             # Custom notification content
-            'notification_body': None,
-            'notification_format': default_notification_format_for_watch,
-            'notification_muted': False,
-            'notification_title': None,
-            'notification_screenshot': False, # Include the latest screenshot if available and supported by the apprise URL
-            'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
-            'paused': False,
-            'previous_md5': False,
-            'proxy': None, # Preferred proxy connection
-            'subtractive_selectors': [],
-            'tag': None,
-            'text_should_not_be_present': [], # Text that should not present
-            # Re #110, so then if this is set to None, we know to use the default value instead
-            # Requires setting to None on submit if it's the same as the default
-            # Should be all None by default, so we use the system default in this case.
-            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
-            'title': None,
-            'trigger_text': [],  # List of text or regex to wait for until a change is detected
-            'url': None,
-            'uuid': str(uuid.uuid4()),
-            'webdriver_delay': None,
-            'webdriver_js_execute_code': None, # Run before change-detection
-        }
+    __history_n = 0
     jitter_seconds = 0
 
     def __init__(self, *arg, **kw):
 
-        self.update(self.__base_config)
+        self.update(base_config)
         self.__datastore_path = kw['datastore_path']
 
         self['uuid'] = str(uuid.uuid4())
@@ -92,13 +114,46 @@ class model(dict):
 
     @property
     def link(self):
+
         url = self.get('url', '')
+        if not is_safe_url(url):
+            return 'DISABLED'
+
+        ready_url = url
         if '{%' in url or '{{' in url:
             from jinja2 import Environment
             # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
             jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
-            return str(jinja2_env.from_string(url).render())
-        return url
+            try:
+                ready_url = str(jinja2_env.from_string(url).render())
+            except Exception as e:
+                from flask import (
+                    flash, Markup, url_for
+                )
+                message = Markup('<a href="{}#general">The URL {} is invalid and cannot be used, click to edit</a>'.format(
+                    url_for('edit_page', uuid=self.get('uuid')), self.get('url', '')))
+                flash(message, 'error')
+                return ''
+
+        return ready_url
+
+    @property
+    def get_fetch_backend(self):
+        """
+        Like just using the `fetch_backend` key but there could be some logic
+        :return:
+        """
+        # Maybe also if is_image etc?
+        # This is because chrome/playwright wont render the PDF in the browser and we will just fetch it and use pdf2html to see the text.
+        if self.is_pdf:
+            return 'html_requests'
+
+        return self.get('fetch_backend')
+
+    @property
+    def is_pdf(self):
+        # content_type field is set in the future
+        return '.pdf' in self.get('url', '').lower() or 'pdf' in self.get('content_type', '').lower()
 
     @property
     def label(self):
@@ -307,3 +362,70 @@ class model(dict):
         if os.path.isfile(fname):
             return fname
         return False
+
+    def pause(self):
+        self['paused'] = True
+
+    def unpause(self):
+        self['paused'] = False
+
+    def toggle_pause(self):
+        self['paused'] ^= True
+
+    def mute(self):
+        self['notification_muted'] = True
+
+    def unmute(self):
+        self['notification_muted'] = False
+
+    def toggle_mute(self):
+        self['notification_muted'] ^= True
+
+    def extract_regex_from_all_history(self, regex):
+        import csv
+        import re
+        import datetime
+        csv_output_filename = False
+        csv_writer = False
+        f = None
+
+        # self.history will be keyed with the full path
+        for k, fname in self.history.items():
+            if os.path.isfile(fname):
+                with open(fname, "r") as f:
+                    contents = f.read()
+                    res = re.findall(regex, contents, re.MULTILINE)
+                    if res:
+                        if not csv_writer:
+                            # A file on the disk can be transferred much faster via flask than a string reply
+                            csv_output_filename = 'report.csv'
+                            f = open(os.path.join(self.watch_data_dir, csv_output_filename), 'w')
+                            # @todo some headers in the future
+                            #fieldnames = ['Epoch seconds', 'Date']
+                            csv_writer = csv.writer(f,
+                                                    delimiter=',',
+                                                    quotechar='"',
+                                                    quoting=csv.QUOTE_MINIMAL,
+                                                    #fieldnames=fieldnames
+                                                    )
+                            csv_writer.writerow(['Epoch seconds', 'Date'])
+                            # csv_writer.writeheader()
+
+                        date_str = datetime.datetime.fromtimestamp(int(k)).strftime('%Y-%m-%d %H:%M:%S')
+                        for r in res:
+                            row = [k, date_str]
+                            if isinstance(r, str):
+                                row.append(r)
+                            else:
+                                row+=r
+                            csv_writer.writerow(row)
+
+        if f:
+            f.close()
+
+        return csv_output_filename
+
+    @property
+    # Return list of tags, stripped and lowercase, used for searching
+    def all_tags(self):
+        return [s.strip().lower() for s in self.get('tag','').split(',')]

changedetectionio/notification.py

@@ -1,5 +1,7 @@
 import apprise
+from jinja2 import Environment, BaseLoader
 from apprise import NotifyFormat
+import json
 
 valid_tokens = {
     'base_url': '',
@@ -16,8 +18,8 @@ valid_tokens = {
 
 default_notification_format_for_watch = 'System default'
 default_notification_format = 'Text'
-default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n'
-default_notification_title = 'ChangeDetection.io Notification - {watch_url}'
+default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
+default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
 
 valid_notification_formats = {
     'Text': NotifyFormat.TEXT,
@@ -27,25 +29,67 @@ valid_notification_formats = {
     default_notification_format_for_watch: default_notification_format_for_watch
 }
 
-def process_notification(n_object, datastore):
+# include the decorator
+from apprise.decorators import notify
 
-    # Get the notification body from datastore
-    n_body = n_object.get('notification_body', default_notification_body)
-    n_title = n_object.get('notification_title', default_notification_title)
-    n_format = valid_notification_formats.get(
-        n_object['notification_format'],
-        valid_notification_formats[default_notification_format],
-    )
+@notify(on="delete")
+@notify(on="deletes")
+@notify(on="get")
+@notify(on="gets")
+@notify(on="post")
+@notify(on="posts")
+@notify(on="put")
+@notify(on="puts")
+def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
+    import requests
+    url = kwargs['meta'].get('url')
+
+    if url.startswith('post'):
+        r = requests.post
+    elif url.startswith('get'):
+        r = requests.get
+    elif url.startswith('put'):
+        r = requests.put
+    elif url.startswith('delete'):
+        r = requests.delete
+
+    url = url.replace('post://', 'http://')
+    url = url.replace('posts://', 'https://')
+    url = url.replace('put://', 'http://')
+    url = url.replace('puts://', 'https://')
+    url = url.replace('get://', 'http://')
+    url = url.replace('gets://', 'https://')
+    url = url.replace('put://', 'http://')
+    url = url.replace('puts://', 'https://')
+    url = url.replace('delete://', 'http://')
+    url = url.replace('deletes://', 'https://')
+
+    # Try to auto-guess if it's JSON
+    headers = {}
+    try:
+        json.loads(body)
+        headers = {'Content-Type': 'application/json; charset=utf-8'}
+    except ValueError as e:
+        pass
+
+
+    r(url, headers=headers, data=body)
+
+
+def process_notification(n_object, datastore):
 
     # Insert variables into the notification content
     notification_parameters = create_notification_parameters(n_object, datastore)
 
-    for n_k in notification_parameters:
-        token = '{' + n_k + '}'
-        val = notification_parameters[n_k]
-        n_title = n_title.replace(token, val)
-        n_body = n_body.replace(token, val)
-
+    # Get the notification body from datastore
+    jinja2_env = Environment(loader=BaseLoader)
+    n_body = jinja2_env.from_string(n_object.get('notification_body', default_notification_body)).render(**notification_parameters)
+    n_title = jinja2_env.from_string(n_object.get('notification_title', default_notification_title)).render(**notification_parameters)
+    n_format = valid_notification_formats.get(
+        n_object['notification_format'],
+        valid_notification_formats[default_notification_format],
+    )
+    
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
     # raise it as an exception
@@ -53,6 +97,7 @@ def process_notification(n_object, datastore):
     sent_objs=[]
     from .apprise_asset import asset
     for url in n_object['notification_urls']:
+        url = jinja2_env.from_string(url).render(**notification_parameters)
         apobj = apprise.Apprise(debug=True, asset=asset)
         url = url.strip()
         if len(url):
@@ -66,7 +111,12 @@ def process_notification(n_object, datastore):
 
                 # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
                 k = '?' if not '?' in url else '&'
-                if not 'avatar_url' in url and not url.startswith('mail'):
+                if not 'avatar_url' in url \
+                        and not url.startswith('mail') \
+                        and not url.startswith('post') \
+                        and not url.startswith('get') \
+                        and not url.startswith('delete') \
+                        and not url.startswith('put'):
                     url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
 
                 if url.startswith('tgram://'):
@@ -144,7 +194,7 @@ def create_notification_parameters(n_object, datastore):
 
     watch_url = n_object['watch_url']
 
-    # Re #148 - Some people have just {base_url} in the body or title, but this may break some notification services
+    # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services
     #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.
     if base_url == '':
         base_url = "<base-url-env-var-not-set>"

changedetectionio/queuedWatchMetaData.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass, field
+from typing import Any
+
+# So that we can queue some metadata in `item`
+# https://docs.python.org/3/library/queue.html#queue.PriorityQueue
+#
+@dataclass(order=True)
+class PrioritizedItem:
+    priority: int
+    item: Any=field(compare=False)

changedetectionio/res/xpath_element_scraper.js

@@ -1,3 +1,16 @@
+// Copyright (C) 2021 Leigh Morresi (dgtlmoon@gmail.com)
+// All rights reserved.
+
+// @file Scrape the page looking for elements of concern (%ELEMENTS%)
+// http://matatk.agrip.org.uk/tests/position-and-width/
+// https://stackoverflow.com/questions/26813480/when-is-element-getboundingclientrect-guaranteed-to-be-updated-accurate
+//
+// Some pages like https://www.londonstockexchange.com/stock/NCCL/ncondezi-energy-limited/analysis
+// will automatically force a scroll somewhere, so include the position offset
+// Lets hope the position doesnt change while we iterate the bbox's, but this is better than nothing
+
+var scroll_y=+document.documentElement.scrollTop || document.body.scrollTop
+
 // Include the getXpath script directly, easier than fetching
 function getxpath(e) {
         var n = e;
@@ -71,8 +84,21 @@ var bbox;
 for (var i = 0; i < elements.length; i++) {
     bbox = elements[i].getBoundingClientRect();
 
-    // forget really small ones
-    if (bbox['width'] < 15 && bbox['height'] < 15) {
+    // Exclude items that are not interactable or visible
+    if(elements[i].style.opacity === "0") {
+        continue
+    }
+    if(elements[i].style.display === "none" || elements[i].style.pointerEvents === "none" ) {
+        continue
+    }
+
+    // Skip really small ones, and where width or height ==0
+    if (bbox['width'] * bbox['height'] < 100) {
+        continue;
+    }
+
+    // Don't include elements that are offset from canvas
+    if (bbox['top']+scroll_y < 0 || bbox['left'] < 0) {
         continue;
     }
 
@@ -109,19 +135,20 @@ for (var i = 0; i < elements.length; i++) {
         continue;
     }
 
+    // @todo Possible to ONLY list where it's clickable to save JSON xfer size
     size_pos.push({
         xpath: xpath_result,
         width: Math.round(bbox['width']),
         height: Math.round(bbox['height']),
         left: Math.floor(bbox['left']),
-        top: Math.floor(bbox['top']),
+        top: Math.floor(bbox['top'])+scroll_y,
         tagName: (elements[i].tagName) ? elements[i].tagName.toLowerCase() : '',
-        tagtype: (elements[i].tagName == 'INPUT' && elements[i].type) ? elements[i].type.toLowerCase() : ''
+        tagtype: (elements[i].tagName == 'INPUT' && elements[i].type) ? elements[i].type.toLowerCase() : '',
+        isClickable: (elements[i].onclick) || window.getComputedStyle(elements[i]).cursor == "pointer"
     });
 
 }
 
-
 // Inject the current one set in the include_filters, which may be a CSS rule
 // used for displaying the current one in VisualSelector, where its not one we generated.
 if (include_filters.length) {
@@ -149,22 +176,40 @@ if (include_filters.length) {
         }
 
         if (q) {
-            bbox = q.getBoundingClientRect();
-        } else {
-            console.log("xpath_element_scraper: filter element "+f+" was not found");
+            // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
+            if (q.hasOwnProperty('getBoundingClientRect')) {
+                bbox = q.getBoundingClientRect();
+                console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
+            } else {
+                try {
+                    // Try and see we can find its ownerElement
+                    bbox = q.ownerElement.getBoundingClientRect();
+                    console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
+                } catch (e) {
+                    console.log("xpath_element_scraper: error looking up ownerElement")
+                }
+            }
+        }
+        
+        if(!q) {
+            console.log("xpath_element_scraper: filter element " + f + " was not found");
         }
 
         if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
             size_pos.push({
                 xpath: f,
-                width: Math.round(bbox['width']),
-                height: Math.round(bbox['height']),
-                left: Math.floor(bbox['left']),
-                top: Math.floor(bbox['top'])
+                width: parseInt(bbox['width']),
+                height: parseInt(bbox['height']),
+                left: parseInt(bbox['left']),
+                top: parseInt(bbox['top'])+scroll_y
             });
         }
     }
 }
 
+// Sort the elements so we find the smallest one first, in other words, we find the smallest one matching in that area
+// so that we dont select the wrapping element by mistake and be unable to select what we want
+size_pos.sort((a, b) => (a.width*a.height > b.width*b.height) ? 1 : -1)
+
 // Window.width required for proper scaling in the frontend
 return {'size_pos': size_pos, 'browser_width': window.innerWidth};

changedetectionio/run_all_tests.sh

@@ -1,104 +0,0 @@
-#!/bin/bash
-
-
-# live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions
-# and I like to restart the server for each test (and have the test cleanup after each test)
-# merge request welcome :)
-
-
-# exit when any command fails
-set -e
-
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-find tests/test_*py -type f|while read test_name
-do
-  echo "TEST RUNNING $test_name"
-  pytest $test_name
-done
-
-echo "RUNNING WITH BASE_URL SET"
-
-# Now re-run some tests with BASE_URL enabled
-# Re #65 - Ability to include a link back to the installation, in the notification.
-export BASE_URL="https://really-unique-domain.io"
-pytest tests/test_notification.py
-
-
-# Re-run with HIDE_REFERER set - could affect login
-export HIDE_REFERER=True
-pytest tests/test_access_control.py
-
-
-# Now for the selenium and playwright/browserless fetchers
-# Note - this is not UI functional tests - just checking that each one can fetch the content
-
-echo "TESTING WEBDRIVER FETCH > SELENIUM/WEBDRIVER..."
-docker run -d --name $$-test_selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome-debug:3.141.59
-# takes a while to spin up
-sleep 5
-export WEBDRIVER_URL=http://localhost:4444/wd/hub
-pytest tests/fetchers/test_content.py
-pytest tests/test_errorhandling.py
-unset WEBDRIVER_URL
-docker kill $$-test_selenium
-
-echo "TESTING WEBDRIVER FETCH > PLAYWRIGHT/BROWSERLESS..."
-# Not all platforms support playwright (not ARM/rPI), so it's not packaged in requirements.txt
-PLAYWRIGHT_VERSION=$(grep -i -E "RUN pip install.+" "$SCRIPT_DIR/../Dockerfile" | grep --only-matching -i -E "playwright[=><~+]+[0-9\.]+")
-echo "using $PLAYWRIGHT_VERSION"
-pip3 install "$PLAYWRIGHT_VERSION"
-docker run -d --name $$-test_browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
-# takes a while to spin up
-sleep 5
-export PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000
-pytest tests/fetchers/test_content.py
-pytest tests/test_errorhandling.py
-pytest tests/visualselector/test_fetch_data.py
-
-unset PLAYWRIGHT_DRIVER_URL
-docker kill $$-test_browserless
-
-# Test proxy list handling, starting two squids on different ports
-# Each squid adds a different header to the response, which is the main thing we test for.
-docker run -d --name $$-squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf -p 3128:3128 ubuntu/squid:4.13-21.10_edge
-docker run -d --name $$-squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf -p 3129:3128 ubuntu/squid:4.13-21.10_edge
-
-
-# So, basic HTTP as env var test
-export HTTP_PROXY=http://localhost:3128
-export HTTPS_PROXY=http://localhost:3128
-pytest tests/proxy_list/test_proxy.py
-docker logs $$-squid-one 2>/dev/null|grep one.changedetection.io
-if [ $? -ne 0 ]
-then
-  echo "Did not see a request to one.changedetection.io in the squid logs (while checking env vars HTTP_PROXY/HTTPS_PROXY)"
-fi
-unset HTTP_PROXY
-unset HTTPS_PROXY
-
-
-# 2nd test actually choose the preferred proxy from proxies.json
-cp tests/proxy_list/proxies.json-example ./test-datastore/proxies.json
-# Makes a watch use a preferred proxy
-pytest tests/proxy_list/test_multiple_proxy.py
-
-# Should be a request in the default "first" squid
-docker logs $$-squid-one 2>/dev/null|grep chosen.changedetection.io
-if [ $? -ne 0 ]
-then
-  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy)"
-fi
-
-# And one in the 'second' squid (user selects this as preferred)
-docker logs $$-squid-two 2>/dev/null|grep chosen.changedetection.io
-if [ $? -ne 0 ]
-then
-  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy)"
-fi
-
-# @todo - test system override proxy selection and watch defaults, setup a 3rd squid?
-docker kill $$-squid-one
-docker kill $$-squid-two
-
-

changedetectionio/run_basic_tests.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+
+# live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions
+# and I like to restart the server for each test (and have the test cleanup after each test)
+# merge request welcome :)
+
+
+# exit when any command fails
+set -e
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+find tests/test_*py -type f|while read test_name
+do
+  echo "TEST RUNNING $test_name"
+  pytest $test_name
+done
+
+echo "RUNNING WITH BASE_URL SET"
+
+# Now re-run some tests with BASE_URL enabled
+# Re #65 - Ability to include a link back to the installation, in the notification.
+export BASE_URL="https://really-unique-domain.io"
+pytest tests/test_notification.py
+
+
+# Re-run with HIDE_REFERER set - could affect login
+export HIDE_REFERER=True
+pytest tests/test_access_control.py

changedetectionio/run_proxy_tests.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# exit when any command fails
+set -e
+
+# Test proxy list handling, starting two squids on different ports
+# Each squid adds a different header to the response, which is the main thing we test for.
+docker run --network changedet-network -d --name squid-one --hostname squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
+docker run --network changedet-network -d --name squid-two --hostname squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
+
+# Used for configuring a custom proxy URL via the UI
+docker run --network changedet-network -d \
+  --name squid-custom \
+  --hostname squid-custom \
+  --rm \
+  -v `pwd`/tests/proxy_list/squid-auth.conf:/etc/squid/conf.d/debian.conf \
+  -v `pwd`/tests/proxy_list/squid-passwords.txt:/etc/squid3/passwords \
+  ubuntu/squid:4.13-21.10_edge
+
+
+## 2nd test actually choose the preferred proxy from proxies.json
+
+docker run --network changedet-network \
+  -v `pwd`/tests/proxy_list/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_multiple_proxy.py'
+
+
+## Should be a request in the default "first" squid
+docker logs squid-one 2>/dev/null|grep chosen.changedetection.io
+if [ $? -ne 0 ]
+then
+  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy - squid one)"
+  exit 1
+fi
+
+# And one in the 'second' squid (user selects this as preferred)
+docker logs squid-two 2>/dev/null|grep chosen.changedetection.io
+if [ $? -ne 0 ]
+then
+  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy - squid two)"
+  exit 1
+fi
+
+
+# Test the UI configurable proxies
+
+docker run --network changedet-network \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py'
+
+
+# Should see a request for one.changedetection.io in there
+docker logs squid-custom 2>/dev/null|grep "TCP_TUNNEL.200.*changedetection.io"
+if [ $? -ne 0 ]
+then
+  echo "Did not see a valid request to changedetection.io in the squid logs (while checking preferred proxy - squid two)"
+  exit 1
+fi
+
+docker kill squid-one squid-two squid-custom

changedetectionio/static/favicons/browserconfig.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<browserconfig>
+    <msapplication>
+        <tile>
+            <square150x150logo src="favicons/mstile-150x150.png"/>
+            <TileColor>#da532c</TileColor>
+        </tile>
+    </msapplication>
+</browserconfig>

changedetectionio/static/favicons/safari-pinned-tab.svg

@@ -0,0 +1,35 @@
+<?xml version="1.0" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN"
+ "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
+<svg version="1.0" xmlns="http://www.w3.org/2000/svg"
+ width="256.000000pt" height="256.000000pt" viewBox="0 0 256.000000 256.000000"
+ preserveAspectRatio="xMidYMid meet">
+<metadata>
+Created by potrace 1.14, written by Peter Selinger 2001-2017
+</metadata>
+<g transform="translate(0.000000,256.000000) scale(0.100000,-0.100000)"
+fill="#000000" stroke="none">
+<path d="M0 1280 l0 -1280 1280 0 1280 0 0 1280 0 1280 -1280 0 -1280 0 0
+-1280z m1555 936 c387 -112 675 -426 741 -810 24 -138 15 -352 -20 -470 -106
+-353 -360 -606 -713 -712 -75 -22 -113 -27 -253 -31 -144 -5 -176 -2 -252 16
+-316 75 -564 271 -707 557 -67 136 -92 237 -98 401 -7 164 5 253 47 378 106
+315 349 556 665 659 114 37 180 45 350 41 125 -2 165 -7 240 -29z"/>
+<path d="M1091 2165 c-364 -82 -629 -328 -738 -682 -24 -80 -27 -103 -27 -258
+-1 -146 2 -182 21 -251 74 -271 259 -497 508 -621 477 -238 1061 -35 1294 450
+61 126 83 220 88 379 7 194 -15 307 -93 461 -126 251 -340 428 -614 507 -99
+29 -343 37 -439 15z m829 -473 c55 -54 100 -106 100 -116 0 -21 -184 -213
+-212 -222 -24 -7 -48 12 -48 38 0 11 26 47 58 80 l57 60 -151 -3 c-145 -4
+-152 -5 -190 -31 -22 -15 -78 -73 -124 -128 l-85 -99 -32 31 -32 31 30 38 c17
+22 70 79 117 128 66 67 97 92 127 100 22 6 106 11 188 11 81 0 147 3 147 8 0
+4 -25 31 -55 61 -55 55 -65 77 -43 99 25 25 50 10 148 -86z m-1002 -101 c46
+-24 141 -121 312 -321 203 -236 290 -330 322 -346 22 -11 60 -14 169 -12 l141
+3 -51 58 c-28 32 -51 64 -51 71 0 18 21 36 43 36 24 0 217 -193 217 -217 0
+-19 -185 -210 -212 -219 -24 -7 -48 12 -48 38 0 10 23 43 50 72 l50 53 -52 7
+c-29 3 -93 6 -142 6 -104 0 -152 12 -200 52 -19 15 -135 144 -258 286 -274
+316 -305 347 -354 361 -22 6 -94 11 -161 11 -67 0 -128 3 -137 6 -22 9 -21 61
+2 67 9 3 86 5 170 6 133 1 158 -2 190 -18z m227 -468 c23 -34 17 -43 -103
+-172 -119 -128 -131 -133 -343 -129 l-154 3 0 35 c0 34 1 35 50 42 28 3 96 7
+153 7 64 1 115 6 136 15 20 8 71 56 127 120 52 58 99 106 105 106 7 0 20 -12
+29 -27z"/>
+</g>
+</svg>

changedetectionio/static/favicons/site.webmanifest

@@ -0,0 +1,19 @@
+{
+    "name": "",
+    "short_name": "",
+    "icons": [
+        {
+            "src": "android-chrome-192x192.png",
+            "sizes": "192x192",
+            "type": "image/png"
+        },
+        {
+            "src": "android-chrome-256x256.png",
+            "sizes": "256x256",
+            "type": "image/png"
+        }
+    ],
+    "theme_color": "#ffffff",
+    "background_color": "#ffffff",
+    "display": "standalone"
+}

changedetectionio/static/images/bell-off.svg

@@ -1,42 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   width="15"
-   height="16.363636"
-   viewBox="0 0 15 16.363636"
-   version="1.1"
-   id="svg4"
-   sodipodi:docname="bell-off.svg"
-   inkscape:version="1.1.1 (1:1.1+202109281949+c3084ef5ed)"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg">
-  <sodipodi:namedview
-     id="namedview5"
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1.0"
-     inkscape:pageshadow="2"
-     inkscape:pageopacity="0.0"
-     inkscape:pagecheckerboard="0"
-     showgrid="false"
-     fit-margin-top="0"
-     fit-margin-left="0"
-     fit-margin-right="0"
-     fit-margin-bottom="0"
-     inkscape:zoom="28.416667"
-     inkscape:cx="-0.59824046"
-     inkscape:cy="12"
-     inkscape:window-width="1554"
-     inkscape:window-height="896"
-     inkscape:window-x="2095"
-     inkscape:window-y="107"
-     inkscape:window-maximized="0"
-     inkscape:current-layer="svg4" />
-  <defs
-     id="defs8" />
-  <path
-     d="m 14.318182,11.762045 v 1.1925 H 5.4102273 L 11.849318,7.1140909 C 12.234545,9.1561364 12.54,11.181818 14.318182,11.762045 Z m -6.7984093,4.601591 c 1.0759091,0 2.0256823,-0.955909 2.0256823,-2.045454 H 5.4545455 c 0,1.089545 0.9879545,2.045454 2.0652272,2.045454 z M 15,2.8622727 0.9177273,15.636136 0,14.627045 l 1.8443182,-1.6725 h -1.1625 v -1.1925 C 4.0070455,10.677273 2.1784091,4.5388636 5.3611364,2.6897727 5.8009091,2.4347727 6.0709091,1.9609091 6.0702273,1.4488636 v -0.00205 C 6.0702273,0.64772727 6.7104545,0 7.5,0 8.2895455,0 8.9297727,0.64772727 8.9297727,1.4468182 v 0.00205 C 8.9290909,1.9602319 9.199773,2.4354591 9.638864,2.6897773 10.364318,3.111141 10.827273,3.7568228 11.1525,4.5129591 L 14.085682,1.8531818 Z M 6.8181818,1.3636364 C 6.8181818,1.74 7.1236364,2.0454545 7.5,2.0454545 7.8763636,2.0454545 8.1818182,1.74 8.1818182,1.3636364 8.1818182,0.98795455 7.8763636,0.68181818 7.5,0.68181818 c -0.3763636,0 -0.6818182,0.30613637 -0.6818182,0.68181822 z"
-     id="path2"
-     style="fill:#f8321b;stroke-width:0.681818;fill-opacity:1" />
+<svg width="15" height="16.363636" viewBox="0 0 15 16.363636" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg">
+  <path d="m 14.318182,11.762045 v 1.1925 H 5.4102273 L 11.849318,7.1140909 C 12.234545,9.1561364 12.54,11.181818 14.318182,11.762045 Z m -6.7984093,4.601591 c 1.0759091,0 2.0256823,-0.955909 2.0256823,-2.045454 H 5.4545455 c 0,1.089545 0.9879545,2.045454 2.0652272,2.045454 z M 15,2.8622727 0.9177273,15.636136 0,14.627045 l 1.8443182,-1.6725 h -1.1625 v -1.1925 C 4.0070455,10.677273 2.1784091,4.5388636 5.3611364,2.6897727 5.8009091,2.4347727 6.0709091,1.9609091 6.0702273,1.4488636 v -0.00205 C 6.0702273,0.64772727 6.7104545,0 7.5,0 8.2895455,0 8.9297727,0.64772727 8.9297727,1.4468182 v 0.00205 C 8.9290909,1.9602319 9.199773,2.4354591 9.638864,2.6897773 10.364318,3.111141 10.827273,3.7568228 11.1525,4.5129591 L 14.085682,1.8531818 Z M 6.8181818,1.3636364 C 6.8181818,1.74 7.1236364,2.0454545 7.5,2.0454545 7.8763636,2.0454545 8.1818182,1.74 8.1818182,1.3636364 8.1818182,0.98795455 7.8763636,0.68181818 7.5,0.68181818 c -0.3763636,0 -0.6818182,0.30613637 -0.6818182,0.68181822 z" id="path2" style="fill:#f8321b;stroke-width:0.681818;fill-opacity:1"/>
 </svg>

changedetectionio/static/images/pdf-icon.svg

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg xmlns="http://www.w3.org/2000/svg" width="75.320129mm" height="92.604164mm" viewBox="0 0 75.320129 92.604164">
+  <g transform="translate(53.548057 -183.975276) scale(1.4843)">
+    <path fill="#ff2116" d="M-29.632812 123.94727c-3.551967 0-6.44336 2.89347-6.44336 6.44531v49.49804c0 3.55185 2.891393 6.44532 6.44336 6.44532H8.2167969c3.5519661 0 6.4433591-2.89335 6.4433591-6.44532v-40.70117s.101353-1.19181-.416015-2.35156c-.484969-1.08711-1.275391-1.84375-1.275391-1.84375a1.0584391 1.0584391 0 0 0-.0059-.008l-9.3906254-9.21094a1.0584391 1.0584391 0 0 0-.015625-.0156s-.8017392-.76344-1.9902344-1.27344c-1.39939552-.6005-2.8417968-.53711-2.8417968-.53711l.021484-.002z" color="#000" font-family="sans-serif" overflow="visible" paint-order="markers fill stroke" style="line-height:normal;font-variant-ligatures:normal;font-variant-position:normal;font-variant-caps:normal;font-variant-numeric:normal;font-variant-alternates:normal;font-feature-settings:normal;text-indent:0;text-align:start;text-decoration-line:none;text-decoration-style:solid;text-decoration-color:#000000;text-transform:none;text-orientation:mixed;white-space:normal;shape-padding:0;isolation:auto;mix-blend-mode:normal;solid-color:#000000;solid-opacity:1"/>
+    <path fill="#f5f5f5" d="M-29.632812 126.06445h28.3789058a1.0584391 1.0584391 0 0 0 .021484 0s1.13480448.011 1.96484378.36719c.79889772.34282 1.36536982.86176 1.36914062.86524.0000125.00001.00391.004.00391.004l9.3671868 9.18945s.564354.59582.837891 1.20899c.220779.49491.234375 1.40039.234375 1.40039a1.0584391 1.0584391 0 0 0-.002.0449v40.74609c0 2.41592-1.910258 4.32813-4.3261717 4.32813H-29.632812c-2.415914 0-4.326172-1.91209-4.326172-4.32813v-49.49804c0-2.41603 1.910258-4.32813 4.326172-4.32813z" color="#000" font-family="sans-serif" overflow="visible" paint-order="markers fill stroke" style="line-height:normal;font-variant-ligatures:normal;font-variant-position:normal;font-variant-caps:normal;font-variant-numeric:normal;font-variant-alternates:normal;font-feature-settings:normal;text-indent:0;text-align:start;text-decoration-line:none;text-decoration-style:solid;text-decoration-color:#000000;text-transform:none;text-orientation:mixed;white-space:normal;shape-padding:0;isolation:auto;mix-blend-mode:normal;solid-color:#000000;solid-opacity:1"/>
+    <path fill="#ff2116" d="M-23.40766 161.09299c-1.45669-1.45669.11934-3.45839 4.39648-5.58397l2.69124-1.33743 1.04845-2.29399c.57665-1.26169 1.43729-3.32036 1.91254-4.5748l.8641-2.28082-.59546-1.68793c-.73217-2.07547-.99326-5.19438-.52872-6.31588.62923-1.51909 2.69029-1.36323 3.50626.26515.63727 1.27176.57212 3.57488-.18329 6.47946l-.6193 2.38125.5455.92604c.30003.50932 1.1764 1.71867 1.9475 2.68743l1.44924 1.80272 1.8033728-.23533c5.72900399-.74758 7.6912472.523 7.6912472 2.34476 0 2.29921-4.4984914 2.48899-8.2760865-.16423-.8499666-.59698-1.4336605-1.19001-1.4336605-1.19001s-2.3665326.48178-3.531704.79583c-1.202707.32417-1.80274.52719-3.564509 1.12186 0 0-.61814.89767-1.02094 1.55026-1.49858 2.4279-3.24833 4.43998-4.49793 5.1723-1.3991.81993-2.86584.87582-3.60433.13733zm2.28605-.81668c.81883-.50607 2.47616-2.46625 3.62341-4.28553l.46449-.73658-2.11497 1.06339c-3.26655 1.64239-4.76093 3.19033-3.98386 4.12664.43653.52598.95874.48237 2.01093-.16792zm21.21809-5.95578c.80089-.56097.68463-1.69142-.22082-2.1472-.70466-.35471-1.2726074-.42759-3.1031574-.40057-1.1249.0767-2.9337647.3034-3.2403347.37237 0 0 .993716.68678 1.434896.93922.58731.33544 2.0145161.95811 3.0565161 1.27706 1.02785.31461 1.6224.28144 2.0729-.0409zm-8.53152-3.54594c-.4847-.50952-1.30889-1.57296-1.83152-2.3632-.68353-.89643-1.02629-1.52887-1.02629-1.52887s-.4996 1.60694-.90948 2.57394l-1.27876 3.16076-.37075.71695s1.971043-.64627 2.97389-.90822c1.0621668-.27744 3.21787-.70134 3.21787-.70134zm-2.74938-11.02573c.12363-1.0375.1761-2.07346-.15724-2.59587-.9246-1.01077-2.04057-.16787-1.85154 2.23517.0636.8084.26443 2.19033.53292 3.04209l.48817 1.54863.34358-1.16638c.18897-.64151.47882-2.02015.64411-3.06364z"/>
+    <path fill="#2c2c2c" d="M-20.930423 167.83862h2.364986q1.133514 0 1.840213.2169.706698.20991 1.189489.9446.482795.72769.482795 1.75625 0 .94459-.391832 1.6233-.391833.67871-1.056548.97958-.65772.30087-2.02913.30087h-.818651v3.72941h-1.581322zm1.581322 1.22447v3.33058h.783664q1.049552 0 1.44838-.39184.405826-.39183.405826-1.27345 0-.65772-.265887-1.06355-.265884-.41282-.587747-.50378-.314866-.098-1.000572-.098zm5.50664-1.22447h2.148082q1.560333 0 2.4909318.55276.9375993.55276 1.4133973 1.6443.482791 1.09153.482791 2.42096 0 1.3994-.4338151 2.49793-.4268149 1.09153-1.3154348 1.76324-.8816233.67172-2.5189212.67172h-2.267031zm1.581326 1.26645v7.018h.657715q1.378411 0 2.001144-.9516.6227329-.95858.6227329-2.5539 0-3.5125-2.6238769-3.5125zm6.4722254-1.26645h5.30372941v1.26645H-4.2075842v2.85478h2.9807225v1.26646h-2.9807225v4.16322h-1.5813254z" font-family="Franklin Gothic Medium Cond" letter-spacing="0" style="line-height:125%;-inkscape-font-specification:'Franklin Gothic Medium Cond'" word-spacing="4.26000023"/>
+  </g>
+</svg>

changedetectionio/static/images/price-tag-icon.svg

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="83.39" height="89.648" enable-background="new 0 0 122.406 122.881" version="1.1" viewBox="0 0 83.39 89.648" xml:space="preserve" xmlns="http://www.w3.org/2000/svg"><g transform="translate(5e-4 -33.234)"><path d="m44.239 42.946-39.111 39.896 34.908 34.91 39.09-39.876-1.149-34.931zm-0.91791 42.273c0.979-0.979 1.507-1.99 1.577-3.027 0.077-1.043-0.248-2.424-0.967-4.135-0.725-1.717-1.348-3.346-1.87-4.885s-0.814-3.014-0.897-4.432c-0.07-1.42 0.134-2.768 0.624-4.045 0.477-1.279 1.348-2.545 2.607-3.804 2.099-2.099 4.535-3.123 7.314-3.065 2.773 0.063 5.457 1.158 8.04 3.294l2.881 3.034c1.946 2.607 2.799 5.33 2.557 8.166-0.235 2.83-1.532 5.426-3.893 7.785l-6.296-6.297c1.291-1.291 2.035-2.531 2.238-3.727 0.191-1.197-0.165-2.252-1.081-3.168-0.821-0.82-1.717-1.195-2.69-1.139-0.967 0.064-1.908 0.547-2.817 1.457-0.922 0.922-1.393 1.914-1.412 2.977s0.306 2.416 0.973 4.064c0.661 1.652 1.24 3.25 1.736 4.801 0.496 1.553 0.782 3.035 0.858 4.445 0.076 1.426-0.127 2.787-0.591 4.104-0.477 1.316-1.336 2.596-2.588 3.848-2.125 2.125-4.522 3.186-7.212 3.18s-5.311-1.063-7.855-3.16l-3.747 3.746-2.964-2.965 3.766-3.764c-2.423-2.996-3.568-5.998-3.447-9.02 0.127-3.014 1.476-5.813 4.045-8.383l6.278 6.277c-1.412 1.412-2.175 2.799-2.277 4.16-0.108 1.367 0.414 2.627 1.571 3.783 0.839 0.84 1.755 1.26 2.741 1.242 0.985-0.017 1.92-0.47 2.798-1.347zm21.127-46.435h17.457c-0.0269 2.2368 0.69936 16.025 0.69936 16.025l0.785 23.858c0.019 0.609-0.221 1.164-0.619 1.564l5e-3 4e-3 -41.236 42.022c-0.82213 0.8378-2.175 0.83-3.004 0l-37.913-37.91c-0.83-0.83-0.83-2.176 0-3.006l41.236-42.021c0.39287-0.42671 1.502-0.53568 1.502-0.53568zm18.011 11.59c-59.392-29.687-29.696-14.843 0 0z"/></g></svg>

changedetectionio/static/images/spread.svg

@@ -1,46 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   width="18"
-   height="19.92"
-   viewBox="0 0 18 19.92"
-   version="1.1"
-   id="svg6"
-   sodipodi:docname="spread.svg"
-   inkscape:version="1.1.1 (1:1.1+202109281949+c3084ef5ed)"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg">
-  <defs
-     id="defs10" />
-  <sodipodi:namedview
-     id="namedview8"
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1.0"
-     inkscape:pageshadow="2"
-     inkscape:pageopacity="0.0"
-     inkscape:pagecheckerboard="0"
-     showgrid="false"
-     fit-margin-top="0"
-     fit-margin-left="0"
-     fit-margin-right="0"
-     fit-margin-bottom="0"
-     inkscape:zoom="28.416667"
-     inkscape:cx="9.0087975"
-     inkscape:cy="9.9941348"
-     inkscape:window-width="1920"
-     inkscape:window-height="1056"
-     inkscape:window-x="1920"
-     inkscape:window-y="0"
-     inkscape:window-maximized="1"
-     inkscape:current-layer="svg6" />
-  <path
-     d="M -3,-2 H 21 V 22 H -3 Z"
-     fill="none"
-     id="path2" />
-  <path
-     d="m 15,14.08 c -0.76,0 -1.44,0.3 -1.96,0.77 L 5.91,10.7 C 5.96,10.47 6,10.24 6,10 6,9.76 5.96,9.53 5.91,9.3 L 12.96,5.19 C 13.5,5.69 14.21,6 15,6 16.66,6 18,4.66 18,3 18,1.34 16.66,0 15,0 c -1.66,0 -3,1.34 -3,3 0,0.24 0.04,0.47 0.09,0.7 L 5.04,7.81 C 4.5,7.31 3.79,7 3,7 1.34,7 0,8.34 0,10 c 0,1.66 1.34,3 3,3 0.79,0 1.5,-0.31 2.04,-0.81 l 7.12,4.16 c -0.05,0.21 -0.08,0.43 -0.08,0.65 0,1.61 1.31,2.92 2.92,2.92 1.61,0 2.92,-1.31 2.92,-2.92 0,-1.61 -1.31,-2.92 -2.92,-2.92 z"
-     id="path4"
-     style="fill:#0078e7;fill-opacity:1" />
+<svg width="18" height="19.92" viewBox="0 0 18 19.92" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg">
+  <path d="M -3,-2 H 21 V 22 H -3 Z" fill="none" id="path2"/>
+  <path d="m 15,14.08 c -0.76,0 -1.44,0.3 -1.96,0.77 L 5.91,10.7 C 5.96,10.47 6,10.24 6,10 6,9.76 5.96,9.53 5.91,9.3 L 12.96,5.19 C 13.5,5.69 14.21,6 15,6 16.66,6 18,4.66 18,3 18,1.34 16.66,0 15,0 c -1.66,0 -3,1.34 -3,3 0,0.24 0.04,0.47 0.09,0.7 L 5.04,7.81 C 4.5,7.31 3.79,7 3,7 1.34,7 0,8.34 0,10 c 0,1.66 1.34,3 3,3 0.79,0 1.5,-0.31 2.04,-0.81 l 7.12,4.16 c -0.05,0.21 -0.08,0.43 -0.08,0.65 0,1.61 1.31,2.92 2.92,2.92 1.61,0 2.92,-1.31 2.92,-2.92 0,-1.61 -1.31,-2.92 -2.92,-2.92 z" id="path4" style="fill:#0078e7;fill-opacity:1"/>
 </svg>

changedetectionio/static/js/browser-steps.js

@@ -10,10 +10,10 @@ $(document).ready(function () {
         }
     })
     var browsersteps_session_id;
-    var browserless_seconds_remaining=0;
+    var browserless_seconds_remaining = 0;
     var apply_buttons_disabled = false;
     var include_text_elements = $("#include_text_elements");
-    var xpath_data;
+    var xpath_data = false;
     var current_selected_i;
     var state_clicked = false;
     var c;
@@ -25,11 +25,42 @@ $(document).ready(function () {
     $(window).resize(function () {
         set_scale();
     });
+    // Should always be disabled
+    $('#browser_steps >li:first-child select').val('Goto site').attr('disabled', 'disabled');
 
-    $('a#browsersteps-tab').click(function () {
+    $('#browsersteps-click-start').click(function () {
+        $("#browsersteps-click-start").fadeOut();
+        $("#browsersteps-selector-wrapper .spinner").fadeIn();
         start();
     });
 
+    $('a#browsersteps-tab').click(function () {
+        reset();
+    });
+
+    window.addEventListener('hashchange', function () {
+        if (window.location.hash == '#browser-steps') {
+            reset();
+        }
+    });
+
+    function reset() {
+        xpath_data = false;
+        $('#browsersteps-img').removeAttr('src');
+        $("#browsersteps-click-start").show();
+        $("#browsersteps-selector-wrapper .spinner").hide();
+        browserless_seconds_remaining = 0;
+        browsersteps_session_id = false;
+        apply_buttons_disabled = false;
+        ctx.clearRect(0, 0, c.width, c.height);
+        set_first_gotosite_disabled();
+    }
+
+    function set_first_gotosite_disabled() {
+        $('#browser_steps >li:first-child select').val('Goto site').attr('disabled', 'disabled');
+        $('#browser_steps >li:first-child').css('opacity', '0.5');
+    }
+
     // Show seconds remaining until playwright/browserless needs to restart the session
     // (See comment at the top of changedetectionio/blueprint/browser_steps/__init__.py )
     setInterval(() => {
@@ -40,21 +71,6 @@ $(document).ready(function () {
     }, "1000")
 
 
-    if (window.location.hash == '#browser-steps') {
-        start();
-    }
-
-    window.addEventListener('hashchange', function () {
-        if (window.location.hash == '#browser-steps') {
-            start();
-        }
-        // For when the page loads
-        if (!window.location.hash || window.location.hash != '#browser-steps') {
-            $("img#browsersteps-img").attr('src', '');
-            return;
-        }
-    });
-
     function set_scale() {
 
         // some things to check if the scaling doesnt work
@@ -87,7 +103,6 @@ $(document).ready(function () {
         // @todo is click better?
         $('#browsersteps-selector-canvas').off("mousemove mousedown click");
         // Undo disable_browsersteps_ui
-        $("#browser_steps select,input").removeAttr('disabled').css('opacity', '1.0');
         $("#browser-steps-ui").css('opacity', '1.0');
 
         // init
@@ -103,7 +118,7 @@ $(document).ready(function () {
             // https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent
             e.preventDefault()
             console.log(e);
-            console.log("current xpath in index is "+current_selected_i);
+            console.log("current xpath in index is " + current_selected_i);
             last_click_xy = {'x': parseInt((1 / x_scale) * e.offsetX), 'y': parseInt((1 / y_scale) * e.offsetY)}
             process_selected(current_selected_i);
             current_selected_i = false;
@@ -118,6 +133,10 @@ $(document).ready(function () {
         });
 
         $('#browsersteps-selector-canvas').bind('mousemove', function (e) {
+            if (!xpath_data) {
+                return;
+            }
+
             // checkbox if find elements is enabled
             ctx.clearRect(0, 0, c.width, c.height);
             ctx.fillStyle = 'rgba(255,0,0, 0.1)';
@@ -153,8 +172,8 @@ $(document).ready(function () {
                         // does it mean sort the xpath list by size (w*h) i think so!
                     } else {
 
-                        if ( include_text_elements[0].checked === true) {
-                        // blue one with background instead?
+                        if (include_text_elements[0].checked === true) {
+                            // blue one with background instead?
                             ctx.fillStyle = 'rgba(0,0,255, 0.1)';
                             ctx.strokeStyle = 'rgba(0,0,200, 0.7)';
                             $('#browsersteps-selector-canvas').css('cursor', 'grab');
@@ -175,7 +194,6 @@ $(document).ready(function () {
     // });
 
 
-
     // callback for clicking on an xpath on the canvas
     function process_selected(xpath_data_index) {
         found_something = false;
@@ -190,25 +208,23 @@ $(document).ready(function () {
             console.log(x);
             if (x && first_available.length) {
                 // @todo will it let you click shit that has a layer ontop? probably not.
-                if (x['tagtype'] === 'text' || x['tagtype'] === 'email' || x['tagName'] === 'textarea' || x['tagtype'] === 'password' || x['tagtype'] === 'search' ) {
+                if (x['tagtype'] === 'text' || x['tagtype'] === 'email' || x['tagName'] === 'textarea' || x['tagtype'] === 'password' || x['tagtype'] === 'search') {
                     $('select', first_available).val('Enter text in field').change();
                     $('input[type=text]', first_available).first().val(x['xpath']);
                     $('input[placeholder="Value"]', first_available).addClass('ok').click().focus();
                     found_something = true;
                 } else {
-                    // Assume it's just for clicking on
-                    // what are we clicking on?
-                    if (x['tagName'].startsWith('h')|| x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit'|| x['tagtype'] === 'checkbox'|| x['tagtype'] === 'radio'|| x['tagtype'] === 'li') {
+                    if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
                         $('select', first_available).val('Click element').change();
                         $('input[type=text]', first_available).first().val(x['xpath']);
                         found_something = true;
                     }
                 }
 
-                first_available.xpath_data_index=xpath_data_index;
+                first_available.xpath_data_index = xpath_data_index;
 
                 if (!found_something) {
-                    if ( include_text_elements[0].checked === true) {
+                    if (include_text_elements[0].checked === true) {
                         // Suggest that we use as filter?
                         // @todo filters should always be in the last steps, nothing non-filter after it
                         found_something = true;
@@ -220,8 +236,6 @@ $(document).ready(function () {
                     }
                 }
             }
-        } else {
-
         }
     }
 
@@ -234,15 +248,15 @@ $(document).ready(function () {
 
     function start() {
         console.log("Starting browser-steps UI");
-        browsersteps_session_id=Date.now();
+        browsersteps_session_id = Date.now();
         // @todo This setting of the first one should be done at the datalayer but wtforms doesnt wanna play nice
         $('#browser_steps >li:first-child').removeClass('empty');
-        $('#browser_steps >li:first-child select').val('Goto site').attr('disabled', 'disabled');
-        $('#browser-steps-ui .loader').show();
+        set_first_gotosite_disabled();
+        $('#browser-steps-ui .loader .spinner').show();
         $('.clear,.remove', $('#browser_steps >li:first-child')).hide();
         $.ajax({
             type: "GET",
-            url: browser_steps_sync_url+"&browsersteps_session_id="+browsersteps_session_id,
+            url: browser_steps_sync_url + "&browsersteps_session_id=" + browsersteps_session_id,
             statusCode: {
                 400: function () {
                     // More than likely the CSRF token was lost when the server restarted
@@ -251,11 +265,12 @@ $(document).ready(function () {
             }
         }).done(function (data) {
             xpath_data = data.xpath_data;
-            $('#browsersteps-img').attr('src', data.screenshot);
             $("#loading-status-text").fadeIn();
             // This should trigger 'Goto site'
+            console.log("Got startup response, requesting Goto-Site (first) step fake click");
             $('#browser_steps >li:first-child .apply').click();
             browserless_seconds_remaining = data.browser_time_remaining;
+            set_first_gotosite_disabled();
         }).fail(function (data) {
             console.log(data);
             alert('There was an error communicating with the server.');
@@ -264,7 +279,7 @@ $(document).ready(function () {
     }
 
     function disable_browsersteps_ui() {
-        $("#browser_steps select,input").attr('disabled', 'disabled').css('opacity', '0.5');
+        set_first_gotosite_disabled();
         $("#browser-steps-ui").css('opacity', '0.3');
         $('#browsersteps-selector-canvas').off("mousemove mousedown click");
     }
@@ -311,11 +326,14 @@ $(document).ready(function () {
 
     // Add the extra buttons to the steps
     $('ul#browser_steps li').each(function (i) {
-            $(this).append('<div class="control">' +
-                '<a data-step-index=' + i + ' class="pure-button button-secondary button-green button-xsmall apply" >Apply</a>&nbsp;' +
-                '<a data-step-index=' + i + ' class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;' +
-                '<a data-step-index=' + i + ' class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>' +
-                '</div>')
+            var s = '<div class="control">' + '<a data-step-index=' + i + ' class="pure-button button-secondary button-green button-xsmall apply" >Apply</a>&nbsp;';
+            if (i > 0) {
+                // The first step never gets these (Goto-site)
+                s += '<a data-step-index=' + i + ' class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;' +
+                    '<a data-step-index=' + i + ' class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>';
+            }
+            s += '</div>';
+            $(this).append(s)
         }
     );
 
@@ -352,15 +370,15 @@ $(document).ready(function () {
 
     $('ul#browser_steps li .control .apply').click(function (event) {
         // sequential requests @todo refactor
-        if(apply_buttons_disabled) {
+        if (apply_buttons_disabled) {
             return;
         }
 
         var current_data = $(event.currentTarget).closest('li');
-        $('#browser-steps-ui .loader').fadeIn();
-        apply_buttons_disabled=true;
-        $('ul#browser_steps li .control .apply').css('opacity',0.5);
-        $("#browsersteps-img").css('opacity',0.65);
+        $('#browser-steps-ui .loader .spinner').fadeIn();
+        apply_buttons_disabled = true;
+        $('ul#browser_steps li .control .apply').css('opacity', 0.5);
+        $("#browsersteps-img").css('opacity', 0.65);
 
         var is_last_step = 0;
         var step_n = $(event.currentTarget).data('step-index');
@@ -372,17 +390,17 @@ $(document).ready(function () {
             }
         });
 
-        if (is_last_step == (step_n+1)) {
+        if (is_last_step == (step_n + 1)) {
             is_last_step = true;
         } else {
             is_last_step = false;
         }
 
-
+        console.log("Requesting step via POST " + $("select[id$='operation']", current_data).first().val());
         // POST the currently clicked step form widget back and await response, redraw
         $.ajax({
             method: "POST",
-            url: browser_steps_sync_url+"&browsersteps_session_id="+browsersteps_session_id,
+            url: browser_steps_sync_url + "&browsersteps_session_id=" + browsersteps_session_id,
             data: {
                 'operation': $("select[id$='operation']", current_data).first().val(),
                 'selector': $("input[id$='selector']", current_data).first().val(),
@@ -395,28 +413,35 @@ $(document).ready(function () {
                     // More than likely the CSRF token was lost when the server restarted
                     alert("There was a problem processing the request, please reload the page.");
                     $("#loading-status-text").hide();
+                    $('#browser-steps-ui .loader .spinner').fadeOut();
+                },
+                401: function (data) {
+                    // More than likely the CSRF token was lost when the server restarted
+                    alert(data.responseText);
+                    $("#loading-status-text").hide();
+                    $('#browser-steps-ui .loader .spinner').fadeOut();
                 }
             }
         }).done(function (data) {
             // it should return the new state (selectors available and screenshot)
             xpath_data = data.xpath_data;
             $('#browsersteps-img').attr('src', data.screenshot);
-            $('#browser-steps-ui .loader').fadeOut();
-            apply_buttons_disabled=false;
-            $("#browsersteps-img").css('opacity',1);
-            $('ul#browser_steps li .control .apply').css('opacity',1);
+            $('#browser-steps-ui .loader .spinner').fadeOut();
+            apply_buttons_disabled = false;
+            $("#browsersteps-img").css('opacity', 1);
+            $('ul#browser_steps li .control .apply').css('opacity', 1);
             browserless_seconds_remaining = data.browser_time_remaining;
             $("#loading-status-text").hide();
+            set_first_gotosite_disabled();
         }).fail(function (data) {
             console.log(data);
             if (data.responseText.includes("Browser session expired")) {
                 disable_browsersteps_ui();
             }
-            apply_buttons_disabled=false;
+            apply_buttons_disabled = false;
             $("#loading-status-text").hide();
-            $('ul#browser_steps li .control .apply').css('opacity',1);
-            $("#browsersteps-img").css('opacity',1);
-            //$('#browsersteps-selector-wrapper .loader').fadeOut(2500);
+            $('ul#browser_steps li .control .apply').css('opacity', 1);
+            $("#browsersteps-img").css('opacity', 1);
         });
 
     });

changedetectionio/static/js/diff-overview.js

@@ -13,6 +13,8 @@ $(document).ready(function () {
         } else if (hash_name === '#error-screenshot') {
             $("img#error-screenshot-img").attr('src', error_screenshot_url);
             $("#settings").hide();
+        } else if (hash_name === '#extract') {
+            $("#settings").hide();
         }
 
 

changedetectionio/static/js/diff-render.js

@@ -1,112 +1,110 @@
-var a = document.getElementById('a');
-var b = document.getElementById('b');
-var result = document.getElementById('result');
+var a = document.getElementById("a");
+var b = document.getElementById("b");
+var result = document.getElementById("result");
 
 function changed() {
-    // https://github.com/kpdecker/jsdiff/issues/389
-    // I would love to use `{ignoreWhitespace: true}` here but it breaks the formatting
-    options = {ignoreWhitespace: document.getElementById('ignoreWhitespace').checked};
+  // https://github.com/kpdecker/jsdiff/issues/389
+  // I would love to use `{ignoreWhitespace: true}` here but it breaks the formatting
+  options = {
+    ignoreWhitespace: document.getElementById("ignoreWhitespace").checked,
+  };
 
-    var diff = Diff[window.diffType](a.textContent, b.textContent, options);
-    var fragment = document.createDocumentFragment();
-    for (var i = 0; i < diff.length; i++) {
-
-        if (diff[i].added && diff[i + 1] && diff[i + 1].removed) {
-            var swap = diff[i];
-            diff[i] = diff[i + 1];
-            diff[i + 1] = swap;
-        }
-
-        var node;
-        if (diff[i].removed) {
-            node = document.createElement('del');
-            node.classList.add("change");
-            node.appendChild(document.createTextNode(diff[i].value));
-
-        } else if (diff[i].added) {
-            node = document.createElement('ins');
-            node.classList.add("change");
-            node.appendChild(document.createTextNode(diff[i].value));
-        } else {
-            node = document.createTextNode(diff[i].value);
-        }
-        fragment.appendChild(node);
+  var diff = Diff[window.diffType](a.textContent, b.textContent, options);
+  var fragment = document.createDocumentFragment();
+  for (var i = 0; i < diff.length; i++) {
+    if (diff[i].added && diff[i + 1] && diff[i + 1].removed) {
+      var swap = diff[i];
+      diff[i] = diff[i + 1];
+      diff[i + 1] = swap;
     }
 
-    result.textContent = '';
-    result.appendChild(fragment);
+    var node;
+    if (diff[i].removed) {
+      node = document.createElement("del");
+      node.classList.add("change");
+      const wrapper = node.appendChild(document.createElement("span"));
+      wrapper.appendChild(document.createTextNode(diff[i].value));
+    } else if (diff[i].added) {
+      node = document.createElement("ins");
+      node.classList.add("change");
+      const wrapper = node.appendChild(document.createElement("span"));
+      wrapper.appendChild(document.createTextNode(diff[i].value));
+    } else {
+      node = document.createTextNode(diff[i].value);
+    }
+    fragment.appendChild(node);
+  }
 
-    // Jump at start
-    inputs.current = 0;
-    next_diff();
+  result.textContent = "";
+  result.appendChild(fragment);
+
+  // Jump at start
+  inputs.current = 0;
+  next_diff();
 }
 
 window.onload = function () {
-
-
-    /* Convert what is options from UTC time.time() to local browser time */
-    var diffList = document.getElementById("diff-version");
-    if (typeof (diffList) != 'undefined' && diffList != null) {
-        for (var option of diffList.options) {
-            var dateObject = new Date(option.value * 1000);
-            option.label = dateObject.toLocaleString();
-        }
+  /* Convert what is options from UTC time.time() to local browser time */
+  var diffList = document.getElementById("diff-version");
+  if (typeof diffList != "undefined" && diffList != null) {
+    for (var option of diffList.options) {
+      var dateObject = new Date(option.value * 1000);
+      option.label = dateObject.toLocaleString();
     }
+  }
 
-    /* Set current version date as local time in the browser also */
-    var current_v = document.getElementById("current-v-date");
-    var dateObject = new Date(newest_version_timestamp*1000);
-    current_v.innerHTML = dateObject.toLocaleString();
-    onDiffTypeChange(document.querySelector('#settings [name="diff_type"]:checked'));
-    changed();
+  /* Set current version date as local time in the browser also */
+  var current_v = document.getElementById("current-v-date");
+  var dateObject = new Date(newest_version_timestamp * 1000);
+  current_v.innerHTML = dateObject.toLocaleString();
+  onDiffTypeChange(
+    document.querySelector('#settings [name="diff_type"]:checked'),
+  );
+  changed();
 };
 
-a.onpaste = a.onchange =
-    b.onpaste = b.onchange = changed;
+a.onpaste = a.onchange = b.onpaste = b.onchange = changed;
 
-if ('oninput' in a) {
-    a.oninput = b.oninput = changed;
+if ("oninput" in a) {
+  a.oninput = b.oninput = changed;
 } else {
-    a.onkeyup = b.onkeyup = changed;
+  a.onkeyup = b.onkeyup = changed;
 }
 
 function onDiffTypeChange(radio) {
-    window.diffType = radio.value;
-// Not necessary
-//	document.title = "Diff " + radio.value.slice(4);
+  window.diffType = radio.value;
+  // Not necessary
+  //	document.title = "Diff " + radio.value.slice(4);
 }
 
-var radio = document.getElementsByName('diff_type');
+var radio = document.getElementsByName("diff_type");
 for (var i = 0; i < radio.length; i++) {
-    radio[i].onchange = function (e) {
-        onDiffTypeChange(e.target);
-        changed();
-    }
-}
-
-document.getElementById('ignoreWhitespace').onchange = function (e) {
+  radio[i].onchange = function (e) {
+    onDiffTypeChange(e.target);
     changed();
+  };
 }
 
+document.getElementById("ignoreWhitespace").onchange = function (e) {
+  changed();
+};
 
-var inputs = document.getElementsByClassName('change');
+var inputs = document.getElementsByClassName("change");
 inputs.current = 0;
 
-
 function next_diff() {
+  var element = inputs[inputs.current];
+  var headerOffset = 80;
+  var elementPosition = element.getBoundingClientRect().top;
+  var offsetPosition = elementPosition - headerOffset + window.scrollY;
 
-    var element = inputs[inputs.current];
-    var headerOffset = 80;
-    var elementPosition = element.getBoundingClientRect().top;
-    var offsetPosition = elementPosition - headerOffset + window.scrollY;
+  window.scrollTo({
+    top: offsetPosition,
+    behavior: "smooth",
+  });
 
-    window.scrollTo({
-        top: offsetPosition,
-        behavior: "smooth"
-    });
-
-    inputs.current++;
-    if (inputs.current >= inputs.length) {
-        inputs.current = 0;
-    }
+  inputs.current++;
+  if (inputs.current >= inputs.length) {
+    inputs.current = 0;
+  }
 }

changedetectionio/static/js/toggle-theme.js

@@ -0,0 +1,24 @@
+/**
+ * @file
+ * Toggles theme between light and dark mode.
+ */
+$(document).ready(function () {
+  const button = document.getElementsByClassName("toggle-theme")[0];
+
+  button.onclick = () => {
+    const htmlElement = document.getElementsByTagName("html");
+    const isDarkMode = htmlElement[0].dataset.darkmode === "true";
+    htmlElement[0].dataset.darkmode = !isDarkMode;
+    if (isDarkMode) {
+      button.classList.remove("dark");
+      setCookieValue(false);
+    } else {
+      button.classList.add("dark");
+      setCookieValue(true);
+    }
+  };
+
+  const setCookieValue = (value) => {
+    document.cookie = `css_dark_mode=${value};max-age=31536000;path=/`
+  }
+});

changedetectionio/static/js/visual-selector.js

@@ -1,4 +1,5 @@
-// Horrible proof of concept code :)
+// Copyright (C) 2021 Leigh Morresi (dgtlmoon@gmail.com)
+// All rights reserved.
 // yes - this is really a hack, if you are a front-ender and want to help, please get in touch!
 
 $(document).ready(function () {
@@ -177,9 +178,10 @@ $(document).ready(function () {
             // Basically, find the most 'deepest'
             var found = 0;
             ctx.fillStyle = 'rgba(205,0,0,0.35)';
-            for (var i = selector_data['size_pos'].length; i !== 0; i--) {
+            // Will be sorted by smallest width*height first
+            for (var i = 0; i <= selector_data['size_pos'].length; i++) {
                 // draw all of them? let them choose somehow?
-                var sel = selector_data['size_pos'][i - 1];
+                var sel = selector_data['size_pos'][i];
                 // If we are in a bounding-box
                 if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
                     &&
@@ -195,7 +197,7 @@ $(document).ready(function () {
                     // no need to keep digging
                     // @todo or, O to go out/up, I to go in
                     // or double click to go up/out the selector?
-                    current_selected_i = i - 1;
+                    current_selected_i = i;
                     found += 1;
                     break;
                 }

changedetectionio/static/js/watch-settings.js

@@ -1,7 +1,7 @@
-$(document).ready(function() {
+$(document).ready(function () {
     function toggle() {
         if ($('input[name="fetch_backend"]:checked').val() == 'html_webdriver') {
-            if(playwright_enabled) {
+            if (playwright_enabled) {
                 // playwright supports headers, so hide everything else
                 // See #664
                 $('#requests-override-options #request-method').hide();
@@ -14,9 +14,14 @@ $(document).ready(function() {
                 $('#requests-override-options').hide();
             }
 
-
             $('#webdriver-override-options').show();
 
+        } else if ($('input[name="fetch_backend"]:checked').val() == 'system') {
+            $('#requests-override-options #request-method').hide();
+            $('#requests-override-options #request-body').hide();
+            $('#ignore-status-codes-option').hide();
+            $('#requests-override-options').hide();
+            $('#webdriver-override-options').hide();
         } else {
 
             $('#requests-override-options').show();

changedetectionio/static/styles/.dockerignore

@@ -0,0 +1,3 @@
+node_modules
+package-lock.json
+

changedetectionio/static/styles/diff.css

@@ -1,10 +1,148 @@
+/**
+ * CSS custom properties (aka variables).
+ */
+:root {
+  --color-white: #fff;
+  --color-grey-50: #111;
+  --color-grey-100: #262626;
+  --color-grey-200: #333;
+  --color-grey-300: #444;
+  --color-grey-325: #555;
+  --color-grey-350: #565d64;
+  --color-grey-400: #666;
+  --color-grey-500: #777;
+  --color-grey-600: #999;
+  --color-grey-700: #cbcbcb;
+  --color-grey-750: #ddd;
+  --color-grey-800: #e0e0e0;
+  --color-grey-850: #eee;
+  --color-grey-900: #f2f2f2;
+  --color-black: #000;
+  --color-dark-red: #a00;
+  --color-light-red: #dd0000;
+  --color-background-page: var(--color-grey-100);
+  --color-background-gradient-first: #5ad8f7;
+  --color-background-gradient-second: #2f50af;
+  --color-background-gradient-third: #9150bf;
+  --color-background: var(--color-white);
+  --color-text: var(--color-grey-200);
+  --color-link: #1b98f8;
+  --color-menu-accent: #ed5900;
+  --color-background-code: var(--color-grey-850);
+  --color-error: var(--color-dark-red);
+  --color-error-input: #ffebeb;
+  --color-error-list: var(--color-light-red);
+  --color-table-background: var(--color-background);
+  --color-table-stripe: var(--color-grey-900);
+  --color-text-tab: var(--color-white);
+  --color-background-tab: rgba(255, 255, 255, 0.2);
+  --color-background-tab-hover: rgba(255, 255, 255, 0.5);
+  --color-text-tab-active: #222;
+  --color-api-key: #0078e7;
+  --color-background-button-primary: #0078e7;
+  --color-background-button-green: #42dd53;
+  --color-background-button-red: #dd4242;
+  --color-background-button-success: rgb(28, 184, 65);
+  --color-background-button-error: rgb(202, 60, 60);
+  --color-text-button-error: var(--color-white);
+  --color-background-button-warning: rgb(202, 60, 60);
+  --color-text-button-warning: var(--color-white);
+  --color-background-button-secondary: rgb(66, 184, 221);
+  --color-background-button-cancel: rgb(200, 200, 200);
+  --color-text-button: var(--color-white);
+  --color-background-button-tag: rgb(99, 99, 99);
+  --color-background-snapshot-age: #dfdfdf;
+  --color-error-text-snapshot-age: var(--color-white);
+  --color-error-background-snapshot-age: #ff0000;
+  --color-background-button-tag-active: #9c9c9c;
+  --color-text-messages: var(--color-white);
+  --color-background-messages-message: rgba(255, 255, 255, .2);
+  --color-background-messages-error: rgba(255, 1, 1, .5);
+  --color-background-messages-notice: rgba(255, 255, 255, .5);
+  --color-border-notification: #ccc;
+  --color-background-checkbox-operations: rgba(0, 0, 0, 0.05);
+  --color-warning: #ff3300;
+  --color-border-warning: var(--color-warning);
+  --color-text-legend: var(--color-white);
+  --color-link-new-version: #e07171;
+  --color-last-checked: #bbb;
+  --color-text-footer: #444;
+  --color-border-watch-table-cell: #eee;
+  --color-text-watch-tag-list: #e70069;
+  --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
+  --color-background-new-watch-input: var(--color-white);
+  --color-text-new-watch-input: var(--color-text);
+  --color-border-input: var(--color-grey-500);
+  --color-shadow-input: var(--color-grey-400);
+  --color-background-input: var(--color-white);
+  --color-text-input: var(--color-text);
+  --color-text-input-description: var(--color-grey-500);
+  --color-text-input-placeholder: var(--color-grey-600);
+  --color-background-table-thead: var(--color-grey-800);
+  --color-border-table-cell: var(--color-grey-700);
+  --color-text-menu-heading: var(--color-grey-350);
+  --color-text-menu-link: var(--color-grey-500);
+  --color-background-menu-link-hover: var(--color-grey-850);
+  --color-text-menu-link-hover: var(--color-grey-300);
+  --color-shadow-jump: var(--color-grey-500);
+  --color-icon-github: var(--color-black);
+  --color-icon-github-hover: var(--color-grey-300);
+  --color-watch-table-error: var(--color-dark-red);
+  --color-watch-table-row-text: var(--color-grey-100); }
+
+html[data-darkmode="true"] {
+  --color-link: #59bdfb;
+  --color-text: var(--color-white);
+  --color-background-gradient-first: #3f90a5;
+  --color-background-gradient-second: #1e316c;
+  --color-background-gradient-third: #4d2c64;
+  --color-background-new-watch-input: var(--color-grey-100);
+  --color-text-new-watch-input: var(--color-text);
+  --color-background-table-thead: var(--color-grey-200);
+  --color-table-background: var(--color-grey-300);
+  --color-table-stripe: var(--color-grey-325);
+  --color-background: var(--color-grey-300);
+  --color-text-menu-heading: var(--color-grey-850);
+  --color-text-menu-link: var(--color-grey-800);
+  --color-border-table-cell: var(--color-grey-400);
+  --color-text-tab-active: var(--color-text);
+  --color-border-input: var(--color-grey-400);
+  --color-shadow-input: var(--color-grey-50);
+  --color-background-input: var(--color-grey-350);
+  --color-text-input-description: var(--color-grey-600);
+  --color-text-input-placeholder: var(--color-grey-600);
+  --color-text-watch-tag-list: #fa3e92;
+  --color-background-code: var(--color-grey-200);
+  --color-background-tab: rgba(0, 0, 0, 0.2);
+  --color-background-tab-hover: rgba(0, 0, 0, 0.5);
+  --color-background-snapshot-age: var(--color-grey-200);
+  --color-shadow-jump: var(--color-grey-200);
+  --color-icon-github: var(--color-white);
+  --color-icon-github-hover: var(--color-grey-700);
+  --color-watch-table-error: var(--color-light-red);
+  --color-watch-table-row-text: var(--color-grey-800); }
+  html[data-darkmode="true"] .icon-spread {
+    filter: hue-rotate(-10deg) brightness(1.5); }
+  html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
+  html[data-darkmode="true"] .watch-table .current-diff-url::after {
+    filter: invert(0.5) hue-rotate(10deg) brightness(2); }
+  html[data-darkmode="true"] .watch-table .watch-controls .state-off img {
+    opacity: 0.3; }
+  html[data-darkmode="true"] .watch-table .watch-controls .state-on img {
+    opacity: 1.0; }
+  html[data-darkmode="true"] .watch-table .unviewed {
+    color: #fff; }
+    html[data-darkmode="true"] .watch-table .unviewed.error {
+      color: var(--color-watch-table-error); }
+
 #diff-ui {
-  background: #fff;
+  background: var(--color-background);
   padding: 2em;
   margin-left: 1em;
   margin-right: 1em;
-  border-radius: 5px;
-  font-size: 11px; }
+  border-radius: 5px; }
+  #diff-ui #text {
+    font-size: 11px; }
   #diff-ui table {
     table-layout: fixed;
     width: 100%; }
@@ -45,6 +183,10 @@ ins {
     margin-left: 1em;
     display: inline-block;
     font-weight: normal; }
+  #settings del {
+    padding: 0.5em; }
+  #settings ins {
+    padding: 0.5em; }
 
 .source {
   position: absolute;

changedetectionio/static/styles/diff.scss

@@ -1,96 +0,0 @@
-#diff-ui {
-
-    background: #fff;
-    padding: 2em;
-    margin-left: 1em;
-    margin-right: 1em;
-    border-radius: 5px;
-    font-size: 11px;
-
-    table {
-        table-layout: fixed;
-        width: 100%;
-    }
-    td {
-        padding: 3px 4px;
-        border: 1px solid transparent;
-        vertical-align: top;
-        font: 1em monospace;
-        text-align: left;
-    }
-    pre {
-            white-space: pre-wrap;
-    }
-}
-h1 {
-	display: inline;
-	font-size: 100%;
-}
-del {
-	text-decoration: none;
-	color: #b30000;
-	background: #fadad7;
-}
-
-ins {
-	background: #eaf2c2;
-	color: #406619;
-	text-decoration: none;
-}
-
-#result {
-	white-space: pre-wrap;
-}
-
-#settings {
-    background: rgba(0,0,0,.05);
-    padding: 1em;
-    border-radius: 10px;
-    margin-bottom: 1em;
-    color: #fff;
-    font-size: 80%;
-    label {
-	    margin-left: 1em;
-	    display: inline-block;
-	    font-weight: normal;
-    }
-}
-
-.source {
-	position: absolute;
-	right: 1%;
-	top: .2em;
-}
-
-@-moz-document url-prefix() {
-	body {
-		height: 99%; /* Hide scroll bar in Firefox */
-	}
-}
-
-td#diff-col div {
-    text-align: justify;
-    white-space: pre-wrap;
-}
-
-.ignored {
-    background-color: #ccc;
-   /*  border: #0d91fa 1px solid; */
-    opacity: 0.7;
-}
-
-.triggered {
-    background-color: #1b98f8;
-}
-
-/* ignored and triggered? make it obvious error */
-.ignored.triggered {
-  background-color: #ff0000;
-}
-
-.tab-pane-inner#screenshot {
-  text-align: center;
-  img {
-    max-width: 99%;
-  }
-}

changedetectionio/static/styles/package.json

@@ -4,7 +4,8 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "build": "node-sass styles.scss -o .;node-sass diff.scss -o ."
+    "watch": "node-sass -w scss -o .",
+    "build": "node-sass scss -o ."
   },
   "author": "",
   "license": "ISC",

changedetectionio/static/styles/scss/diff.scss

@@ -0,0 +1,121 @@
+@import "parts/_variables.scss";
+
+#diff-ui {
+
+  background: var(--color-background);
+  padding: 2em;
+  margin-left: 1em;
+  margin-right: 1em;
+  border-radius: 5px;
+
+  // The first tab 'text' diff
+  #text {
+    font-size: 11px;
+  }
+
+  table {
+    table-layout: fixed;
+    width: 100%;
+  }
+
+  td {
+    padding: 3px 4px;
+    border: 1px solid transparent;
+    vertical-align: top;
+    font: 1em monospace;
+    text-align: left;
+  }
+
+  pre {
+    white-space: pre-wrap;
+  }
+}
+
+h1 {
+  display: inline;
+  font-size: 100%;
+}
+
+del {
+  text-decoration: none;
+  color: #b30000;
+  background: #fadad7;
+}
+
+ins {
+  background: #eaf2c2;
+  color: #406619;
+  text-decoration: none;
+}
+
+#result {
+  white-space: pre-wrap;
+
+  .change {
+    span {}
+  }
+}
+
+#settings {
+  background: rgba(0, 0, 0, .05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+  color: #fff;
+  font-size: 80%;
+
+  label {
+    margin-left: 1em;
+    display: inline-block;
+    font-weight: normal;
+  }
+
+  del {
+    padding: 0.5em;
+  }
+
+  ins {
+    padding: 0.5em;
+  }
+}
+
+.source {
+  position: absolute;
+  right: 1%;
+  top: .2em;
+}
+
+@-moz-document url-prefix() {
+  body {
+    height: 99%;
+    /* Hide scroll bar in Firefox */
+  }
+}
+
+td#diff-col div {
+  text-align: justify;
+  white-space: pre-wrap;
+}
+
+.ignored {
+  background-color: #ccc;
+  /*  border: #0d91fa 1px solid; */
+  opacity: 0.7;
+}
+
+.triggered {
+  background-color: #1b98f8;
+}
+
+/* ignored and triggered? make it obvious error */
+.ignored.triggered {
+  background-color: #ff0000;
+}
+
+.tab-pane-inner#screenshot {
+  text-align: center;
+
+  img {
+    max-width: 99%;
+  }
+}

changedetectionio/static/styles/scss/parts/_browser-steps.scss

@@ -6,6 +6,11 @@
   }
 
   li {
+    &:not(:first-child) {
+      &:hover {
+        opacity: 1.0;
+      }
+    }
     list-style: decimal;
     padding: 5px;
     .control {
@@ -70,6 +75,8 @@
     transform: translate(-50%, -50%);
     margin-left: -40px;
     z-index: 100;
+    max-width: 350px;
+    text-align: center;
   }
 
   /* nice tall skinny one */
@@ -78,4 +85,11 @@
     height: 80px;
     font-size: 3px;
   }
+
+  #browsersteps-click-start {
+    &:hover {
+      cursor: pointer;
+    }
+    color: var(--color-grey-400);
+  }
 }

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -0,0 +1,17 @@
+ul#requests-extra_proxies {
+  list-style: none;
+  /* tidy up the table to look more "inline" */
+  li {
+    > label {
+      display: none;
+    }
+
+  }
+  /* each proxy entry is a `table` */
+  table {
+    tr {
+      display: inline;
+    }
+  }
+}
+

changedetectionio/static/styles/scss/parts/_variables.scss

@@ -0,0 +1,175 @@
+/**
+ * CSS custom properties (aka variables).
+ */
+
+:root {
+  --color-white: #fff;
+  --color-grey-50: #111;
+  --color-grey-100: #262626;
+  --color-grey-200: #333;
+  --color-grey-300: #444;
+  --color-grey-325: #555;
+  --color-grey-350: #565d64;
+  --color-grey-400: #666;
+  --color-grey-500: #777;
+  --color-grey-600: #999;
+  --color-grey-700: #cbcbcb;
+  --color-grey-750: #ddd;
+  --color-grey-800: #e0e0e0;
+  --color-grey-850: #eee;
+  --color-grey-900: #f2f2f2;
+  --color-black: #000;
+  --color-dark-red: #a00;
+  --color-light-red: #dd0000;
+
+  --color-background-page: var(--color-grey-100);
+  --color-background-gradient-first: #5ad8f7;
+  --color-background-gradient-second: #2f50af;
+  --color-background-gradient-third: #9150bf;
+  --color-background: var(--color-white);
+  --color-text: var(--color-grey-200);
+  --color-link: #1b98f8;
+  --color-menu-accent: #ed5900;
+  --color-background-code: var(--color-grey-850);
+  --color-error: var(--color-dark-red);
+  --color-error-input: #ffebeb;
+  --color-error-list: var(--color-light-red);
+  --color-table-background: var(--color-background);
+  --color-table-stripe: var(--color-grey-900);
+  --color-text-tab: var(--color-white);
+  --color-background-tab: rgba(255, 255, 255, 0.2);
+  --color-background-tab-hover: rgba(255, 255, 255, 0.5);
+  --color-text-tab-active: #222;
+  --color-api-key: #0078e7;
+
+  --color-background-button-primary: #0078e7;
+  --color-background-button-green: #42dd53;
+  --color-background-button-red: #dd4242;
+  --color-background-button-success: rgb(28, 184, 65);
+  --color-background-button-error: rgb(202, 60, 60);
+  --color-text-button-error: var(--color-white);
+  --color-background-button-warning: rgb(202, 60, 60);
+  --color-text-button-warning: var(--color-white);
+  --color-background-button-secondary: rgb(66, 184, 221);
+  --color-background-button-cancel: rgb(200, 200, 200);
+  --color-text-button: var(--color-white);
+  --color-background-button-tag: rgb(99, 99, 99);
+  --color-background-snapshot-age: #dfdfdf;
+  --color-error-text-snapshot-age: var(--color-white);
+  --color-error-background-snapshot-age: #ff0000;
+  --color-background-button-tag-active: #9c9c9c;
+
+  --color-text-messages: var(--color-white);
+  --color-background-messages-message: rgba(255, 255, 255, .2);
+  --color-background-messages-error: rgba(255, 1, 1, .5);
+  --color-background-messages-notice: rgba(255, 255, 255, .5);
+  --color-border-notification: #ccc;
+
+  --color-background-checkbox-operations: rgba(0, 0, 0, 0.05);
+  --color-warning: #ff3300;
+  --color-border-warning: var(--color-warning);
+  --color-text-legend: var(--color-white);
+
+  --color-link-new-version: #e07171;
+  --color-last-checked: #bbb;
+  --color-text-footer: #444;
+  --color-border-watch-table-cell: #eee;
+
+  --color-text-watch-tag-list: #e70069;
+  --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
+  --color-background-new-watch-input: var(--color-white);
+  --color-text-new-watch-input: var(--color-text);
+
+  --color-border-input: var(--color-grey-500);
+  --color-shadow-input: var(--color-grey-400);
+  --color-background-input: var(--color-white);
+  --color-text-input: var(--color-text);
+  --color-text-input-description: var(--color-grey-500);
+  --color-text-input-placeholder: var(--color-grey-600);
+
+  --color-background-table-thead: var(--color-grey-800);
+  --color-border-table-cell: var(--color-grey-700);
+
+  --color-text-menu-heading: var(--color-grey-350);
+  --color-text-menu-link: var(--color-grey-500);
+  --color-background-menu-link-hover: var(--color-grey-850);
+  --color-text-menu-link-hover: var(--color-grey-300);
+
+  --color-shadow-jump: var(--color-grey-500);
+  --color-icon-github: var(--color-black);
+  --color-icon-github-hover: var(--color-grey-300);
+
+  --color-watch-table-error: var(--color-dark-red);
+  --color-watch-table-row-text: var(--color-grey-100);
+}
+
+html[data-darkmode="true"] {
+  --color-link: #59bdfb;
+  --color-text: var(--color-white);
+
+  --color-background-gradient-first: #3f90a5;
+  --color-background-gradient-second: #1e316c;
+  --color-background-gradient-third: #4d2c64;
+
+  --color-background-new-watch-input: var(--color-grey-100);
+  --color-text-new-watch-input: var(--color-text);
+  --color-background-table-thead: var(--color-grey-200);
+  --color-table-background: var(--color-grey-300);
+  --color-table-stripe: var(--color-grey-325);
+  --color-background: var(--color-grey-300);
+  --color-text-menu-heading: var(--color-grey-850);
+  --color-text-menu-link: var(--color-grey-800);
+  --color-border-table-cell: var(--color-grey-400);
+  --color-text-tab-active: var(--color-text);
+
+  --color-border-input: var(--color-grey-400);
+  --color-shadow-input: var(--color-grey-50);
+  --color-background-input: var(--color-grey-350);
+  --color-text-input-description: var(--color-grey-600);
+  --color-text-input-placeholder: var(--color-grey-600);
+  --color-text-watch-tag-list: #fa3e92;
+  --color-background-code: var(--color-grey-200);
+
+  --color-background-tab: rgba(0, 0, 0, 0.2);
+  --color-background-tab-hover: rgba(0, 0, 0, 0.5);
+
+  --color-background-snapshot-age: var(--color-grey-200);
+  --color-shadow-jump: var(--color-grey-200);
+  --color-icon-github: var(--color-white);
+  --color-icon-github-hover: var(--color-grey-700);
+  --color-watch-table-error: var(--color-light-red);
+  --color-watch-table-row-text: var(--color-grey-800);
+
+
+  .icon-spread {
+    filter: hue-rotate(-10deg) brightness(1.5);
+  }
+
+  .watch-table {
+
+    .title-col a[target="_blank"]::after,
+    .current-diff-url::after {
+      filter: invert(.5) hue-rotate(10deg) brightness(2);
+    }
+
+    .watch-controls {
+      .state-off {
+        img {
+          opacity: 0.3;
+        }
+      }
+      .state-on {
+        img {
+          opacity: 1.0;
+        }
+      }
+    }
+
+    .unviewed {
+      color: #fff;
+      &.error {
+        color: var(--color-watch-table-error);
+      }
+    }
+  }
+}

changedetectionio/static/styles/scss/styles.scss

@@ -1,42 +1,115 @@
 /*
  * -- BASE STYLES --
- * Most of these are inherited from Base, but I want to change a few.
-nvm use v14.18.1 && npm install && npm run build
- * or npm run watch
  */
 
-@import "parts/spinners";
-@import "parts/browser-steps";
-@import "parts/_arrows.scss";
+@import "parts/_arrows";
+@import "parts/_browser-steps";
+@import "parts/_extra_proxies";
+@import "parts/_spinners";
+@import "parts/_variables";
 
 body {
-  color: #333;
-  background: #262626;
+  color: var(--color-text);
+  background: var(--color-background-page);
+}
+
+.visually-hidden {
+  clip: rect(0 0 0 0);
+  clip-path: inset(50%);
+  height: 1px;
+  overflow: hidden;
+  position: absolute;
+  white-space: nowrap;
+  width: 1px;
+}
+
+// Row icons like chrome, pdf, share, etc
+.status-icon {
+  display: inline-block;
+  height: 1rem;
+  vertical-align: middle;
 }
 
 .pure-table-even {
-  background: #fff;
+  background: var(--color-background);
 }
 
 /* Some styles from https://css-tricks.com/ */
 a {
   text-decoration: none;
-  color: #1b98f8;
+  color: var(--color-link);
 }
 
 a.github-link {
-  color: #fff;
+  color: var(--color-icon-github);
+  margin: 0 1rem 0 0.5rem;
+
+  svg {
+    fill: currentColor;
+  }
+
+  &:hover {
+    color: var(--color-icon-github-hover);
+  }
+}
+
+button.toggle-theme {
+  width: 4rem;
+  background: transparent;
+  border: none;
+  cursor: pointer;
+
+  color: var(--color-icon-github);
+
+  &:hover {
+    color: var(--color-icon-github-hover);
+  }
+
+  svg {
+    fill: currentColor;
+  }
+
+  .icon-light {
+    display: block;
+  }
+
+  .icon-dark {
+    display: none;
+  }
+
+  &.dark {
+    .icon-light {
+      display: none;
+    }
+
+    .icon-dark {
+      display: block;
+    }
+  }
 }
 
 .pure-menu-horizontal {
-  background: #fff;
+  background: var(--color-background);
   padding: 5px;
   display: flex;
   justify-content: space-between;
-  border-bottom: 2px solid #ed5900;
+  border-bottom: 2px solid var(--color-menu-accent);
   align-items: center;
 }
 
+.pure-menu-heading {
+  color: var(--color-text-menu-heading);
+}
+
+.pure-menu-link {
+  color: var(--color-text-menu-link);
+
+  &:hover {
+    background-color: var(--color-background-menu-link-hover);
+    color: var(--color-text-menu-link-hover);
+  }
+}
+
 section.content {
   padding-top: 5em;
   padding-bottom: 1em;
@@ -47,7 +120,8 @@ section.content {
 }
 
 code {
-  background: #eee;
+  background: var(--color-background-code);
+  color: var(--color-text);
 }
 
 /* table related */
@@ -55,30 +129,36 @@ code {
   width: 100%;
   font-size: 80%;
 
-  tr.unviewed {
-    font-weight: bold;
+  tr {
+    &.unviewed {
+      font-weight: bold;
+    }
+    &.error {
+      color: var(--color-watch-table-error);
+    }
+    color: var(--color-watch-table-row-text);
   }
 
-  .error {
-    color: #a00;
-  }
 
   td {
     white-space: nowrap;
+    &.title-col {
+      word-break: break-all;
+      white-space: normal;
+    }
   }
 
-  td.title-col {
-    word-break: break-all;
-    white-space: normal;
-  }
 
   th {
     white-space: nowrap;
+
     a {
       font-weight: normal;
+
       &.active {
         font-weight: bolder;
       }
+
       &.inactive {
         .arrow {
           display: none;
@@ -87,14 +167,15 @@ code {
     }
   }
 
-  .title-col a[target="_blank"]::after, .current-diff-url::after {
+  .title-col a[target="_blank"]::after,
+  .current-diff-url::after {
     content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
     margin: 0 3px 0 5px;
   }
 }
 
 .watch-tag-list {
-  color: #e70069;
+  color: var(--color-text-watch-tag-list);
   white-space: nowrap;
 }
 
@@ -126,10 +207,11 @@ code {
 
 body:after {
   content: "";
-  background: linear-gradient(130deg, #5ad8f7, #2f50af 41.07%, #9150bf 84.05%);
+  background: linear-gradient(130deg, var(--color-background-gradient-first), var(--color-background-gradient-second) 41.07%, var(--color-background-gradient-third) 84.05%);
 }
 
-body:after, body:before {
+body:after,
+body:before {
   display: block;
   height: 650px;
   position: absolute;
@@ -149,7 +231,8 @@ body::before {
   background-size: cover
 }
 
-body:after, body:before {
+body:after,
+body:before {
   -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
   clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
 }
@@ -165,51 +248,57 @@ body:after, body:before {
   display: block;
 }
 
+.pure-button-primary,
+a.pure-button-primary,
+.pure-button-selected,
+a.pure-button-selected {
+  background-color: var(--color-background-button-primary);
+}
 
 .button-secondary {
-  color: white;
+  color: var(--color-text-button);
   border-radius: 4px;
   text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
 }
 
 .button-success {
-  background: rgb(28, 184, 65);
-  /* this is a green */
+  background: var(--color-background-button-success);
 }
 
 .button-tag {
-  background: rgb(99, 99, 99);
-  color: #fff;
+  background: var(--color-background-button-tag);
+  color: var(--color-text-button);
   font-size: 65%;
   border-bottom-left-radius: initial;
   border-bottom-right-radius: initial;
 
   &.active {
-    background: #9c9c9c;
+    background: var(--color-background-button-tag-active);
     font-weight: bold;
   }
 
 }
 
 .button-error {
-  background: rgb(202, 60, 60);
-  /* this is a maroon */
+  background: var(--color-background-button-error);
+  color: var(--color-text-button-error);
 }
 
 .button-warning {
-  background: rgb(223, 117, 20);
-  /* this is an orange */
+  background: var(--color-background-button-warning);
+  color: var(--color-text-button-warning);
 }
 
 .button-secondary {
-  background: rgb(66, 184, 221);
-  /* this is a light blue */
+  background: var(--color-background-button-secondary);
 }
 
-
 .button-cancel {
-  background: rgb(200, 200, 200);
-  /* this is a green */
+  background: var(--color-background-button-cancel);
+}
+
+#save_button {
+  margin-right: 1rem;
 }
 
 .messages {
@@ -217,50 +306,62 @@ body:after, body:before {
     list-style: none;
     padding: 1em;
     border-radius: 10px;
-    color: #fff;
+    color: var(--color-text-messages);
     font-weight: bold;
 
     &.message {
-      background: rgba(255, 255, 255, .2);
+      background: var(--color-background-messages-message);
     }
 
     &.error {
-      background: rgba(255, 1, 1, .5);
+      background: var(--color-background-messages-error);
     }
 
     &.notice {
-      background: rgba(255, 255, 255, .5);
+      background: var(--color-background-messages-notice);
     }
   }
 
   &.with-share-link {
-    > *:hover {
+    >*:hover {
       cursor: pointer;
     }
   }
 }
 
+.notifications-wrapper {
+  padding: 0.5rem 0 1rem 0;
+}
+
+label {
+ &:hover {
+   cursor: pointer;
+ }  
+}
+
 #notification-customisation {
-  border: 1px solid #ccc;
+  border: 1px solid var(--color-border-notification);
   padding: 0.5rem;
   border-radius: 5px;
 }
 
 #notification-error-log {
-  border: 1px solid #ccc;
+  border: 1px solid var(--color-border-notification);
   padding: 1rem;
   border-radius: 5px;
   overflow-wrap: break-word;
 }
 
 #token-table {
-  &.pure-table td, &.pure-table th {
+
+  &.pure-table td,
+  &.pure-table th {
     font-size: 80%;
   }
 }
 
 #new-watch-form {
-  background: rgba(0, 0, 0, .05);
+  background: var(--color-background-new-watch-form);
   padding: 1em;
   border-radius: 10px;
   margin-bottom: 1em;
@@ -270,19 +371,25 @@ body:after, body:before {
     margin-bottom: 5px;
   }
 
+  input:not(.pure-button) {
+    background-color: var(--color-background-new-watch-input);
+    color: var(--color-text-new-watch-input);
+  }
+
   .label {
     display: none;
   }
 
   legend {
-    color: #fff;
+    color: var(--color-text-legend);
     font-weight: bold;
   }
 
   #watch-add-wrapper-zone {
-    > div {
+    >div {
       display: inline-block;
     }
+
     @media only screen and (max-width: 760px) {
       #url {
         width: 100%;
@@ -300,15 +407,15 @@ body:after, body:before {
   position: fixed;
   left: 0px;
   top: 120px;
-  background: #fff;
+  background: var(--color-background);
   padding: 10px;
   border-top-right-radius: 5px;
   border-bottom-right-radius: 5px;
-  box-shadow: 5px 0 5px -2px #888;
+  box-shadow: 1px 1px 4px var(--color-shadow-jump);
 
   a {
-    color: #1b98f8;
-    cursor: grabbing;
+    color: var(--color-link);
+    cursor: pointer;
     -moz-user-select: none;
     -webkit-user-select: none;
     -ms-user-select: none;
@@ -319,8 +426,8 @@ body:after, body:before {
 
 footer {
   padding: 10px;
-  background: #fff;
-  color: #444;
+  background: var(--color-background);
+  color: var(--color-text-footer);
   text-align: center;
 }
 
@@ -343,7 +450,7 @@ footer {
   position: absolute;
   top: 60px;
   font-size: 65%;
-  background: #fff;
+  background: var(--color-background);
   padding: 10px;
 
   &#left-sticky {
@@ -362,10 +469,12 @@ footer {
 }
 
 #new-version-text a {
-  color: #e07171;
+  color: var(--color-link-new-version);
 }
 
 .watch-controls {
+  color: #f8321b;
+
   .state-on {
     img {
       opacity: 0.8;
@@ -383,7 +492,6 @@ footer {
       opacity: 0.8;
     }
   }
-
 }
 
 .monospaced-textarea {
@@ -392,7 +500,8 @@ footer {
     font-family: monospace;
     white-space: pre;
     overflow-wrap: normal;
-    overflow-x: scroll;
+    // No scrollbars until needed.
+    overflow-x: auto;
   }
 }
 
@@ -407,7 +516,9 @@ footer {
     }
   }
 
-  .pure-control-group, .pure-group, .pure-controls {
+  .pure-control-group,
+  .pure-group,
+  .pure-controls {
     padding-bottom: 1em;
 
     div {
@@ -415,28 +526,32 @@ footer {
     }
 
     .checkbox {
-      > * {
+      >* {
         display: inline;
         vertical-align: middle;
       }
 
-      > label {
+      >label {
         padding-left: 5px;
       }
     }
+
+    legend {
+      color: var(--color-text-legend);
+    }
   }
 
   /* The input fields with errors */
   .error {
     input {
-      background-color: #ffebeb;
+      background-color: var(--color-error-input);
     }
   }
 
   /* The list of errors */
   ul.errors {
     padding: .5em .6em;
-    border: 1px solid #dd0000;
+    border: 1px solid var(--color-error-list);
     border-radius: 4px;
     vertical-align: middle;
     -webkit-box-sizing: border-box;
@@ -444,7 +559,7 @@ footer {
 
     li {
       margin-left: 1em;
-      color: #dd0000;
+      color: var(--color-error-list);
     }
   }
 
@@ -462,7 +577,7 @@ footer {
       list-style: none;
 
       li {
-        > * {
+        >* {
           display: inline-block;
         }
       }
@@ -471,21 +586,25 @@ footer {
 }
 
 
-@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+@media only screen and (max-width: 760px),
+(min-device-width: 768px) and (max-device-width: 1024px) {
   .box {
     max-width: 95%
   }
+
   .edit-form {
     padding: 0.5em;
     margin: 0;
   }
+
   #nav-menu {
     overflow-x: scroll;
   }
 }
 
 
-@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 800px) {
+@media only screen and (max-width: 760px),
+(min-device-width: 768px) and (max-device-width: 800px) {
 
   div.sticky-tab#hosted-sticky {
     top: 60px;
@@ -514,24 +633,29 @@ footer {
   and also iPads specifically.
   */
   .watch-table {
+
     /* Force table to not be like tables anymore */
-    thead, tbody, th, td, tr {
+    thead,
+    tbody,
+    th,
+    td,
+    tr {
       display: block;
     }
 
     .last-checked {
-      > span {
+      >span {
         vertical-align: middle;
       }
     }
 
     .last-checked::before {
-      color: #555;
+      color: var(--color-last-checked);
       content: "Last Checked ";
     }
 
     .last-changed::before {
-      color: #555;
+      color: var(--color-last-checked);
       content: "Last Changed ";
     }
 
@@ -547,15 +671,17 @@ footer {
       left: -9999px;
     }
 
-    .pure-table td, .pure-table th {
+    .pure-table td,
+    .pure-table th {
       border: none;
     }
 
     td {
       /* Behave  like a "row" */
       border: none;
-      border-bottom: 1px solid #eee;
+      border-bottom: 1px solid var(--color-border-watch-table-cell);
       vertical-align: middle;
+
       &:before {
         /* Top/left values mimic padding */
         top: 6px;
@@ -568,11 +694,11 @@ footer {
 
     &.pure-table-striped {
       tr {
-        background-color: #fff;
+        background-color: var(--color-table-background);
       }
 
       tr:nth-child(2n-1) {
-        background-color: #eee;
+        background-color: var(--color-table-stripe);
       }
 
       tr:nth-child(2n-1) td {
@@ -583,12 +709,66 @@ footer {
   }
 }
 
+.pure-table {
+  border-color: var(--color-border-table-cell);
+
+  thead {
+    background-color: var(--color-background-table-thead);
+    color: var(--color-text);
+  }
+
+  td,
+  th {
+    border-left-color: var(--color-border-table-cell);
+  }
+}
+
+.pure-table-striped {
+  tr:nth-child(2n-1) {
+    td {
+      background-color: var(--color-table-stripe);
+    }
+  }
+}
+
+.pure-form input[type=color],
+.pure-form input[type=date],
+.pure-form input[type=datetime-local],
+.pure-form input[type=datetime],
+.pure-form input[type=email],
+.pure-form input[type=month],
+.pure-form input[type=number],
+.pure-form input[type=password],
+.pure-form input[type=search],
+.pure-form input[type=tel],
+.pure-form input[type=text],
+.pure-form input[type=time],
+.pure-form input[type=url],
+.pure-form input[type=week],
+.pure-form select,
+.pure-form textarea {
+  border: var(--color-border-input);
+  box-shadow: inset 0 1px 3px var(--color-shadow-input);
+  background-color: var(--color-background-input);
+  color: var(--color-text-input);
+
+  &:active {
+    background-color: var(--color-background-input);
+  }
+}
+
+input::placeholder,
+textarea::placeholder {
+  color: var(--color-text-input-placeholder);
+}
+
 
 /** Desktop vs mobile input field strategy
 - We dont use 'size' with <input> because `size` is too unreliable to override, and will often push-out
 - Rely always on width in CSS
 */
 @media only screen and (min-width: 761px) {
+
   /* m-d is medium-desktop */
   .m-d {
     min-width: 80%;
@@ -605,16 +785,23 @@ footer {
     li {
       margin-right: 3px;
       display: inline-block;
-      color: #fff;
+      color: var(--color-text-tab);
       border-top-left-radius: 5px;
       border-top-right-radius: 5px;
-      background-color: rgba(255, 255, 255, 0.2);
+      background-color: var(--color-background-tab);
 
-      &.active, :target {
-        background-color: #fff;
+      &:not(.active) {
+        &:hover {
+          background-color: var(--color-background-tab-hover);
+        }
+      }
+
+      &.active,
+      :target {
+        background-color: var(--color-background);
 
         a {
-          color: #222;
+          color: var(--color-text-tab-active);
           font-weight: bold;
         }
       }
@@ -622,22 +809,24 @@ footer {
       a {
         display: block;
         padding: 0.8em;
-        color: #fff;
+        color: var(--color-text-tab);
       }
     }
   }
 }
 
 $form-edge-padding: 20px;
+
 .pure-form-stacked {
-  > div:first-child {
+  >div:first-child {
     display: block;
   }
 }
 
 .login-form {
   .inner {
-    background: #fff;;
+    background: var(--color-background);
+    ;
     padding: $form-edge-padding;
     border-radius: 5px;
   }
@@ -667,11 +856,13 @@ $form-edge-padding: 20px;
 #selector-header {
   padding-bottom: 1em;
 }
+
 body.full-width {
   .edit-form {
     width: 95%;
   }
 }
+
 .edit-form {
   min-width: 70%;
   /* so it cant overflow */
@@ -682,17 +873,21 @@ body.full-width {
   }
 
   .inner {
-    background: #fff;;
+    background: var(--color-background);
     padding: $form-edge-padding;
   }
 
   #actions {
     display: block;
-    background: #fff;
+    background: var(--color-background);
   }
 
   .pure-form-message-inline {
     padding-left: 0;
+    color: var(--color-text-input-description);
+    code {
+      font-size: .875em;
+    }
   }
 }
 
@@ -716,14 +911,15 @@ ul {
   height: 100%;
   overflow-y: scroll;
   position: relative;
+
   //width: 100%;
-  > img {
+  >img {
     position: absolute;
     z-index: 4;
     max-width: 100%;
   }
 
-  > canvas {
+  >canvas {
     position: relative;
     z-index: 5;
     max-width: 100%;
@@ -751,54 +947,61 @@ ul {
 }
 
 #api-key-copy {
-  color: #0078e7;
+  color: var(--color-api-key);
 }
 
 .button-green {
-  background-color: #42dd53;
+  background-color: var(--color-background-button-green);
 }
 
 .button-red {
-  background-color: #dd4242;
+  background-color: var(--color-background-button-red);
 }
 
 .noselect {
-  -webkit-touch-callout: none; /* iOS Safari */
-  -webkit-user-select: none; /* Safari */
-  -moz-user-select: none; /* Old versions of Firefox */
-  -ms-user-select: none; /* Internet Explorer/Edge */
+  -webkit-touch-callout: none;
+  /* iOS Safari */
+  -webkit-user-select: none;
+  /* Safari */
+  -moz-user-select: none;
+  /* Old versions of Firefox */
+  -ms-user-select: none;
+  /* Internet Explorer/Edge */
   user-select: none;
   /* Non-prefixed version, currently
-                                   supported by Chrome, Edge, Opera and Firefox */
+    supported by Chrome, Edge, Opera and Firefox */
 }
 
 .snapshot-age {
   padding: 4px;
-  background-color: #dfdfdf;
+  margin: 0.5rem 0;
+  background-color: var(--color-background-snapshot-age);
   border-radius: 3px;
   font-weight: bold;
   margin-bottom: 4px;
+
   &.error {
-    background-color: #ff0000;
-    color: #fff;
+    background-color: var(--color-error-background-snapshot-age);
+    color: var(--color-error-text-snapshot-age);
   }
 }
 
 #checkbox-operations {
-  background: rgba(0, 0, 0, 0.05);
+  background: var(--color-background-checkbox-operations);
   padding: 1em;
   border-radius: 10px;
   margin-bottom: 1em;
   display: none;
 }
+
 .checkbox-uuid {
-  > * {
+  >* {
     vertical-align: middle;
   }
 }
 
 .inline-warning {
-  > span {
+  >span {
     display: inline-block;
     vertical-align: middle;
   }
@@ -809,8 +1012,35 @@ ul {
     vertical-align: middle;
   }
 
-  border: 1px solid #ff3300;
+  border: 1px solid var(--color-border-warning);
   padding: 0.5rem;
   border-radius: 5px;
-  color: #ff3300;
+  color: var(--color-warning);
 }
+
+/* automatic price following helpers */
+.tracking-ldjson-price-data {
+  background-color: var(--color-background-button-green);
+  color: #000;
+  padding: 3px;
+  border-radius: 3px;
+  white-space: nowrap;
+}
+
+.ldjson-price-track-offer {
+  a.pure-button {
+    border-radius: 3px;
+    padding: 3px;
+    background-color: var(--color-background-button-green);
+  }
+
+  font-weight: bold;
+  font-style: italic;
+}
+
+.price-follow-tag-icon {
+  display: inline-block;
+  height: 0.8rem;
+  vertical-align: middle;
+}
+

changedetectionio/static/styles/styles.css

@@ -1,47 +1,23 @@
 /*
  * -- BASE STYLES --
- * Most of these are inherited from Base, but I want to change a few.
-nvm use v14.18.1 && npm install && npm run build
- * or npm run watch
  */
-/* spinner */
-.spinner,
-.spinner:after {
-  border-radius: 50%;
-  width: 10px;
-  height: 10px; }
-
-.spinner {
-  margin: 0px auto;
-  font-size: 3px;
-  vertical-align: middle;
+.arrow {
+  border: solid #1b98f8;
+  border-width: 0 2px 2px 0;
   display: inline-block;
-  text-indent: -9999em;
-  border-top: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-right: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-left: 1.1em solid #2668ed;
-  -webkit-transform: translateZ(0);
-  -ms-transform: translateZ(0);
-  transform: translateZ(0);
-  -webkit-animation: load8 1.1s infinite linear;
-  animation: load8 1.1s infinite linear; }
-
-@-webkit-keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg); }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg); } }
-
-@keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg); }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg); } }
+  padding: 3px; }
+  .arrow.right {
+    transform: rotate(-45deg);
+    -webkit-transform: rotate(-45deg); }
+  .arrow.left {
+    transform: rotate(135deg);
+    -webkit-transform: rotate(135deg); }
+  .arrow.up, .arrow.asc {
+    transform: rotate(-135deg);
+    -webkit-transform: rotate(-135deg); }
+  .arrow.down, .arrow.desc {
+    transform: rotate(45deg);
+    -webkit-transform: rotate(45deg); }
 
 #browser_steps {
   /* convert rows to horizontal cells */ }
@@ -50,6 +26,8 @@ nvm use v14.18.1 && npm install && npm run build
   #browser_steps li {
     list-style: decimal;
     padding: 5px; }
+    #browser_steps li:not(:first-child):hover {
+      opacity: 1.0; }
     #browser_steps li .control {
       padding-left: 5px;
       padding-right: 5px; }
@@ -96,53 +74,273 @@ nvm use v14.18.1 && npm install && npm run build
     top: 50%;
     transform: translate(-50%, -50%);
     margin-left: -40px;
-    z-index: 100; }
+    z-index: 100;
+    max-width: 350px;
+    text-align: center; }
   #browsersteps-selector-wrapper .spinner, #browsersteps-selector-wrapper .spinner:after {
     width: 80px;
     height: 80px;
     font-size: 3px; }
+  #browsersteps-selector-wrapper #browsersteps-click-start {
+    color: var(--color-grey-400); }
+    #browsersteps-selector-wrapper #browsersteps-click-start:hover {
+      cursor: pointer; }
 
-.arrow {
-  border: solid #1b98f8;
-  border-width: 0 2px 2px 0;
+ul#requests-extra_proxies {
+  list-style: none;
+  /* tidy up the table to look more "inline" */
+  /* each proxy entry is a `table` */ }
+  ul#requests-extra_proxies li > label {
+    display: none; }
+  ul#requests-extra_proxies table tr {
+    display: inline; }
+
+/* spinner */
+.spinner,
+.spinner:after {
+  border-radius: 50%;
+  width: 10px;
+  height: 10px; }
+
+.spinner {
+  margin: 0px auto;
+  font-size: 3px;
+  vertical-align: middle;
   display: inline-block;
-  padding: 3px; }
-  .arrow.right {
-    transform: rotate(-45deg);
-    -webkit-transform: rotate(-45deg); }
-  .arrow.left {
-    transform: rotate(135deg);
-    -webkit-transform: rotate(135deg); }
-  .arrow.up, .arrow.asc {
-    transform: rotate(-135deg);
-    -webkit-transform: rotate(-135deg); }
-  .arrow.down, .arrow.desc {
-    transform: rotate(45deg);
-    -webkit-transform: rotate(45deg); }
+  text-indent: -9999em;
+  border-top: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-right: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-left: 1.1em solid #2668ed;
+  -webkit-transform: translateZ(0);
+  -ms-transform: translateZ(0);
+  transform: translateZ(0);
+  -webkit-animation: load8 1.1s infinite linear;
+  animation: load8 1.1s infinite linear; }
+
+@-webkit-keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg); }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg); } }
+
+@keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg); }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg); } }
+
+/**
+ * CSS custom properties (aka variables).
+ */
+:root {
+  --color-white: #fff;
+  --color-grey-50: #111;
+  --color-grey-100: #262626;
+  --color-grey-200: #333;
+  --color-grey-300: #444;
+  --color-grey-325: #555;
+  --color-grey-350: #565d64;
+  --color-grey-400: #666;
+  --color-grey-500: #777;
+  --color-grey-600: #999;
+  --color-grey-700: #cbcbcb;
+  --color-grey-750: #ddd;
+  --color-grey-800: #e0e0e0;
+  --color-grey-850: #eee;
+  --color-grey-900: #f2f2f2;
+  --color-black: #000;
+  --color-dark-red: #a00;
+  --color-light-red: #dd0000;
+  --color-background-page: var(--color-grey-100);
+  --color-background-gradient-first: #5ad8f7;
+  --color-background-gradient-second: #2f50af;
+  --color-background-gradient-third: #9150bf;
+  --color-background: var(--color-white);
+  --color-text: var(--color-grey-200);
+  --color-link: #1b98f8;
+  --color-menu-accent: #ed5900;
+  --color-background-code: var(--color-grey-850);
+  --color-error: var(--color-dark-red);
+  --color-error-input: #ffebeb;
+  --color-error-list: var(--color-light-red);
+  --color-table-background: var(--color-background);
+  --color-table-stripe: var(--color-grey-900);
+  --color-text-tab: var(--color-white);
+  --color-background-tab: rgba(255, 255, 255, 0.2);
+  --color-background-tab-hover: rgba(255, 255, 255, 0.5);
+  --color-text-tab-active: #222;
+  --color-api-key: #0078e7;
+  --color-background-button-primary: #0078e7;
+  --color-background-button-green: #42dd53;
+  --color-background-button-red: #dd4242;
+  --color-background-button-success: rgb(28, 184, 65);
+  --color-background-button-error: rgb(202, 60, 60);
+  --color-text-button-error: var(--color-white);
+  --color-background-button-warning: rgb(202, 60, 60);
+  --color-text-button-warning: var(--color-white);
+  --color-background-button-secondary: rgb(66, 184, 221);
+  --color-background-button-cancel: rgb(200, 200, 200);
+  --color-text-button: var(--color-white);
+  --color-background-button-tag: rgb(99, 99, 99);
+  --color-background-snapshot-age: #dfdfdf;
+  --color-error-text-snapshot-age: var(--color-white);
+  --color-error-background-snapshot-age: #ff0000;
+  --color-background-button-tag-active: #9c9c9c;
+  --color-text-messages: var(--color-white);
+  --color-background-messages-message: rgba(255, 255, 255, .2);
+  --color-background-messages-error: rgba(255, 1, 1, .5);
+  --color-background-messages-notice: rgba(255, 255, 255, .5);
+  --color-border-notification: #ccc;
+  --color-background-checkbox-operations: rgba(0, 0, 0, 0.05);
+  --color-warning: #ff3300;
+  --color-border-warning: var(--color-warning);
+  --color-text-legend: var(--color-white);
+  --color-link-new-version: #e07171;
+  --color-last-checked: #bbb;
+  --color-text-footer: #444;
+  --color-border-watch-table-cell: #eee;
+  --color-text-watch-tag-list: #e70069;
+  --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
+  --color-background-new-watch-input: var(--color-white);
+  --color-text-new-watch-input: var(--color-text);
+  --color-border-input: var(--color-grey-500);
+  --color-shadow-input: var(--color-grey-400);
+  --color-background-input: var(--color-white);
+  --color-text-input: var(--color-text);
+  --color-text-input-description: var(--color-grey-500);
+  --color-text-input-placeholder: var(--color-grey-600);
+  --color-background-table-thead: var(--color-grey-800);
+  --color-border-table-cell: var(--color-grey-700);
+  --color-text-menu-heading: var(--color-grey-350);
+  --color-text-menu-link: var(--color-grey-500);
+  --color-background-menu-link-hover: var(--color-grey-850);
+  --color-text-menu-link-hover: var(--color-grey-300);
+  --color-shadow-jump: var(--color-grey-500);
+  --color-icon-github: var(--color-black);
+  --color-icon-github-hover: var(--color-grey-300);
+  --color-watch-table-error: var(--color-dark-red);
+  --color-watch-table-row-text: var(--color-grey-100); }
+
+html[data-darkmode="true"] {
+  --color-link: #59bdfb;
+  --color-text: var(--color-white);
+  --color-background-gradient-first: #3f90a5;
+  --color-background-gradient-second: #1e316c;
+  --color-background-gradient-third: #4d2c64;
+  --color-background-new-watch-input: var(--color-grey-100);
+  --color-text-new-watch-input: var(--color-text);
+  --color-background-table-thead: var(--color-grey-200);
+  --color-table-background: var(--color-grey-300);
+  --color-table-stripe: var(--color-grey-325);
+  --color-background: var(--color-grey-300);
+  --color-text-menu-heading: var(--color-grey-850);
+  --color-text-menu-link: var(--color-grey-800);
+  --color-border-table-cell: var(--color-grey-400);
+  --color-text-tab-active: var(--color-text);
+  --color-border-input: var(--color-grey-400);
+  --color-shadow-input: var(--color-grey-50);
+  --color-background-input: var(--color-grey-350);
+  --color-text-input-description: var(--color-grey-600);
+  --color-text-input-placeholder: var(--color-grey-600);
+  --color-text-watch-tag-list: #fa3e92;
+  --color-background-code: var(--color-grey-200);
+  --color-background-tab: rgba(0, 0, 0, 0.2);
+  --color-background-tab-hover: rgba(0, 0, 0, 0.5);
+  --color-background-snapshot-age: var(--color-grey-200);
+  --color-shadow-jump: var(--color-grey-200);
+  --color-icon-github: var(--color-white);
+  --color-icon-github-hover: var(--color-grey-700);
+  --color-watch-table-error: var(--color-light-red);
+  --color-watch-table-row-text: var(--color-grey-800); }
+  html[data-darkmode="true"] .icon-spread {
+    filter: hue-rotate(-10deg) brightness(1.5); }
+  html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
+  html[data-darkmode="true"] .watch-table .current-diff-url::after {
+    filter: invert(0.5) hue-rotate(10deg) brightness(2); }
+  html[data-darkmode="true"] .watch-table .watch-controls .state-off img {
+    opacity: 0.3; }
+  html[data-darkmode="true"] .watch-table .watch-controls .state-on img {
+    opacity: 1.0; }
+  html[data-darkmode="true"] .watch-table .unviewed {
+    color: #fff; }
+    html[data-darkmode="true"] .watch-table .unviewed.error {
+      color: var(--color-watch-table-error); }
 
 body {
-  color: #333;
-  background: #262626; }
+  color: var(--color-text);
+  background: var(--color-background-page); }
+
+.visually-hidden {
+  clip: rect(0 0 0 0);
+  clip-path: inset(50%);
+  height: 1px;
+  overflow: hidden;
+  position: absolute;
+  white-space: nowrap;
+  width: 1px; }
+
+.status-icon {
+  display: inline-block;
+  height: 1rem;
+  vertical-align: middle; }
 
 .pure-table-even {
-  background: #fff; }
+  background: var(--color-background); }
 
 /* Some styles from https://css-tricks.com/ */
 a {
   text-decoration: none;
-  color: #1b98f8; }
+  color: var(--color-link); }
 
 a.github-link {
-  color: #fff; }
+  color: var(--color-icon-github);
+  margin: 0 1rem 0 0.5rem; }
+  a.github-link svg {
+    fill: currentColor; }
+  a.github-link:hover {
+    color: var(--color-icon-github-hover); }
+
+button.toggle-theme {
+  width: 4rem;
+  background: transparent;
+  border: none;
+  cursor: pointer;
+  color: var(--color-icon-github); }
+  button.toggle-theme:hover {
+    color: var(--color-icon-github-hover); }
+  button.toggle-theme svg {
+    fill: currentColor; }
+  button.toggle-theme .icon-light {
+    display: block; }
+  button.toggle-theme .icon-dark {
+    display: none; }
+  button.toggle-theme.dark .icon-light {
+    display: none; }
+  button.toggle-theme.dark .icon-dark {
+    display: block; }
 
 .pure-menu-horizontal {
-  background: #fff;
+  background: var(--color-background);
   padding: 5px;
   display: flex;
   justify-content: space-between;
-  border-bottom: 2px solid #ed5900;
+  border-bottom: 2px solid var(--color-menu-accent);
   align-items: center; }
 
+.pure-menu-heading {
+  color: var(--color-text-menu-heading); }
+
+.pure-menu-link {
+  color: var(--color-text-menu-link); }
+  .pure-menu-link:hover {
+    background-color: var(--color-background-menu-link-hover);
+    color: var(--color-text-menu-link-hover); }
+
 section.content {
   padding-top: 5em;
   padding-bottom: 1em;
@@ -152,21 +350,24 @@ section.content {
   justify-content: center; }
 
 code {
-  background: #eee; }
+  background: var(--color-background-code);
+  color: var(--color-text); }
 
 /* table related */
 .watch-table {
   width: 100%;
   font-size: 80%; }
-  .watch-table tr.unviewed {
-    font-weight: bold; }
-  .watch-table .error {
-    color: #a00; }
+  .watch-table tr {
+    color: var(--color-watch-table-row-text); }
+    .watch-table tr.unviewed {
+      font-weight: bold; }
+    .watch-table tr.error {
+      color: var(--color-watch-table-error); }
   .watch-table td {
     white-space: nowrap; }
-  .watch-table td.title-col {
-    word-break: break-all;
-    white-space: normal; }
+    .watch-table td.title-col {
+      word-break: break-all;
+      white-space: normal; }
   .watch-table th {
     white-space: nowrap; }
     .watch-table th a {
@@ -175,12 +376,13 @@ code {
         font-weight: bolder; }
       .watch-table th a.inactive .arrow {
         display: none; }
-  .watch-table .title-col a[target="_blank"]::after, .watch-table .current-diff-url::after {
+  .watch-table .title-col a[target="_blank"]::after,
+  .watch-table .current-diff-url::after {
     content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
     margin: 0 3px 0 5px; }
 
 .watch-tag-list {
-  color: #e70069;
+  color: var(--color-text-watch-tag-list);
   white-space: nowrap; }
 
 .box {
@@ -203,9 +405,10 @@ code {
 
 body:after {
   content: "";
-  background: linear-gradient(130deg, #5ad8f7, #2f50af 41.07%, #9150bf 84.05%); }
+  background: linear-gradient(130deg, var(--color-background-gradient-first), var(--color-background-gradient-second) 41.07%, var(--color-background-gradient-third) 84.05%); }
 
-body:after, body:before {
+body:after,
+body:before {
   display: block;
   height: 650px;
   position: absolute;
@@ -221,7 +424,8 @@ body::before {
   content: "";
   background-size: cover; }
 
-body:after, body:before {
+body:after,
+body:before {
   -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
   clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%); }
 
@@ -234,83 +438,99 @@ body:after, body:before {
   max-width: 400px;
   display: block; }
 
+.pure-button-primary,
+a.pure-button-primary,
+.pure-button-selected,
+a.pure-button-selected {
+  background-color: var(--color-background-button-primary); }
+
 .button-secondary {
-  color: white;
+  color: var(--color-text-button);
   border-radius: 4px;
   text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2); }
 
 .button-success {
-  background: #1cb841;
-  /* this is a green */ }
+  background: var(--color-background-button-success); }
 
 .button-tag {
-  background: #636363;
-  color: #fff;
+  background: var(--color-background-button-tag);
+  color: var(--color-text-button);
   font-size: 65%;
   border-bottom-left-radius: initial;
   border-bottom-right-radius: initial; }
   .button-tag.active {
-    background: #9c9c9c;
+    background: var(--color-background-button-tag-active);
     font-weight: bold; }
 
 .button-error {
-  background: #ca3c3c;
-  /* this is a maroon */ }
+  background: var(--color-background-button-error);
+  color: var(--color-text-button-error); }
 
 .button-warning {
-  background: #df7514;
-  /* this is an orange */ }
+  background: var(--color-background-button-warning);
+  color: var(--color-text-button-warning); }
 
 .button-secondary {
-  background: #42b8dd;
-  /* this is a light blue */ }
+  background: var(--color-background-button-secondary); }
 
 .button-cancel {
-  background: #c8c8c8;
-  /* this is a green */ }
+  background: var(--color-background-button-cancel); }
+
+#save_button {
+  margin-right: 1rem; }
 
 .messages li {
   list-style: none;
   padding: 1em;
   border-radius: 10px;
-  color: #fff;
+  color: var(--color-text-messages);
   font-weight: bold; }
   .messages li.message {
-    background: rgba(255, 255, 255, 0.2); }
+    background: var(--color-background-messages-message); }
   .messages li.error {
-    background: rgba(255, 1, 1, 0.5); }
+    background: var(--color-background-messages-error); }
   .messages li.notice {
-    background: rgba(255, 255, 255, 0.5); }
+    background: var(--color-background-messages-notice); }
 
 .messages.with-share-link > *:hover {
   cursor: pointer; }
 
+.notifications-wrapper {
+  padding: 0.5rem 0 1rem 0; }
+
+label:hover {
+  cursor: pointer; }
+
 #notification-customisation {
-  border: 1px solid #ccc;
+  border: 1px solid var(--color-border-notification);
   padding: 0.5rem;
   border-radius: 5px; }
 
 #notification-error-log {
-  border: 1px solid #ccc;
+  border: 1px solid var(--color-border-notification);
   padding: 1rem;
   border-radius: 5px;
   overflow-wrap: break-word; }
 
-#token-table.pure-table td, #token-table.pure-table th {
+#token-table.pure-table td,
+#token-table.pure-table th {
   font-size: 80%; }
 
 #new-watch-form {
-  background: rgba(0, 0, 0, 0.05);
+  background: var(--color-background-new-watch-form);
   padding: 1em;
   border-radius: 10px;
   margin-bottom: 1em; }
   #new-watch-form input {
     display: inline-block;
     margin-bottom: 5px; }
+  #new-watch-form input:not(.pure-button) {
+    background-color: var(--color-background-new-watch-input);
+    color: var(--color-text-new-watch-input); }
   #new-watch-form .label {
     display: none; }
   #new-watch-form legend {
-    color: #fff;
+    color: var(--color-text-legend);
     font-weight: bold; }
   #new-watch-form #watch-add-wrapper-zone > div {
     display: inline-block; }
@@ -325,14 +545,14 @@ body:after, body:before {
   position: fixed;
   left: 0px;
   top: 120px;
-  background: #fff;
+  background: var(--color-background);
   padding: 10px;
   border-top-right-radius: 5px;
   border-bottom-right-radius: 5px;
-  box-shadow: 5px 0 5px -2px #888; }
+  box-shadow: 1px 1px 4px var(--color-shadow-jump); }
   #diff-jump a {
-    color: #1b98f8;
-    cursor: grabbing;
+    color: var(--color-link);
+    cursor: pointer;
     -moz-user-select: none;
     -webkit-user-select: none;
     -ms-user-select: none;
@@ -341,8 +561,8 @@ body:after, body:before {
 
 footer {
   padding: 10px;
-  background: #fff;
-  color: #444;
+  background: var(--color-background);
+  color: var(--color-text-footer);
   text-align: center; }
 
 #feed-icon {
@@ -361,7 +581,7 @@ footer {
   position: absolute;
   top: 60px;
   font-size: 65%;
-  background: #fff;
+  background: var(--color-background);
   padding: 10px; }
   .sticky-tab#left-sticky {
     left: 0px; }
@@ -373,9 +593,10 @@ footer {
     font-weight: bold; }
 
 #new-version-text a {
-  color: #e07171; }
+  color: var(--color-link-new-version); }
 
 .watch-controls {
+  color: #f8321b;
   /* default */ }
   .watch-controls .state-on img {
     opacity: 0.8; }
@@ -390,7 +611,7 @@ footer {
   font-family: monospace;
   white-space: pre;
   overflow-wrap: normal;
-  overflow-x: scroll; }
+  overflow-x: auto; }
 
 .pure-form {
   /* The input fields with errors */
@@ -400,27 +621,39 @@ footer {
     .pure-form fieldset ul {
       padding-bottom: 0px;
       margin-bottom: 0px; }
-  .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
+  .pure-form .pure-control-group,
+  .pure-form .pure-group,
+  .pure-form .pure-controls {
     padding-bottom: 1em; }
-    .pure-form .pure-control-group div, .pure-form .pure-group div, .pure-form .pure-controls div {
+    .pure-form .pure-control-group div,
+    .pure-form .pure-group div,
+    .pure-form .pure-controls div {
       margin: 0px; }
-    .pure-form .pure-control-group .checkbox > *, .pure-form .pure-group .checkbox > *, .pure-form .pure-controls .checkbox > * {
+    .pure-form .pure-control-group .checkbox > *,
+    .pure-form .pure-group .checkbox > *,
+    .pure-form .pure-controls .checkbox > * {
       display: inline;
       vertical-align: middle; }
-    .pure-form .pure-control-group .checkbox > label, .pure-form .pure-group .checkbox > label, .pure-form .pure-controls .checkbox > label {
+    .pure-form .pure-control-group .checkbox > label,
+    .pure-form .pure-group .checkbox > label,
+    .pure-form .pure-controls .checkbox > label {
       padding-left: 5px; }
+    .pure-form .pure-control-group legend,
+    .pure-form .pure-group legend,
+    .pure-form .pure-controls legend {
+      color: var(--color-text-legend); }
   .pure-form .error input {
-    background-color: #ffebeb; }
+    background-color: var(--color-error-input); }
   .pure-form ul.errors {
     padding: .5em .6em;
-    border: 1px solid #dd0000;
+    border: 1px solid var(--color-error-list);
     border-radius: 4px;
     vertical-align: middle;
     -webkit-box-sizing: border-box;
     box-sizing: border-box; }
     .pure-form ul.errors li {
       margin-left: 1em;
-      color: #dd0000; }
+      color: var(--color-error-list); }
   .pure-form label {
     font-weight: bold; }
   .pure-form textarea {
@@ -462,15 +695,19 @@ footer {
     /* Force table to not be like tables anymore */
     /* Force table to not be like tables anymore */
     /* Hide table headers (but not display: none;, for accessibility) */ }
-    .watch-table thead, .watch-table tbody, .watch-table th, .watch-table td, .watch-table tr {
+    .watch-table thead,
+    .watch-table tbody,
+    .watch-table th,
+    .watch-table td,
+    .watch-table tr {
       display: block; }
     .watch-table .last-checked > span {
       vertical-align: middle; }
     .watch-table .last-checked::before {
-      color: #555;
+      color: var(--color-last-checked);
       content: "Last Checked "; }
     .watch-table .last-changed::before {
-      color: #555;
+      color: var(--color-last-checked);
       content: "Last Changed "; }
     .watch-table td.inline {
       display: inline-block; }
@@ -478,12 +715,13 @@ footer {
       position: absolute;
       top: -9999px;
       left: -9999px; }
-    .watch-table .pure-table td, .watch-table .pure-table th {
+    .watch-table .pure-table td,
+    .watch-table .pure-table th {
       border: none; }
     .watch-table td {
       /* Behave  like a "row" */
       border: none;
-      border-bottom: 1px solid #eee;
+      border-bottom: 1px solid var(--color-border-watch-table-cell);
       vertical-align: middle; }
       .watch-table td:before {
         /* Top/left values mimic padding */
@@ -493,12 +731,66 @@ footer {
         padding-right: 10px;
         white-space: nowrap; }
     .watch-table.pure-table-striped tr {
-      background-color: #fff; }
+      background-color: var(--color-table-background); }
     .watch-table.pure-table-striped tr:nth-child(2n-1) {
-      background-color: #eee; }
+      background-color: var(--color-table-stripe); }
     .watch-table.pure-table-striped tr:nth-child(2n-1) td {
       background-color: inherit; } }
 
+.pure-table {
+  border-color: var(--color-border-table-cell); }
+  .pure-table thead {
+    background-color: var(--color-background-table-thead);
+    color: var(--color-text); }
+  .pure-table td,
+  .pure-table th {
+    border-left-color: var(--color-border-table-cell); }
+
+.pure-table-striped tr:nth-child(2n-1) td {
+  background-color: var(--color-table-stripe); }
+
+.pure-form input[type=color],
+.pure-form input[type=date],
+.pure-form input[type=datetime-local],
+.pure-form input[type=datetime],
+.pure-form input[type=email],
+.pure-form input[type=month],
+.pure-form input[type=number],
+.pure-form input[type=password],
+.pure-form input[type=search],
+.pure-form input[type=tel],
+.pure-form input[type=text],
+.pure-form input[type=time],
+.pure-form input[type=url],
+.pure-form input[type=week],
+.pure-form select,
+.pure-form textarea {
+  border: var(--color-border-input);
+  box-shadow: inset 0 1px 3px var(--color-shadow-input);
+  background-color: var(--color-background-input);
+  color: var(--color-text-input); }
+  .pure-form input[type=color]:active,
+  .pure-form input[type=date]:active,
+  .pure-form input[type=datetime-local]:active,
+  .pure-form input[type=datetime]:active,
+  .pure-form input[type=email]:active,
+  .pure-form input[type=month]:active,
+  .pure-form input[type=number]:active,
+  .pure-form input[type=password]:active,
+  .pure-form input[type=search]:active,
+  .pure-form input[type=tel]:active,
+  .pure-form input[type=text]:active,
+  .pure-form input[type=time]:active,
+  .pure-form input[type=url]:active,
+  .pure-form input[type=week]:active,
+  .pure-form select:active,
+  .pure-form textarea:active {
+    background-color: var(--color-background-input); }
+
+input::placeholder,
+textarea::placeholder {
+  color: var(--color-text-input-placeholder); }
+
 /** Desktop vs mobile input field strategy
 - We dont use 'size' with <input> because `size` is too unreliable to override, and will often push-out
 - Rely always on width in CSS
@@ -515,25 +807,29 @@ footer {
   .tabs ul li {
     margin-right: 3px;
     display: inline-block;
-    color: #fff;
+    color: var(--color-text-tab);
     border-top-left-radius: 5px;
     border-top-right-radius: 5px;
-    background-color: rgba(255, 255, 255, 0.2); }
-    .tabs ul li.active, .tabs ul li :target {
-      background-color: #fff; }
-      .tabs ul li.active a, .tabs ul li :target a {
-        color: #222;
+    background-color: var(--color-background-tab); }
+    .tabs ul li:not(.active):hover {
+      background-color: var(--color-background-tab-hover); }
+    .tabs ul li.active,
+    .tabs ul li :target {
+      background-color: var(--color-background); }
+      .tabs ul li.active a,
+      .tabs ul li :target a {
+        color: var(--color-text-tab-active);
         font-weight: bold; }
     .tabs ul li a {
       display: block;
       padding: 0.8em;
-      color: #fff; }
+      color: var(--color-text-tab); }
 
 .pure-form-stacked > div:first-child {
   display: block; }
 
 .login-form .inner {
-  background: #fff;
+  background: var(--color-background);
   padding: 20px;
   border-radius: 5px; }
 
@@ -563,13 +859,16 @@ body.full-width .edit-form {
   .edit-form .box-wrap {
     position: relative; }
   .edit-form .inner {
-    background: #fff;
+    background: var(--color-background);
     padding: 20px; }
   .edit-form #actions {
     display: block;
-    background: #fff; }
+    background: var(--color-background); }
   .edit-form .pure-form-message-inline {
-    padding-left: 0; }
+    padding-left: 0;
+    color: var(--color-text-input-description); }
+    .edit-form .pure-form-message-inline code {
+      font-size: .875em; }
 
 ul {
   padding-left: 1em;
@@ -606,13 +905,13 @@ ul {
   cursor: pointer; }
 
 #api-key-copy {
-  color: #0078e7; }
+  color: var(--color-api-key); }
 
 .button-green {
-  background-color: #42dd53; }
+  background-color: var(--color-background-button-green); }
 
 .button-red {
-  background-color: #dd4242; }
+  background-color: var(--color-background-button-red); }
 
 .noselect {
   -webkit-touch-callout: none;
@@ -625,20 +924,21 @@ ul {
   /* Internet Explorer/Edge */
   user-select: none;
   /* Non-prefixed version, currently
-                                   supported by Chrome, Edge, Opera and Firefox */ }
+    supported by Chrome, Edge, Opera and Firefox */ }
 
 .snapshot-age {
   padding: 4px;
-  background-color: #dfdfdf;
+  margin: 0.5rem 0;
+  background-color: var(--color-background-snapshot-age);
   border-radius: 3px;
   font-weight: bold;
   margin-bottom: 4px; }
   .snapshot-age.error {
-    background-color: #ff0000;
-    color: #fff; }
+    background-color: var(--color-error-background-snapshot-age);
+    color: var(--color-error-text-snapshot-age); }
 
 #checkbox-operations {
-  background: rgba(0, 0, 0, 0.05);
+  background: var(--color-background-checkbox-operations);
   padding: 1em;
   border-radius: 10px;
   margin-bottom: 1em;
@@ -648,10 +948,10 @@ ul {
   vertical-align: middle; }
 
 .inline-warning {
-  border: 1px solid #ff3300;
+  border: 1px solid var(--color-border-warning);
   padding: 0.5rem;
   border-radius: 5px;
-  color: #ff3300; }
+  color: var(--color-warning); }
   .inline-warning > span {
     display: inline-block;
     vertical-align: middle; }
@@ -659,3 +959,24 @@ ul {
     display: inline;
     height: 26px;
     vertical-align: middle; }
+
+/* automatic price following helpers */
+.tracking-ldjson-price-data {
+  background-color: var(--color-background-button-green);
+  color: #000;
+  padding: 3px;
+  border-radius: 3px;
+  white-space: nowrap; }
+
+.ldjson-price-track-offer {
+  font-weight: bold;
+  font-style: italic; }
+  .ldjson-price-track-offer a.pure-button {
+    border-radius: 3px;
+    padding: 3px;
+    background-color: var(--color-background-button-green); }
+
+.price-follow-tag-icon {
+  display: inline-block;
+  height: 0.8rem;
+  vertical-align: middle; }

changedetectionio/store.py

@@ -1,20 +1,20 @@
 from flask import (
     flash
 )
-import json
-import logging
-import os
-import threading
-import time
-import uuid as uuid_builder
+
+from . model import App, Watch
 from copy import deepcopy
 from os import path, unlink
 from threading import Lock
+import json
+import logging
+import os
 import re
 import requests
 import secrets
-
-from . model import App, Watch
+import threading
+import time
+import uuid as uuid_builder
 
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
@@ -36,7 +36,6 @@ class ChangeDetectionStore:
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
         self.needs_write = False
-        self.proxy_list = None
         self.start_time = time.time()
         self.stop_thread = False
         # Base definition for all watchers
@@ -78,10 +77,10 @@ class ChangeDetectionStore:
                     self.__data['watching'][uuid] = Watch.model(datastore_path=self.datastore_path, default=watch)
                     print("Watching:", uuid, self.__data['watching'][uuid]['url'])
 
-        # First time ran, doesnt exist.
-        except (FileNotFoundError, json.decoder.JSONDecodeError):
+        # First time ran, Create the datastore.
+        except (FileNotFoundError):
             if include_default_watches:
-                print("Creating JSON store at", self.datastore_path)
+                print("No JSON DB found at {}, creating JSON store at {}".format(self.json_store_path, self.datastore_path))
                 self.add_watch(url='https://news.ycombinator.com/',
                                tag='Tech news',
                                extras={'fetch_backend': 'html_requests'})
@@ -89,9 +88,11 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://changedetection.io/CHANGELOG.txt',
                                tag='changedetection.io',
                                extras={'fetch_backend': 'html_requests'})
-
         self.__data['version_tag'] = version_tag
 
+        # Just to test that proxies.json if it exists, doesnt throw a parsing error on startup
+        test_list = self.proxy_list
+
         # Helper to remove password protection
         password_reset_lockfile = "{}/removepassword.lock".format(self.datastore_path)
         if path.isfile(password_reset_lockfile):
@@ -116,11 +117,6 @@ class ChangeDetectionStore:
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['api_access_token'] = secret
 
-        # Proxy list support - available as a selection in settings when text file is imported
-        proxy_list_file = "{}/proxies.json".format(self.datastore_path)
-        if path.isfile(proxy_list_file):
-            self.import_proxy_list(proxy_list_file)
-
         # Bump the update version by running updates
         self.run_updates()
 
@@ -175,14 +171,6 @@ class ChangeDetectionStore:
 
     @property
     def data(self):
-        has_unviewed = False
-        for uuid, watch in self.__data['watching'].items():
-            # #106 - Be sure this is None on empty string, False, None, etc
-            # Default var for fetch_backend
-            # @todo this may not be needed anymore, or could be easily removed
-            if not self.__data['watching'][uuid]['fetch_backend']:
-                self.__data['watching'][uuid]['fetch_backend'] = self.__data['settings']['application']['fetch_backend']
-
         # Re #152, Return env base_url if not overriden, @todo also prefer the proxy pass url
         env_base_url = os.getenv('BASE_URL','')
         if not self.__data['settings']['application']['base_url']:
@@ -250,12 +238,15 @@ class ChangeDetectionStore:
     def clear_watch_history(self, uuid):
         import pathlib
 
-        self.__data['watching'][uuid].update(
-            {'last_checked': 0,
-             'last_viewed': 0,
-             'previous_md5': False,
-             'last_notification_error': False,
-             'last_error': False})
+        self.__data['watching'][uuid].update({
+                'last_checked': 0,
+                'has_ldjson_price_data': None,
+                'last_error': False,
+                'last_notification_error': False,
+                'last_viewed': 0,
+                'previous_md5': False,
+                'track_ldjson_price_data': None,
+            })
 
         # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
         for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"):
@@ -289,6 +280,7 @@ class ChangeDetectionStore:
                 # List of permissible attributes we accept from the wild internet
                 for k in [
                     'body',
+                    'browser_steps',
                     'css_filter',
                     'extract_text',
                     'extract_title_as_title',
@@ -317,9 +309,12 @@ class ChangeDetectionStore:
                 logging.error("Error fetching metadata for shared watch link", url, str(e))
                 flash("Error fetching metadata for {}".format(url), 'error')
                 return False
+        from .model.Watch import is_safe_url
+        if not is_safe_url(url):
+            flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error')
+            return None
 
         with self.lock:
-
             # #Re 569
             new_watch = Watch.model(datastore_path=self.datastore_path, default={
                 'url': url,
@@ -459,10 +454,30 @@ class ChangeDetectionStore:
                     print ("Removing",item)
                     unlink(item)
 
-    def import_proxy_list(self, filename):
-        with open(filename) as f:
-            self.proxy_list = json.load(f)
-            print ("Registered proxy list", list(self.proxy_list.keys()))
+    @property
+    def proxy_list(self):
+        proxy_list = {}
+        proxy_list_file = os.path.join(self.datastore_path, 'proxies.json')
+
+        # Load from external config file
+        if path.isfile(proxy_list_file):
+            with open("{}/proxies.json".format(self.datastore_path)) as f:
+                proxy_list = json.load(f)
+
+        # Mapping from UI config if available
+        extras = self.data['settings']['requests'].get('extra_proxies')
+        if extras:
+            i=0
+            for proxy in extras:
+                i += 0
+                if proxy.get('proxy_name') and proxy.get('proxy_url'):
+                    k = "ui-" + str(i) + proxy.get('proxy_name')
+                    proxy_list[k] = {'label': proxy.get('proxy_name'), 'url': proxy.get('proxy_url')}
+
+
+        return proxy_list if len(proxy_list) else None
+
+
 
 
     def get_preferred_proxy_for_watch(self, uuid):
@@ -472,11 +487,10 @@ class ChangeDetectionStore:
         :return: proxy "key" id
         """
 
-        proxy_id = None
         if self.proxy_list is None:
             return None
 
-        # If its a valid one
+        # If it's a valid one
         watch = self.data['watching'].get(uuid)
 
         if watch.get('proxy') and watch.get('proxy') in list(self.proxy_list.keys()):
@@ -489,8 +503,9 @@ class ChangeDetectionStore:
             if self.proxy_list.get(system_proxy_id):
                 return system_proxy_id
 
-        # Fallback - Did not resolve anything, use the first available
-        if system_proxy_id is None:
+
+        # Fallback - Did not resolve anything, or doesnt exist, use the first available
+        if system_proxy_id is None or not self.proxy_list.get(system_proxy_id):
             first_default = list(self.proxy_list)[0]
             return first_default
 
@@ -620,4 +635,54 @@ class ChangeDetectionStore:
                     watch['include_filters'] = [existing_filter]
             except:
                 continue
-        return
+        return
+
+    # Convert old static notification tokens to jinja2 tokens
+    def update_9(self):
+        # Each watch
+        import re
+        # only { } not {{ or }}
+        r = r'(?<!{){(?!{)(\w+)(?<!})}(?!})'
+        for uuid, watch in self.data['watching'].items():
+            try:
+                n_body = watch.get('notification_body', '')
+                if n_body:
+                    watch['notification_body'] = re.sub(r, r'{{\1}}', n_body)
+
+                n_title = watch.get('notification_title')
+                if n_title:
+                    watch['notification_title'] = re.sub(r, r'{{\1}}', n_title)
+
+                n_urls = watch.get('notification_urls')
+                if n_urls:
+                    for i, url in enumerate(n_urls):
+                        watch['notification_urls'][i] = re.sub(r, r'{{\1}}', url)
+
+            except:
+                continue
+
+        # System wide
+        n_body = self.data['settings']['application'].get('notification_body')
+        if n_body:
+            self.data['settings']['application']['notification_body'] = re.sub(r, r'{{\1}}', n_body)
+
+        n_title = self.data['settings']['application'].get('notification_title')
+        if n_body:
+            self.data['settings']['application']['notification_title'] = re.sub(r, r'{{\1}}', n_title)
+
+        n_urls =  self.data['settings']['application'].get('notification_urls')
+        if n_urls:
+            for i, url in enumerate(n_urls):
+                self.data['settings']['application']['notification_urls'][i] = re.sub(r, r'{{\1}}', url)
+
+        return
+
+    # Some setups may have missed the correct default, so it shows the wrong config in the UI, although it will default to system-wide
+    def update_10(self):
+        for uuid, watch in self.data['watching'].items():
+            try:
+                if not watch.get('fetch_backend', ''):
+                    watch['fetch_backend'] = 'system'
+            except:
+                continue
+        return

changedetectionio/templates/_common_fields.jinja

@@ -16,14 +16,16 @@
                                 <li><code>discord://</code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
                                 <li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
                                 <li><code>tgram://</code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
+                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
                               </ul>
                             </div>
-                            <br/>
-                            <a id="send-test-notification" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Send test notification</a>
-{% if emailprefix %}
-                            <a id="add-email-helper" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Add email</a>
-{% endif %}
-                            <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Notification debug logs</a>
+                            <div class="notifications-wrapper">
+                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Send test notification</a>
+                            {% if emailprefix %}
+                              <a id="add-email-helper" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Add email</a>
+                            {% endif %}
+                              <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Notification debug logs</a>
+                            </div>
                         </div>
                         <div id="notification-customisation" class="pure-control-group">
                             <div class="pure-control-group">
@@ -40,8 +42,9 @@
                                 <span class="pure-form-message-inline">Format for all notifications</span>
                             </div>
                             <div class="pure-controls">
-                            <span class="pure-form-message-inline">
-                                These tokens can be used in the notification body and title to customise the notification text.
+                            <p class="pure-form-message-inline">
+                                You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL.
+                            </p>
 
                                 <table class="pure-table" id="token-table">
                                     <thead>
@@ -52,52 +55,49 @@
                                     </thead>
                                     <tbody>
                                     <tr>
-                                        <td><code>{base_url}</code></td>
+                                        <td><code>{{ '{{ base_url }}' }}</code></td>
                                         <td>The URL of the changedetection.io instance you are running.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{watch_url}</code></td>
+                                        <td><code>{{ '{{ watch_url }}' }}</code></td>
                                         <td>The URL being watched.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{watch_uuid}</code></td>
+                                        <td><code>{{ '{{ watch_uuid }}' }}</code></td>
                                         <td>The UUID of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{watch_title}</code></td>
+                                        <td><code>{{ '{{ watch_title }}' }}</code></td>
                                         <td>The title of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{watch_tag}</code></td>
-                                        <td>The tag of the watch.</td>
+                                        <td><code>{{ '{{ watch_tag }}' }}</code></td>
+                                        <td>The watch label / tag</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{preview_url}</code></td>
+                                        <td><code>{{ '{{ preview_url }}' }}</code></td>
                                         <td>The URL of the preview page generated by changedetection.io.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{diff}</code></td>
+                                        <td><code>{{ '{{ diff_url }}' }}</code></td>
                                         <td>The diff output - differences only</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{diff_full}</code></td>
+                                        <td><code>{{ '{{ diff_full }}' }}</code></td>
                                         <td>The diff output - full difference output</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{diff_url}</code></td>
-                                        <td>The URL of the diff page generated by changedetection.io.</td>
-                                    </tr>
-                                    <tr>
-                                        <td><code>{current_snapshot}</code></td>
+                                        <td><code>{{ '{{ current_snapshot }}' }}</code></td>
                                         <td>The current snapshot value, useful when combined with JSON or CSS filters
                                         </td>
                                     </tr>
                                     </tbody>
                                 </table>
-                                <br/>
-                                URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
-                                Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
-                            </span>
+                                <div class="pure-form-message-inline">
+                                    <br>
+                                    URLs generated by changedetection.io (such as <code>{{ '{{ diff_url }}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
+                                    Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
+                                </div>
                             </div>
                         </div>
 {% endmacro %}

changedetectionio/templates/base.html

@@ -1,110 +1,152 @@
-<!doctype html>
-<html lang="en">
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <meta name="description" content="Self hosted website change detection.">
+<!DOCTYPE html>
+<html lang="en" data-darkmode="{{ get_darkmode_state() }}">
+
+  <head>
+    <meta charset="utf-8"/>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+    <meta name="description" content="Self hosted website change detection."/>
     <title>Change Detection{{extra_title}}</title>
-    <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag %}- {{active_tag}}{% endif %}" href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}" />
-    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}">
-    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}">
+    <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag %}- {{active_tag}}{% endif %}" href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"/>
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}"/>
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}"/>
     {% if extra_stylesheets %}
-        {% for m in extra_stylesheets %}
-        <link rel="stylesheet" href="{{ m }}?ver=1000">
-        {% endfor %}
+      {% for m in extra_stylesheets %}
+        <link rel="stylesheet" href="{{ m }}?ver=1000"/>
+      {% endfor %}
     {% endif %}
+
+    <link rel="apple-touch-icon" sizes="180x180" href="{{url_for('static_content', group='favicons', filename='apple-touch-icon.png')}}"/>
+    <link rel="icon" type="image/png" sizes="32x32" href="{{url_for('static_content', group='favicons', filename='favicon-32x32.png')}}"/>
+    <link rel="icon" type="image/png" sizes="16x16" href="{{url_for('static_content', group='favicons', filename='favicon-16x16.png')}}"/>
+    <link rel="manifest" href="{{url_for('static_content', group='favicons', filename='site.webmanifest')}}"/>
+    <link rel="mask-icon" href="{{url_for('static_content', group='favicons', filename='safari-pinned-tab.svg')}}" color="#5bbad5"/>
+    <link rel="shortcut icon" href="{{url_for('static_content', group='favicons', filename='favicon.ico')}}"/>
+    <meta name="msapplication-TileColor" content="#da532c"/>
+    <meta name="msapplication-config" content="favicons/browserconfig.xml"/>
+    <meta name="theme-color" content="#ffffff"/>
+
     <style>
-    body::before {
-        background-image: url({{url_for('static_content', group='images', filename='gradient-border.png')}});
-    }
+      body::before {
+        background-image: url({{url_for('static_content', group='images', filename='gradient-border.png') }});
+      }
     </style>
     <script type="text/javascript" src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
+  </head>
 
-</head>
-
-<body>
-
-<div class="header">
-
-    <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
+  <body>
+    <div class="header">
+      <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
         {% if has_password and not current_user.is_authenticated %}
-            <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener"><strong>Change</strong>Detection.io</a>
+          <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener">
+            <strong>Change</strong>Detection.io</a>
         {% else %}
-            <a class="pure-menu-heading" href="{{url_for('index')}}"><strong>Change</strong>Detection.io</a>
+          <a class="pure-menu-heading" href="{{url_for('index')}}">
+            <strong>Change</strong>Detection.io</a>
         {% endif %}
         {% if current_diff_url %}
-        <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</span></a>
+          <a class="current-diff-url" href="{{ current_diff_url }}">
+            <span style="max-width: 30%; overflow: hidden">{{ current_diff_url }}</span></a>
         {% else %}
-        {% if new_version_available and not (has_password and not current_user.is_authenticated) %}
-        <span id="new-version-text" class="pure-menu-heading"><a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a></span>
-        {% endif %}
+          {% if new_version_available and not(has_password and not current_user.is_authenticated) %}
+            <span id="new-version-text" class="pure-menu-heading">
+              <a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a>
+            </span>
+          {% endif %}
         {% endif %}
 
-        <ul class="pure-menu-list"  id="top-right-menu">
-        {% if current_user.is_authenticated or not has_password %}
-            {% if not current_diff_url %}
-            <li class="pure-menu-item">
+        <ul class="pure-menu-list" id="top-right-menu">
+          {% if current_user.is_authenticated or not has_password %}
+            {% if not
+            current_diff_url %}
+              <li class="pure-menu-item">
                 <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
-            </li>
-            <li class="pure-menu-item">
+              </li>
+              <li class="pure-menu-item">
                 <a href="{{ url_for('import_page')}}" class="pure-menu-link">IMPORT</a>
-            </li>
-            <li class="pure-menu-item">
+              </li>
+              <li class="pure-menu-item">
                 <a href="{{ url_for('get_backup')}}" class="pure-menu-link">BACKUP</a>
-            </li>
+              </li>
             {% else %}
-            <li class="pure-menu-item">
+              <li class="pure-menu-item">
                 <a href="{{ url_for('edit_page', uuid=uuid, next='diff') }}" class="pure-menu-link">EDIT</a>
-            </li>
+              </li>
             {% endif %}
-        {% else %}
+          {% else %}
             <li class="pure-menu-item">
-                <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
+              <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
             </li>
-        {% endif %}
-
-        {% if current_user.is_authenticated %}
-            <li class="pure-menu-item"><a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a></li>
-        {% endif %}
-            <li class="pure-menu-item"><a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
-                <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16"
-                     version="1.1"
-                     width="32" aria-hidden="true">
-                    <path fill-rule="evenodd"
-                          d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
-                </svg>
-            </a></li>
+          {% endif %}
+          {% if current_user.is_authenticated %}
+            <li class="pure-menu-item">
+              <a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a>
+            </li>
+          {% endif %}
+          <li class="pure-menu-item">
+            {% if dark_mode %}
+            {% set darkClass = 'dark' %}
+            {% endif %}
+            <button class="toggle-theme {{darkClass}}" type="button" title="Toggle Light/Dark Mode">
+              <span class="visually-hidden">Toggle light/dark mode</span>
+              <span class="icon-light">
+                {% include "svgs/light-mode-toggle-icon.svg" %}
+              </span>
+              <span class="icon-dark">
+                {% include "svgs/dark-mode-toggle-icon.svg" %}
+              </span>
+            </button>
+          </li>
+          <li class="pure-menu-item">
+            <a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
+              {% include "svgs/github.svg" %}
+            </a>
+          </li>
         </ul>
+      </div>
     </div>
-</div>
-{% if hosted_sticky %}<div class="sticky-tab" id="hosted-sticky"><a href="https://lemonade.changedetection.io/start?ref={{guid}}">Let us host your instance!</a></div>{% endif %}
-{% if left_sticky %}<div class="sticky-tab" id="left-sticky"><a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a></div> {% endif %}
-{% if right_sticky %}<div class="sticky-tab" id="right-sticky">{{ right_sticky }}</div> {% endif %}
-<section class="content">
-    <header>
+    {% if hosted_sticky %}
+      <div class="sticky-tab" id="hosted-sticky">
+        <a href="https://lemonade.changedetection.io/start?ref={{guid}}">Let us host your instance!</a>
+      </div>
+    {% endif %}
+    {% if left_sticky %}
+      <div class="sticky-tab" id="left-sticky">
+        <a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a>
+      </div>
+    {% endif %}
+    {% if right_sticky %}
+      <div class="sticky-tab" id="right-sticky">{{ right_sticky }}</div>
+    {% endif %}
+    <section class="content">
+      <header>
         {% block header %}{% endblock %}
-    </header>
+      </header>
 
-    {% with messages = get_flashed_messages(with_categories=true) %}
-      {% if messages %}
-        <ul class=messages>
-        {% for category, message in messages %}
-          <li class="{{ category }}">{{ message }}</li>
-        {% endfor %}
+      {% with messages = get_flashed_messages(with_categories = true) %}
+      {% if
+      messages %}
+        <ul class="messages">
+          {% for category, message in messages %}
+            <li class="{{ category }}">{{ message }}</li>
+          {% endfor %}
         </ul>
       {% endif %}
-    {% endwith %}
-
-    {% if session['share-link'] %}
+      {% endwith %}
+      {% if session['share-link'] %}
         <ul class="messages with-share-link">
-          <li class="message">Share this link: <span id="share-link">{{ session['share-link'] }}</span> <img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='copy.svg')}}" /></li>
+          <li class="message">
+            Share this link:
+            <span id="share-link">{{ session['share-link'] }}</span>
+            <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='copy.svg')}}"/>
+          </li>
         </ul>
-    {% endif %}
+      {% endif %}
+      {% block content %}{% endblock %}
+    </section>
+    <script
+      type="text/javascript"
+      src="{{url_for('static_content', group='js', filename='toggle-theme.js')}}"
+      defer></script>
+  </body>
 
-    {% block content %}
-
-    {% endblock %}
-</section>
-
-</body>
 </html>

changedetectionio/templates/clear_all_history.html

@@ -1,32 +1,49 @@
-{% extends 'base.html' %}
-
-{% block content %}
+{% extends 'base.html' %} {% block content %}
 <div class="edit-form">
-    <div class="box-wrap inner">
-    <form class="pure-form pure-form-stacked" action="{{url_for('clear_all_history')}}" method="POST">
-        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
-        <fieldset>
-            <div class="pure-control-group">
-                This will remove version history (snapshots) for ALL watches, but keep your list of URLs! <br/>
-                You may like to use the <strong>BACKUP</strong> link first.<br/>
-            </div>
-            <br/>
-            <div class="pure-control-group">
-                <label for="confirmtext">Confirmation text</label>
-                <input type="text" id="confirmtext" required="" name="confirmtext" value="" size="10"/>
-                <span class="pure-form-message-inline">Type in the word <strong>clear</strong> to confirm that you understand.</span>
-            </div>
-            <br/>
-            <div class="pure-control-group">
-                <button type="submit" class="pure-button pure-button-primary">Clear History!</button>
-            </div>
-            <br/>
-            <div class="pure-control-group">
-                <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Cancel</a>
-            </div>
-        </fieldset>
+  <div class="box-wrap inner">
+    <form
+      class="pure-form pure-form-stacked"
+      action="{{url_for('clear_all_history')}}"
+      method="POST"
+    >
+      <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
+      <fieldset>
+        <div class="pure-control-group">
+          This will remove version history (snapshots) for ALL watches, but keep
+          your list of URLs! <br />
+          You may like to use the <strong>BACKUP</strong> link first.<br />
+        </div>
+        <br />
+        <div class="pure-control-group">
+          <label for="confirmtext">Confirmation text</label>
+          <input
+            type="text"
+            id="confirmtext"
+            required=""
+            name="confirmtext"
+            value=""
+            size="10"
+          />
+          <span class="pure-form-message-inline"
+            >Type in the word <strong>clear</strong> to confirm that you
+            understand.</span
+          >
+        </div>
+        <br />
+        <div class="pure-control-group">
+          <button type="submit" class="pure-button pure-button-primary">
+            Clear History!
+          </button>
+        </div>
+        <br />
+        <div class="pure-control-group">
+          <a href="{{url_for('index')}}" class="pure-button button-cancel"
+            >Cancel</a
+          >
+        </div>
+      </fieldset>
     </form>
-    </div>
+  </div>
 </div>
 
 {% endblock %}

changedetectionio/templates/diff.html

@@ -1,5 +1,5 @@
 {% extends 'base.html' %}
-
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% block content %}
 <script>
     const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
@@ -58,6 +58,7 @@
         {% if last_error_screenshot %}<li class="tab" id="error-screenshot-tab"><a href="#error-screenshot">Error Screenshot</a></li> {% endif %}
         <li class="tab" id=""><a href="#text">Text</a></li>
         <li class="tab" id="screenshot-tab"><a href="#screenshot">Screenshot</a></li>
+        <li class="tab" id="extract-tab"><a href="#extract">Extract Data</a></li>
     </ul>
 </div>
 
@@ -75,8 +76,12 @@
     </div>
 
      <div class="tab-pane-inner" id="text">
-         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.
-         </div>
+         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.</div>
+
+         {% if password_enabled_and_share_is_off %}
+           <div class="tip">Pro-tip: You can enable <strong>"share access when password is enabled"</strong> from settings</div>
+         {% endif %}
+
          <div class="snapshot-age">{{watch_a.snapshot_text_ctime|format_timestamp_timeago}}</div>
 
          <table>
@@ -108,6 +113,37 @@
            <strong>Screenshot requires Playwright/WebDriver enabled</strong>
          {% endif %}
      </div>
+    <div class="tab-pane-inner" id="extract">
+        <form id="extract-data-form" class="pure-form pure-form-stacked edit-form"
+              action="{{ url_for('diff_history_page', uuid=uuid) }}#extract"
+              method="POST">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+
+            <p>This tool will extract text data from all of the watch history.</p>
+
+            <div class="pure-control-group">
+                {{ render_field(extract_form.extract_regex) }}
+                <span class="pure-form-message-inline">
+                    A <strong>RegEx</strong> is a pattern that identifies exactly which part inside of the text that you want to extract.<br/>
+
+                    <p>
+                        For example, to extract only the numbers from text &dash;</br>
+                        <strong>Raw text</strong>: <code>Temperature <span style="color: red">5.5</span>°C in Sydney</code></br>
+                        <strong>RegEx to extract:</strong> <code>Temperature <span style="color: red">([0-9\.]+)</span></code><br/>
+                    </p>
+                    <p>
+                        <a href="https://RegExr.com/">Be sure to test your RegEx here.</a>
+                    </p>
+                    <p>
+                        Each RegEx group bracket <code>()</code> will be in its own column, the first column value is always the date.
+                    </p>
+                </span>
+            </div>
+            <div class="pure-control-group">
+                {{ render_button(extract_form.extract_submit_button) }}
+            </div>
+        </form>
+    </div>
 </div>
 
 <script>

changedetectionio/templates/edit.html

@@ -166,8 +166,12 @@ User-Agent: wonderbra 1.0") }}
                             <div id="browser-steps-ui" class="noselect"  style="width: 100%; background-color: #eee; border-radius: 5px;">
 
                                 <div class="noselect"  id="browsersteps-selector-wrapper" style="width: 100%">
-                                    <span class="loader">
-                                        <div class="spinner"></div>
+                                    <span class="loader" >
+                                        <span id="browsersteps-click-start">
+                                            <h2 >Click here to Start</h2>
+                                            Please allow 10-15 seconds for the browser to connect.
+                                        </span>
+                                        <div class="spinner"  style="display: none;"></div>
                                     </span>
                                     <img  class="noselect" id="browsersteps-img" src="" style="max-width: 100%; width: 100%;" />
                                     <canvas  class="noselect" id="browsersteps-selector-canvas" style="max-width: 100%; width: 100%;"></canvas>
@@ -240,6 +244,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                           <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br/>
                         {% endif %}
                         <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br/>
+
                     <ul>
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
                         <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).

changedetectionio/templates/settings.html

@@ -21,6 +21,7 @@
             <li class="tab"><a href="#fetching">Fetching</a></li>
             <li class="tab"><a href="#filters">Global Filters</a></li>
             <li class="tab"><a href="#api">API</a></li>
+            <li class="tab"><a href="#proxies">CAPTCHA &amp; Proxies</a></li>
         </ul>
     </div>
     <div class="box-wrap inner">
@@ -56,11 +57,16 @@
                         {% endif %}
                     </div>
 
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.application.form.shared_diff_access, class="shared_diff_access") }}
+                        <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page)
+                        </span>
+                    </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
                         class="m-d") }}
                         <span class="pure-form-message-inline">
-                            Base URL used for the <code>{base_url}</code> token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"),
+                            Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"),
                             <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
                         </span>
                     </div>
@@ -170,14 +176,29 @@ nav
                     </div>
                 </div>
             </div>
+            <div class="tab-pane-inner" id="proxies">
 
+                <p><strong>Tip</strong>: You can connect to websites using <a href="https://brightdata.grsm.io/n0r16zf7eivq">BrightData</a> proxies, their service <strong>WebUnlocker</strong> will solve most CAPTCHAs, whilst their <strong>Residential Proxies</strong> may help to avoid CAPTCHA altogether. </p>
+                <p>It may be easier to try <strong>WebUnlocker</strong> first, WebUnlocker also supports country selection.</p>
+                <p>
+                    When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the example URL below<br/>
+                    The Proxy URL with BrightData should start with <code>http://brd-customer...</code>
+                </p>
+
+                <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p>
+
+
+                <div class="pure-control-group">
+                    {{ render_field(form.requests.form.extra_proxies) }}
+                    <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span>
+                </div>
+            </div>
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
                     <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
                     <a href="{{url_for('clear_all_history')}}" class="pure-button button-small button-cancel">Clear Snapshot History</a>
                 </div>
-
             </div>
         </form>
     </div>

changedetectionio/templates/svgs/dark-mode-toggle-icon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" shape-rendering="geometricPrecision" text-rendering="geometricPrecision" image-rendering="optimizeQuality" fill-rule="evenodd" clip-rule="evenodd" viewBox="0 0 512 262.86"><path fill-rule="nonzero" d="M316.78 16.55h-205.9c-30.5 0-58.22 12.48-78.31 32.57C12.47 69.21 0 96.93 0 127.44c0 30.5 12.47 58.22 32.57 78.31 20.09 20.1 47.81 32.57 78.31 32.57h193.25c21.54 15.43 47.9 24.54 76.26 24.54h.18c36.14 0 69.02-14.79 92.83-38.6 23.8-23.81 38.6-56.67 38.6-92.83 0-36.15-14.78-69.03-38.63-92.8C449.53 14.8 416.67 0 380.57 0h-.18c-23.02 0-44.72 6.02-63.61 16.55zm70.62 97.17.43.09c.82-3.45 2.83-6.19 6.04-8.16 3.2-1.98 6.53-2.57 10.01-1.75l.1-.43c-3.47-.82-6.2-2.83-8.17-6.03-1.98-3.22-2.57-6.55-1.75-10.01l-.43-.1c-.82 3.47-2.83 6.2-6.03 8.18-3.21 1.98-6.55 2.56-10.02 1.74l-.1.43c3.47.82 6.2 2.84 8.18 6.04 1.99 3.19 2.56 6.52 1.74 10zm36.87 16.77.53.12c1.02-4.35 3.55-7.78 7.58-10.26 4.02-2.49 8.2-3.22 12.56-2.19l.13-.53c-4.35-1.03-7.78-3.55-10.26-7.59-2.49-4.03-3.22-8.22-2.2-12.56l-.53-.12c-1.02 4.35-3.55 7.77-7.58 10.26-4.02 2.49-8.21 3.22-12.56 2.19l-.13.53c4.36 1.03 7.78 3.55 10.26 7.58 2.49 4.02 3.22 8.22 2.2 12.57zm-38.79-61.01c-15.69 7.67-26.98 23.26-28.29 41.93-1.96 27.88 19.05 52.06 46.92 54.02 13.23.93 25.64-3.32 35.22-11.02 4.75-3.82 9.66-.45 7.59 4.36-11.33 26.42-38.45 44.04-68.74 41.91-38.29-2.69-67.14-35.91-64.45-74.19C316.3 89.8 347.05 61.67 383.44 62c6.71.06 8.13 4.5 2.04 7.48zm-5.09-53.95h.18c63.75 0 115.91 52.15 115.91 115.9 0 63.75-52.23 115.91-115.91 115.91h-.18c-63.68 0-115.91-52.16-115.91-115.91s52.16-115.9 115.91-115.9z"/></svg>

changedetectionio/templates/svgs/github.svg

@@ -0,0 +1,3 @@
+<svg class="octicon octicon-mark-github v-align-middle" height="32" viewbox="0 0 16 16" version="1.1" width="32" aria-hidden="true">
+  <path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
+</svg>

changedetectionio/templates/svgs/light-mode-toggle-icon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" shape-rendering="geometricPrecision" text-rendering="geometricPrecision" image-rendering="optimizeQuality" fill-rule="evenodd" clip-rule="evenodd" viewBox="0 0 512 256.04"><path d="M128.02 0h.18c22.03 0 42.83 5.66 61 15.6h210.38c30.89 0 59 12.65 79.38 33.04C499.35 68.99 512 97.1 512 128.02c0 30.92-12.66 59.03-33.02 79.4l-.42.38c-20.34 20.15-48.29 32.64-78.98 32.64H189.24c-18.17 9.93-38.98 15.6-61.04 15.6h-.18c-35.2 0-67.22-14.41-90.42-37.6C14.41 195.25 0 163.24 0 128.02s14.4-67.24 37.59-90.43l.91-.83C61.65 14.05 93.29 0 128.02 0zm-5.95 54.42c0-1.95.8-3.73 2.08-5 2.74-2.77 7.27-2.76 10.02-.01l.14.16a7.042 7.042 0 0 1 1.94 4.85v12.95c0 1.95-.8 3.73-2.08 5.01-2.75 2.75-7.27 2.75-10.02 0a7.084 7.084 0 0 1-2.08-5.01V54.42zm6.05 31.17c11.72 0 22.32 4.75 30 12.43 7.67 7.68 12.43 18.29 12.43 30 0 11.72-4.75 22.32-12.43 30s-18.28 12.43-30 12.43c-11.72 0-22.32-4.75-30.01-12.43-7.67-7.68-12.43-18.28-12.43-30 0-11.72 4.76-22.32 12.43-30 7.69-7.67 18.3-12.43 30.01-12.43zm-56.33-5.34a7.114 7.114 0 0 1-2.07-5.01c0-3.9 3.18-7.09 7.09-7.09 1.81 0 3.62.69 5 2.07l9.16 9.16a7.065 7.065 0 0 1 2.08 5.01c0 1.8-.7 3.62-2.08 5.01a7.057 7.057 0 0 1-5.01 2.08c-1.8 0-3.61-.7-5-2.07l-9.17-9.16zm-17.28 53.81c-1.95 0-3.73-.8-5-2.08-2.77-2.74-2.76-7.27-.01-10.01l.15-.14a7.04 7.04 0 0 1 4.86-1.94h12.94a7.082 7.082 0 0 1 7.09 7.09c0 1.95-.8 3.73-2.07 5.01a7.099 7.099 0 0 1-5.02 2.07H54.51zm25.82 50.28a7.049 7.049 0 0 1-5 2.07c-3.91 0-7.09-3.16-7.09-7.08 0-1.81.68-3.62 2.07-5.01l9.31-9.29a7.02 7.02 0 0 1 4.86-1.94 7.09 7.09 0 0 1 7.09 7.09c0 1.79-.69 3.6-2.08 4.99l-9.16 9.17zm53.82 17.29c0 1.94-.8 3.73-2.08 5-2.74 2.76-7.27 2.75-10.02 0l-.13-.15a7.033 7.033 0 0 1-1.94-4.85v-12.95c0-1.96.8-3.73 2.07-5.01 2.76-2.75 7.27-2.75 10.03 0a7.1 7.1 0 0 1 2.07 5.01v12.95zm50.28-25.83a7.055 7.055 0 0 1 2.07 5.01c0 3.89-3.18 7.09-7.08 7.09-1.81 0-3.63-.69-5.01-2.07l-9.16-9.16a7.095 7.095 0 0 1-2.07-5.02c0-3.9 3.18-7.09 7.08-7.09 1.8 0 3.61.7 5 2.08l9.17 9.16zm17.29-53.82c1.93 0 3.73.81 5 2.08 2.76 2.75 2.75 7.27 0 10.02l-.15.14a7.098 7.098 0 0 1-4.85 1.94h-12.95c-1.96 0-3.74-.8-5.01-2.08-2.76-2.75-2.76-7.27 0-10.02a7.049 7.049 0 0 1 5.01-2.08h12.95zM175.89 71.7a7.074 7.074 0 0 1 5-2.07c3.9 0 7.1 3.19 7.1 7.09 0 1.81-.69 3.62-2.07 5l-9.32 9.31a7.12 7.12 0 0 1-4.86 1.93c-3.91 0-7.09-3.18-7.09-7.09 0-1.8.7-3.61 2.08-5l9.16-9.17zm34.17-41.87c2.96 2.47 5.81 5.07 8.53 7.8 23.22 23.15 37.63 55.17 37.63 90.39s-14.42 67.23-37.6 90.42a130.2 130.2 0 0 1-8.5 7.77h189.46c26.83 0 51.24-10.91 69.02-28.5l.32-.35c17.79-17.79 28.85-42.35 28.85-69.34 0-26.99-11.06-51.55-28.85-69.35-17.77-17.8-42.33-28.84-69.34-28.84H210.06zm-82.04-14.71h.18c62.09 0 112.89 50.81 112.89 112.9 0 62.1-50.86 112.9-112.89 112.9h-.18c-62.03 0-112.9-50.8-112.9-112.9 0-62.09 50.81-112.9 112.9-112.9z"/></svg>

changedetectionio/templates/watch-overview.html

@@ -14,7 +14,7 @@
             <div id="watch-add-wrapper-zone">
                 <div>
                     {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch group") }}
+                    {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch label / tag") }}
                 </div>
                 <div>
                     {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
@@ -32,6 +32,7 @@
         <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="unpause">UnPause</button>
         <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="mute">Mute</button>
         <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="unmute">UnMute</button>
+        <button class="pure-button button-secondary button-xsmall" style="font-size: 70%" name="op" value="recheck">Recheck</button>
         <button class="pure-button button-secondary button-xsmall" style="font-size: 70%" name="op" value="notification-default">Use default notification</button>
         <button class="pure-button button-secondary button-xsmall" style="background: #dd4242; font-size: 70%" name="op" value="delete">Delete</button>
     </div>
@@ -80,24 +81,44 @@
                 <td class="inline checkbox-uuid" ><input name="uuids"  type="checkbox" value="{{ watch.uuid}} "/> <span>{{ loop.index }}</span></td>
                 <td class="inline watch-controls">
                     {% if not watch.paused %}
-                    <a class="state-off" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks"/></a>
+                    <a class="state-off" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause"/></a>
                     {% else %}
-                    <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks"/></a>
+                    <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause"/></a>
                     {% endif %}
-                    <a class="state-{{'on' if watch.notification_muted}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications"/></a>
+                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute"/></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
-                    <a href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
+                    <a class="link-spread" href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img class="status-icon"  src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="status-icon icon icon-spread" title="Create a link to share watch config with others" /></a>
 
-                    {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
+                    {% if watch.get_fetch_backend == "html_webdriver"
+                         or (  watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
+                    %}
+                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a chrome browser" />
+                    {% endif %}
 
+                    {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" />{% endif %}
                     {% if watch.last_error is defined and watch.last_error != False %}
-                    <div class="fetch-error">{{ watch.last_error }}</div>
+                    <div class="fetch-error">{{ watch.last_error }}
+
+                        {% if '403' in watch.last_error %}
+                            {% if has_proxies %}
+                                <a href="{{ url_for('settings_page', uuid=watch.uuid) }}#proxies">Try other proxies/location</a>&nbsp;
+                            {% endif %}
+                            <a href="{{ url_for('settings_page', uuid=watch.uuid) }}#proxies">Try adding external proxies/locations</a>
+                        
+                        {% endif %}
+                    </div>
                     {% endif %}
                     {% if watch.last_notification_error is defined and watch.last_notification_error != False %}
                     <div class="fetch-error notification-error"><a href="{{url_for('notification_logs')}}">{{ watch.last_notification_error }}</a></div>
                     {% endif %}
+                    {% if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data']  %}
+                    <div class="ldjson-price-track-offer">Embedded price data detected, follow only price data? <a href="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">Yes</a> <a href="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="">No</a></div>
+                    {% endif %}
+                    {% if watch['track_ldjson_price_data'] == 'accepted' %}
+                    <span class="tracking-ldjson-price-data" title="Automatically following embedded price information"><img src="{{url_for('static_content', group='images', filename='price-tag-icon.svg')}}"  class="status-icon price-follow-tag-icon"/> Price</span>
+                    {% endif %}
                     {% if not active_tag %}
                     <span class="watch-tag-list">{{ watch.tag}}</span>
                     {% endif %}
@@ -111,13 +132,13 @@
                 </td>
                 <td>
                     <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
-                       class="recheck pure-button button-small pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
-                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
+                       class="recheck pure-button pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button pure-button-primary">Edit</a>
                     {% if watch.history_n >= 2 %}
-                    <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary diff-link">Diff</a>
+                    <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">Diff</a>
                     {% else %}
                         {% if watch.history_n == 1 or (watch.history_n ==0 and watch.error_text_ctime )%}
-                            <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
+                            <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary">Preview</a>
                         {% endif %}
                     {% endif %}
                 </td>

changedetectionio/tests/proxy_list/proxies.json-example

@@ -1,10 +1,10 @@
 {
   "proxy-one": {
-    "label": "One",
-    "url": "http://127.0.0.1:3128"
+    "label": "Proxy One",
+    "url": "http://squid-one:3128"
   },
   "proxy-two": {
-    "label": "two",
-    "url": "http://127.0.0.1:3129"
+    "label": "Proxy Two",
+    "url": "http://squid-two:3128"
   }
 }

changedetectionio/tests/proxy_list/squid-auth.conf

@@ -0,0 +1,48 @@
+acl localnet src 0.0.0.1-0.255.255.255  # RFC 1122 "this" network (LAN)
+acl localnet src 10.0.0.0/8             # RFC 1918 local private network (LAN)
+acl localnet src 100.64.0.0/10          # RFC 6598 shared address space (CGN)
+acl localnet src 169.254.0.0/16         # RFC 3927 link-local (directly plugged) machines
+acl localnet src 172.16.0.0/12          # RFC 1918 local private network (LAN)
+acl localnet src 192.168.0.0/16         # RFC 1918 local private network (LAN)
+acl localnet src fc00::/7               # RFC 4193 local private network range
+acl localnet src fe80::/10              # RFC 4291 link-local (directly plugged) machines
+acl localnet src 159.65.224.174
+acl SSL_ports port 443
+acl Safe_ports port 80          # http
+acl Safe_ports port 21          # ftp
+acl Safe_ports port 443         # https
+acl Safe_ports port 70          # gopher
+acl Safe_ports port 210         # wais
+acl Safe_ports port 1025-65535  # unregistered ports
+acl Safe_ports port 280         # http-mgmt
+acl Safe_ports port 488         # gss-http
+acl Safe_ports port 591         # filemaker
+acl Safe_ports port 777         # multiling http
+acl CONNECT method CONNECT
+
+http_access deny !Safe_ports
+http_access deny CONNECT !SSL_ports
+#http_access allow localhost manager
+http_access deny manager
+#http_access allow localhost
+#http_access allow localnet
+
+auth_param basic program /usr/lib/squid3/basic_ncsa_auth /etc/squid3/passwords
+auth_param basic realm proxy
+acl authenticated proxy_auth REQUIRED
+http_access allow authenticated
+http_access deny all
+
+
+http_port 3128
+coredump_dir /var/spool/squid
+refresh_pattern ^ftp:           1440    20%     10080
+refresh_pattern ^gopher:        1440    0%      1440
+refresh_pattern -i (/cgi-bin/|\?) 0     0%      0
+refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims
+refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims
+refresh_pattern \/InRelease$ 0 0% 0 refresh-ims
+refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims
+refresh_pattern .               0       20%     4320
+logfile_rotate 0
+

changedetectionio/tests/proxy_list/squid-passwords.txt

@@ -0,0 +1 @@
+test:$apr1$xvhFolTA$E/kz5/Rw1ewcyaSUdwqZs.

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -0,0 +1,50 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from ..util import live_server_setup, wait_for_all_checks
+
+# just make a request, we will grep in the docker logs to see it actually got called
+def test_select_custom(client, live_server):
+    live_server_setup(live_server)
+
+    # Goto settings, add our custom one
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-ignore_whitespace": "y",
+            "application-fetch_backend": "html_requests",
+            "requests-extra_proxies-0-proxy_name": "custom-test-proxy",
+            # test:awesome is set in tests/proxy_list/squid-passwords.txt
+            "requests-extra_proxies-0-proxy_url": "http://test:awesome@squid-custom:3128",
+        },
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    res = client.post(
+        url_for("import_page"),
+        # Because a URL wont show in squid/proxy logs due it being SSLed
+        # Use plain HTTP or a specific domain-name here
+        data={"urls": "https://changedetection.io/CHANGELOG.txt"},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    res = client.get(url_for("index"))
+    assert b'Proxy Authentication Required' not in res.data
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    # We should see something via proxy
+    assert b'<div class=""> - 0.' in res.data
+
+    #
+    # Now we should see the request in the container logs for "squid-squid-custom" because it will be the only default
+

changedetectionio/tests/test_access_control.py

@@ -1,18 +1,34 @@
+from . util import live_server_setup, extract_UUID_from_client
 from flask import url_for
-from . util import live_server_setup
+import time
 
-def test_check_access_control(app, client):
+def test_check_access_control(app, client, live_server):
     # Still doesnt work, but this is closer.
+    live_server_setup(live_server)
 
     with app.test_client(use_cookies=True) as c:
         # Check we don't have any password protection enabled yet.
         res = c.get(url_for("settings_page"))
         assert b"Remove password" not in res.data
 
-        # Enable password check.
+        # add something that we can hit via diff page later
+        res = c.post(
+            url_for("import_page"),
+            data={"urls": url_for('test_random_content_endpoint', _external=True)},
+            follow_redirects=True
+        )
+
+        assert b"1 Imported" in res.data
+        time.sleep(2)
+        res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        assert b'1 watches queued for rechecking.' in res.data
+        time.sleep(2)
+
+        # Enable password check and diff page access bypass
         res = c.post(
             url_for("settings_page"),
             data={"application-password": "foobar",
+                  "application-shared_diff_access": "True",
                   "requests-time_between_check-minutes": 180,
                   'application-fetch_backend': "html_requests"},
             follow_redirects=True
@@ -22,9 +38,15 @@ def test_check_access_control(app, client):
 
         # Check we hit the login
         res = c.get(url_for("index"), follow_redirects=True)
-
+        # Should be logged out
         assert b"Login" in res.data
 
+        # The diff page should return something valid when logged out
+        res = client.get(url_for("diff_history_page", uuid="first"))
+        assert b'Random content' in res.data
+
+
+
         # Menu should not be available yet
         #        assert b"SETTINGS" not in res.data
         #        assert b"BACKUP" not in res.data
@@ -109,3 +131,25 @@ def test_check_access_control(app, client):
 
         assert b"Password protection enabled" not in res.data
 
+        # Now checking the diff access
+        # Enable password check and diff page access bypass
+        res = c.post(
+            url_for("settings_page"),
+            data={"application-password": "foobar",
+                  # Should be disabled
+#                  "application-shared_diff_access": "True",
+                  "requests-time_between_check-minutes": 180,
+                  'application-fetch_backend': "html_requests"},
+            follow_redirects=True
+        )
+
+        assert b"Password protection enabled." in res.data
+
+        # Check we hit the login
+        res = c.get(url_for("index"), follow_redirects=True)
+        # Should be logged out
+        assert b"Login" in res.data
+
+        # The diff page should return something valid when logged out
+        res = client.get(url_for("diff_history_page", uuid="first"))
+        assert b'Random content' not in res.data

changedetectionio/tests/test_api.py

@@ -53,14 +53,15 @@ def is_valid_uuid(val):
         return False
 
 
-def test_api_simple(client, live_server):
+def test_setup(client, live_server):
     live_server_setup(live_server)
 
+def test_api_simple(client, live_server):
+
     api_key = extract_api_key_from_UI(client)
 
     # Create a watch
     set_original_response()
-    watch_uuid = None
 
     # Validate bad URL
     test_url = url_for('test_endpoint', _external=True,
@@ -80,25 +81,34 @@ def test_api_simple(client, live_server):
         headers={'content-type': 'application/json', 'x-api-key': api_key},
         follow_redirects=True
     )
-    s = json.loads(res.data)
-    assert is_valid_uuid(s['uuid'])
-    watch_uuid = s['uuid']
+
+    assert is_valid_uuid(res.json.get('uuid'))
+    watch_uuid = res.json.get('uuid')
     assert res.status_code == 201
 
     time.sleep(3)
 
     # Verify its in the list and that recheck worked
     res = client.get(
-        url_for("createwatch"),
+        url_for("createwatch", tag="OnE"),
         headers={'x-api-key': api_key}
     )
-    assert watch_uuid in json.loads(res.data).keys()
-    before_recheck_info = json.loads(res.data)[watch_uuid]
+    assert watch_uuid in res.json.keys()
+    before_recheck_info = res.json[watch_uuid]
     assert before_recheck_info['last_checked'] != 0
     #705 `last_changed` should be zero on the first check
     assert before_recheck_info['last_changed'] == 0
     assert before_recheck_info['title'] == 'My test URL'
 
+    # Check the limit by tag doesnt return anything when nothing found
+    res = client.get(
+        url_for("createwatch", tag="Something else entirely"),
+        headers={'x-api-key': api_key}
+    )
+    assert len(res.json) == 0
+
+    time.sleep(2)
+
     set_modified_response()
     # Trigger recheck of all ?recheck_all=1
     client.get(
@@ -112,7 +122,7 @@ def test_api_simple(client, live_server):
         url_for("createwatch"),
         headers={'x-api-key': api_key},
     )
-    after_recheck_info = json.loads(res.data)[watch_uuid]
+    after_recheck_info = res.json[watch_uuid]
     assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']
     assert after_recheck_info['last_changed'] != 0
 
@@ -121,12 +131,11 @@ def test_api_simple(client, live_server):
         url_for("watchhistory", uuid=watch_uuid),
         headers={'x-api-key': api_key},
     )
-    history = json.loads(res.data)
-    assert len(history) == 2, "Should have two history entries (the original and the changed)"
+    assert len(res.json) == 2, "Should have two history entries (the original and the changed)"
 
     # Fetch a snapshot by timestamp, check the right one was found
     res = client.get(
-        url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(history.keys())[-1]),
+        url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(res.json.keys())[-1]),
         headers={'x-api-key': api_key},
     )
     assert b'which has this one new line' in res.data
@@ -143,7 +152,7 @@ def test_api_simple(client, live_server):
         url_for("watch", uuid=watch_uuid),
         headers={'x-api-key': api_key}
     )
-    watch = json.loads(res.data)
+    watch = res.json
     # @todo how to handle None/default global values?
     assert watch['history_n'] == 2, "Found replacement history section, which is in its own API"
 
@@ -152,10 +161,46 @@ def test_api_simple(client, live_server):
         url_for("systeminfo"),
         headers={'x-api-key': api_key},
     )
-    info = json.loads(res.data)
-    assert info.get('watch_count') == 1
-    assert info.get('uptime') > 0.5
+    assert res.json.get('watch_count') == 1
+    assert res.json.get('uptime') > 0.5
 
+    ######################################################
+    # Mute and Pause, check it worked
+    res = client.get(
+        url_for("watch", uuid=watch_uuid, paused='paused'),
+        headers={'x-api-key': api_key}
+    )
+    assert b'OK' in res.data
+    res = client.get(
+        url_for("watch", uuid=watch_uuid,  muted='muted'),
+        headers={'x-api-key': api_key}
+    )
+    assert b'OK' in res.data
+    res = client.get(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key}
+    )
+    assert res.json.get('paused') == True
+    assert res.json.get('notification_muted') == True
+
+    # Now unpause, unmute
+    res = client.get(
+        url_for("watch", uuid=watch_uuid,  muted='unmuted'),
+        headers={'x-api-key': api_key}
+    )
+    assert b'OK' in res.data
+    res = client.get(
+        url_for("watch", uuid=watch_uuid, paused='unpaused'),
+        headers={'x-api-key': api_key}
+    )
+    assert b'OK' in res.data
+    res = client.get(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key}
+    )
+    assert res.json.get('paused') == 0
+    assert res.json.get('notification_muted') == 0
+    ######################################################
 
     # Finally delete the watch
     res = client.delete(
@@ -169,9 +214,7 @@ def test_api_simple(client, live_server):
         url_for("createwatch"),
         headers={'x-api-key': api_key}
     )
-    watch_list = json.loads(res.data)
-    assert len(watch_list) == 0, "Watch list should be empty"
-
+    assert len(res.json) == 0, "Watch list should be empty"
 
 def test_access_denied(client, live_server):
     # `config_api_token_enabled` Should be On by default
@@ -203,3 +246,97 @@ def test_access_denied(client, live_server):
         url_for("createwatch")
     )
     assert res.status_code == 200
+
+    # Cleanup everything
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-fetch_backend": "html_requests",
+            "application-api_access_token_enabled": "y"
+        },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+def test_api_watch_PUT_update(client, live_server):
+
+    #live_server_setup(live_server)
+    api_key = extract_api_key_from_UI(client)
+    time.sleep(1)
+    # Create a watch
+    set_original_response()
+    test_url = url_for('test_endpoint', _external=True,
+                       headers={'x-api-key': api_key}, )
+
+    # Create new
+    res = client.post(
+        url_for("createwatch"),
+        data=json.dumps({"url": test_url, 'tag': "One, Two", "title": "My test URL", 'headers': {'cookie': 'yum'} }),
+        headers={'content-type': 'application/json', 'x-api-key': api_key},
+        follow_redirects=True
+    )
+
+    assert res.status_code == 201
+
+    time.sleep(1)
+
+    # Get a listing, it will be the first one
+    res = client.get(
+        url_for("createwatch"),
+        headers={'x-api-key': api_key}
+    )
+
+    watch_uuid = list(res.json.keys())[0]
+
+    # Check in the edit page just to be sure
+    res = client.get(
+        url_for("edit_page", uuid=watch_uuid),
+    )
+    assert b"cookie: yum" in res.data, "'cookie: yum' found in 'headers' section"
+
+    # HTTP PUT ( UPDATE an existing watch )
+    res = client.put(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key, 'content-type': 'application/json'},
+        data=json.dumps({"title": "new title", 'time_between_check': {'minutes': 552}, 'headers': {'cookie': 'all eaten'}}),
+    )
+    assert res.status_code == 200, "HTTP PUT update was sent OK"
+
+    # HTTP GET single watch, title should be updated
+    res = client.get(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key}
+    )
+    assert res.json.get('title') == 'new title'
+
+    # Check in the edit page just to be sure
+    res = client.get(
+        url_for("edit_page", uuid=watch_uuid),
+    )
+    assert b"new title" in res.data, "new title found in edit page"
+    assert b"552" in res.data, "552 minutes found in edit page"
+    assert b"One, Two" in res.data, "Tag 'One, Two' was found"
+    assert b"cookie: all eaten" in res.data, "'cookie: all eaten' found in 'headers' section"
+
+    ######################################################
+
+    # HTTP PUT try a field that doenst exist
+
+    # HTTP PUT an update
+    res = client.put(
+        url_for("watch", uuid=watch_uuid),
+        headers={'x-api-key': api_key, 'content-type': 'application/json'},
+        data=json.dumps({"title": "new title", "some other field": "uh oh"}),
+    )
+
+    assert res.status_code == 400, "Should get error 400 when we give a field that doesnt exist"
+    # Message will come from `flask_expects_json`
+    assert b'Additional properties are not allowed' in res.data
+
+    # Cleanup everything
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -0,0 +1,146 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from .util import live_server_setup, extract_UUID_from_client, extract_api_key_from_UI
+
+def set_response_with_ldjson():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div class="sametext">Some text thats the same</div>
+     <div class="changetext">Some text that will change</div>
+     <script type="application/ld+json">
+        {
+           "@context":"https://schema.org/",
+           "@type":"Product",
+           "@id":"https://www.some-virtual-phone-shop.com/celular-iphone-14/p",
+           "name":"Celular Iphone 14 Pro Max 256Gb E Sim A16 Bionic",
+           "brand":{
+              "@type":"Brand",
+              "name":"APPLE"
+           },
+           "image":"https://www.some-virtual-phone-shop.com/15509426/image.jpg",
+           "description":"You dont need it",
+           "mpn":"111111",
+           "sku":"22222",
+           "offers":{
+              "@type":"AggregateOffer",
+              "lowPrice":8097000,
+              "highPrice":8099900,
+              "priceCurrency":"COP",
+              "offers":[
+                 {
+                    "@type":"Offer",
+                    "price":8097000,
+                    "priceCurrency":"COP",
+                    "availability":"http://schema.org/InStock",
+                    "sku":"102375961",
+                    "itemCondition":"http://schema.org/NewCondition",
+                    "seller":{
+                       "@type":"Organization",
+                       "name":"ajax"
+                    }
+                 }
+              ],
+              "offerCount":1
+           }
+        }
+       </script>
+     </body>
+     </html>
+"""
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_response_without_ldjson():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div class="sametext">Some text thats the same</div>
+     <div class="changetext">Some text that will change</div>     
+     </body>
+     </html>
+"""
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+# actually only really used by the distll.io importer, but could be handy too
+def test_check_ldjson_price_autodetect(client, live_server):
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    set_response_with_ldjson()
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    time.sleep(3)
+
+    # Should get a notice that it's available
+    res = client.get(url_for("index"))
+    assert b'ldjson-price-track-offer' in res.data
+
+    # Accept it
+    uuid = extract_UUID_from_client(client)
+
+    client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
+    time.sleep(2)
+
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(2)
+    # Offer should be gone
+    res = client.get(url_for("index"))
+    assert b'Embedded price data' not in res.data
+    assert b'tracking-ldjson-price-data' in res.data
+
+    # and last snapshop (via API) should be just the price
+    api_key = extract_api_key_from_UI(client)
+    res = client.get(
+        url_for("watchsinglehistory", uuid=uuid, timestamp='latest'),
+        headers={'x-api-key': api_key},
+    )
+
+    # Should see this (dont know where the whitespace came from)
+    assert b'"highPrice": 8099900' in res.data
+    # And not this cause its not the ld-json
+    assert b"So let's see what happens" not in res.data
+
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+
+    ##########################################################################################
+    # And we shouldnt see the offer
+    set_response_without_ldjson()
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    time.sleep(3)
+    res = client.get(url_for("index"))
+    assert b'ldjson-price-track-offer' not in res.data
+    
+    ##########################################################################################
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_backend.py

@@ -3,7 +3,7 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI
 
 sleep_time_for_fetch_thread = 3
 
@@ -67,7 +67,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Force recheck
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches are queued for rechecking.' in res.data
+    assert b'1 watches queued for rechecking.' in res.data
 
     wait_for_all_checks(client)
 
@@ -76,7 +76,8 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'unviewed' in res.data
 
     # #75, and it should be in the RSS feed
-    res = client.get(url_for("rss"))
+    rss_token = extract_rss_token_from_UI(client)
+    res = client.get(url_for("rss", token=rss_token, _external=True))
     expected_url = url_for('test_endpoint', _external=True)
     assert b'<rss' in res.data
 

changedetectionio/tests/test_element_removal.py

@@ -121,7 +121,7 @@ def test_element_removal_full(client, live_server):
         url_for("import_page"), data={"urls": test_url}, follow_redirects=True
     )
     assert b"1 Imported" in res.data
-
+    time.sleep(1)
     # Goto the edit page, add the filter data
     # Not sure why \r needs to be added - absent of the #changetext this is not necessary
     subtractive_selectors_data = "header\r\nfooter\r\nnav\r\n#changetext"

changedetectionio/tests/test_encoding.py

@@ -38,9 +38,6 @@ def test_check_encoding_detection(client, live_server):
         follow_redirects=True
     )
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(2)
 

changedetectionio/tests/test_errorhandling.py

@@ -77,7 +77,8 @@ def test_DNS_errors(client, live_server):
     time.sleep(3)
 
     res = client.get(url_for("index"))
-    assert b'Name or service not known' in res.data
+    found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
+    assert found_name_resolution_error
     # Should always record that we tried
     assert bytes("just now".encode('utf-8')) in res.data
 

changedetectionio/tests/test_extract_csv.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from urllib.request import urlopen
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+
+sleep_time_for_fetch_thread = 3
+
+
+
+def test_check_extract_text_from_diff(client, live_server):
+    import time
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("Now it's {} seconds since epoch, time flies!".format(str(time.time())))
+
+    live_server_setup(live_server)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url_for('test_endpoint', _external=True)},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+    time.sleep(1)
+
+    # Load in 5 different numbers/changes
+    last_date=""
+    for n in range(5):
+        # Give the thread time to pick it up
+        print("Bumping snapshot and checking.. ", n)
+        last_date = str(time.time())
+        with open("test-datastore/endpoint-content.txt", "w") as f:
+            f.write("Now it's {} seconds since epoch, time flies!".format(last_date))
+
+        client.get(url_for("form_watch_checknow"), follow_redirects=True)
+        wait_for_all_checks(client)
+
+    res = client.post(
+        url_for("diff_history_page", uuid="first"),
+        data={"extract_regex": "Now it's ([0-9\.]+)",
+              "extract_submit_button": "Extract as CSV"},
+        follow_redirects=False
+    )
+
+    assert b'Nothing matches that RegEx' not in res.data
+    assert res.content_type == 'text/csv'
+
+    # Read the csv reply as stringio
+    from io import StringIO
+    import csv
+
+    f = StringIO(res.data.decode('utf-8'))
+    reader = csv.reader(f, delimiter=',')
+    output=[]
+
+    for row in reader:
+        output.append(row)
+
+    assert output[0][0] == 'Epoch seconds'
+
+    # Header line + 1 origin/first + 5 changes
+    assert(len(output) == 7)
+
+    # We expect to find the last bumped date in the changes in the last field of the spreadsheet
+    assert(output[6][2] == last_date)
+    # And nothing else, only that group () of the decimal and .
+    assert "time flies" not in output[6][2]

changedetectionio/tests/test_filter_exist_changes.py

@@ -73,17 +73,17 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     # Just a regular notification setting, this will be used by the special 'filter not found' notification
     notification_form_data = {"notification_urls": notification_url,
-                              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
-                              "notification_body": "BASE URL: {base_url}\n"
-                                                   "Watch URL: {watch_url}\n"
-                                                   "Watch UUID: {watch_uuid}\n"
-                                                   "Watch title: {watch_title}\n"
-                                                   "Watch tag: {watch_tag}\n"
-                                                   "Preview: {preview_url}\n"
-                                                   "Diff URL: {diff_url}\n"
-                                                   "Snapshot: {current_snapshot}\n"
-                                                   "Diff: {diff}\n"
-                                                   "Diff Full: {diff_full}\n"
+                              "notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
+                              "notification_body": "BASE URL: {{base_url}}\n"
+                                                   "Watch URL: {{watch_url}}\n"
+                                                   "Watch UUID: {{watch_uuid}}\n"
+                                                   "Watch title: {{watch_title}}\n"
+                                                   "Watch tag: {{watch_tag}}\n"
+                                                   "Preview: {{preview_url}}\n"
+                                                   "Diff URL: {{diff_url}}\n"
+                                                   "Snapshot: {{current_snapshot}}\n"
+                                                   "Diff: {{diff}}\n"
+                                                   "Diff Full: {{diff_full}}\n"
                                                    ":-)",
                               "notification_format": "Text"}
 

changedetectionio/tests/test_filter_failure_notification.py

@@ -1,8 +1,7 @@
 import os
 import time
-import re
 from flask import url_for
-from .util import set_original_response, live_server_setup
+from .util import set_original_response, live_server_setup, extract_UUID_from_client
 from changedetectionio.model import App
 
 
@@ -56,17 +55,17 @@ def run_filter_test(client, content_filter):
 
     # Just a regular notification setting, this will be used by the special 'filter not found' notification
     notification_form_data = {"notification_urls": notification_url,
-                              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
-                              "notification_body": "BASE URL: {base_url}\n"
-                                                   "Watch URL: {watch_url}\n"
-                                                   "Watch UUID: {watch_uuid}\n"
-                                                   "Watch title: {watch_title}\n"
-                                                   "Watch tag: {watch_tag}\n"
-                                                   "Preview: {preview_url}\n"
-                                                   "Diff URL: {diff_url}\n"
-                                                   "Snapshot: {current_snapshot}\n"
-                                                   "Diff: {diff}\n"
-                                                   "Diff Full: {diff_full}\n"
+                              "notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
+                              "notification_body": "BASE URL: {{base_url}}\n"
+                                                   "Watch URL: {{watch_url}}\n"
+                                                   "Watch UUID: {{watch_uuid}}\n"
+                                                   "Watch title: {{watch_title}}\n"
+                                                   "Watch tag: {{watch_tag}}\n"
+                                                   "Preview: {{preview_url}}\n"
+                                                   "Diff URL: {{diff_url}}\n"
+                                                   "Snapshot: {{current_snapshot}}\n"
+                                                   "Diff: {{diff}}\n"
+                                                   "Diff Full: {{diff_full}}\n"
                                                    ":-)",
                               "notification_format": "Text"}
 
@@ -84,6 +83,7 @@ def run_filter_test(client, content_filter):
         data=notification_form_data,
         follow_redirects=True
     )
+
     assert b"Updated watch." in res.data
     time.sleep(3)
 
@@ -120,6 +120,10 @@ def run_filter_test(client, content_filter):
         notification = f.read()
     assert not 'CSS/xPath filter was not present in the page' in notification
 
+    # Re #1247 - All tokens got replaced
+    uuid = extract_UUID_from_client(client)
+    assert uuid in notification
+
     # cleanup for the next
     client.get(
         url_for("form_delete", uuid="all"),

changedetectionio/tests/test_ignore_text.py

@@ -101,9 +101,6 @@ def test_check_ignore_text_functionality(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -199,9 +196,6 @@ def test_check_global_ignore_text_functionality(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 

changedetectionio/tests/test_ignorestatuscode.py

@@ -69,8 +69,6 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
     assert b"1 Imported" in res.data
 
     time.sleep(sleep_time_for_fetch_thread)
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     set_some_changed_response()
     time.sleep(sleep_time_for_fetch_thread)
@@ -104,9 +102,6 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -119,11 +114,9 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
+
     #  Make a change
     set_some_changed_response()
 

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -394,6 +394,48 @@ def check_json_ext_filter(json_filter, client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
+def test_ignore_json_order(client, live_server):
+    # A change in order shouldn't trigger a notification
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write('{"hello" : 123, "world": 123}')
+
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    time.sleep(2)
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write('{"world" : 123, "hello": 123}')
+
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(2)
+
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+    # Just to be sure it still works
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write('{"world" : 123, "hello": 124}')
+
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(2)
+
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
 def test_check_jsonpath_ext_filter(client, live_server):
     check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server)
 

changedetectionio/tests/test_notification.py

@@ -90,17 +90,17 @@ def test_check_notification(client, live_server):
     print (">>>> Notification URL: "+notification_url)
 
     notification_form_data = {"notification_urls": notification_url,
-                              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
-                              "notification_body": "BASE URL: {base_url}\n"
-                                                   "Watch URL: {watch_url}\n"
-                                                   "Watch UUID: {watch_uuid}\n"
-                                                   "Watch title: {watch_title}\n"
-                                                   "Watch tag: {watch_tag}\n"
-                                                   "Preview: {preview_url}\n"
-                                                   "Diff URL: {diff_url}\n"
-                                                   "Snapshot: {current_snapshot}\n"
-                                                   "Diff: {diff}\n"
-                                                   "Diff Full: {diff_full}\n"
+                              "notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
+                              "notification_body": "BASE URL: {{base_url}}\n"
+                                                   "Watch URL: {{watch_url}}\n"
+                                                   "Watch UUID: {{watch_uuid}}\n"
+                                                   "Watch title: {{watch_title}}\n"
+                                                   "Watch tag: {{watch_tag}}\n"
+                                                   "Preview: {{preview_url}}\n"
+                                                   "Diff URL: {{diff_url}}\n"
+                                                   "Snapshot: {{current_snapshot}}\n"
+                                                   "Diff: {{diff}}\n"
+                                                   "Diff Full: {{diff_full}}\n"
                                                    ":-)",
                               "notification_screenshot": True,
                               "notification_format": "Text"}
@@ -179,7 +179,6 @@ def test_check_notification(client, live_server):
         logging.debug(">>> Skipping BASE_URL check")
 
 
-
     # This should insert the {current_snapshot}
     set_more_modified_response()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
@@ -237,10 +236,10 @@ def test_check_notification(client, live_server):
         follow_redirects=True
     )
 
-
 def test_notification_validation(client, live_server):
-    #live_server_setup(live_server)
-    time.sleep(3)
+
+    time.sleep(1)
+
     # re #242 - when you edited an existing new entry, it would not correctly show the notification settings
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
@@ -268,21 +267,6 @@ def test_notification_validation(client, live_server):
 #    )
 #    assert b"Notification Body and Title is required when a Notification URL is used" in res.data
 
-    # Now adding a wrong token should give us an error
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-notification_title": "New ChangeDetection.io Notification - {watch_url}",
-              "application-notification_body": "Rubbish: {rubbish}\n",
-              "application-notification_format": "Text",
-              "application-notification_urls": "json://localhost/foobar",
-              "requests-time_between_check-minutes": 180,
-              "fetch_backend": "html_requests"
-              },
-        follow_redirects=True
-    )
-
-    assert bytes("is not a valid token".encode('utf-8')) in res.data
-
     # cleanup for the next
     client.get(
         url_for("form_delete", uuid="all"),
@@ -290,3 +274,58 @@ def test_notification_validation(client, live_server):
     )
 
 
+
+def test_notification_custom_endpoint_and_jinja2(client, live_server):
+    time.sleep(1)
+
+    # test_endpoint - that sends the contents of a file
+    # test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt)
+
+    # CUSTOM JSON BODY CHECK for POST://
+    set_original_response()
+    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}"
+
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
+              "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }',
+              # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+              "application-notification_urls": test_notification_url,
+              "application-minutes_between_check": 180,
+              "application-fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+    assert b'Settings updated' in res.data
+
+    # Add a watch and trigger a HTTP POST
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tag": 'nice one'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    time.sleep(2)
+    set_modified_response()
+
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(2)
+
+
+    with open("test-datastore/notification.txt", 'r') as f:
+        x=f.read()
+        j = json.loads(x)
+        assert j['url'].startswith('http://localhost')
+        assert j['secret'] == 444
+
+    # URL check, this will always be converted to lowercase
+    assert os.path.isfile("test-datastore/notification-url.txt")
+    with open("test-datastore/notification-url.txt", 'r') as f:
+        notification_url = f.read()
+        assert 'xxx=http' in notification_url
+
+    os.unlink("test-datastore/notification-url.txt")
+

changedetectionio/tests/test_notification_errors.py

@@ -11,23 +11,23 @@ def test_check_notification_error_handling(client, live_server):
     set_original_response()
 
     # Give the endpoint time to spin up
-    time.sleep(3)
+    time.sleep(2)
 
-    # use a different URL so that it doesnt interfere with the actual check until we are ready
+    # Set a URL and fetch it, then set a notification URL which is going to give errors
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": "https://changedetection.io/CHANGELOG.txt", "tag": ''},
+        data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
 
-    time.sleep(10)
+    time.sleep(2)
+    set_modified_response()
 
-    # Check we capture the failure, we can just use trigger_check = y here
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"notification_urls": "jsons://broken-url.changedetection.io/test",
+        data={"notification_urls": "jsons://broken-url-xxxxxxxx123/test",
               "notification_title": "xxx",
               "notification_body": "xxxxx",
               "notification_format": "Text",
@@ -36,15 +36,14 @@ def test_check_notification_error_handling(client, live_server):
               "title": "",
               "headers": "",
               "time_between_check-minutes": "180",
-              "fetch_backend": "html_requests",
-              "trigger_check": "y"},
+              "fetch_backend": "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
 
     found=False
     for i in range(1, 10):
-        time.sleep(1)
+
         logging.debug("Fetching watch overview....")
         res = client.get(
             url_for("index"))
@@ -53,6 +52,7 @@ def test_check_notification_error_handling(client, live_server):
             found=True
             break
 
+        time.sleep(1)
 
     assert found
 
@@ -60,7 +60,7 @@ def test_check_notification_error_handling(client, live_server):
     # The error should show in the notification logs
     res = client.get(
         url_for("notification_logs"))
-    assert bytes("Name or service not known".encode('utf-8')) in res.data
+    found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
+    assert found_name_resolution_error
 
-
-    # And it should be listed on the watch overview
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_pdf.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from .util import set_original_response, set_modified_response, live_server_setup
+
+sleep_time_for_fetch_thread = 3
+
+# `subtractive_selectors` should still work in `source:` type requests
+def test_fetch_pdf(client, live_server):
+    import shutil
+    shutil.copy("tests/test.pdf", "test-datastore/endpoint-test.pdf")
+
+    live_server_setup(live_server)
+    test_url = url_for('test_pdf_endpoint', _external=True)
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b'PDF-1.5' not in res.data
+    assert b'hello world' in res.data
+
+    # So we know if the file changes in other ways
+    import hashlib
+    md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper()
+    # We should have one
+    assert len(md5) >0
+    # And it's going to be in the document
+    assert b'Document checksum - '+bytes(str(md5).encode('utf-8')) in res.data

changedetectionio/tests/test_request.py

@@ -20,6 +20,8 @@ def test_headers_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    time.sleep(1)
+
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},
@@ -174,6 +176,7 @@ def test_method_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    time.sleep(2)
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},
@@ -181,6 +184,8 @@ def test_method_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    time.sleep(2)
+
     # Attempt to add a method which is not valid
     res = client.post(
         url_for("edit_page", uuid="first"),
@@ -206,7 +211,7 @@ def test_method_in_request(client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick up the first version
-    time.sleep(5)
+    time.sleep(2)
 
     # The service should echo back the request verb
     res = client.get(
@@ -217,7 +222,7 @@ def test_method_in_request(client, live_server):
     # The test call service will return the verb as the body
     assert b"PATCH" in res.data
 
-    time.sleep(5)
+    time.sleep(2)
 
     watches_with_method = 0
     with open('test-datastore/url-watches.json') as f:

changedetectionio/tests/test_rss.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI
+
+
+def test_rss_and_token(client, live_server):
+    set_original_response()
+    live_server_setup(live_server)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url_for('test_random_content_endpoint', _external=True)},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+    rss_token = extract_rss_token_from_UI(client)
+
+    time.sleep(2)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(2)
+
+    # Add our URL to the import page
+    res = client.get(
+        url_for("rss", token="bad token", _external=True),
+        follow_redirects=True
+    )
+
+    assert b"Access denied, bad token" in res.data
+
+    res = client.get(
+        url_for("rss", token=rss_token, _external=True),
+        follow_redirects=True
+    )
+    assert b"Access denied, bad token" not in res.data
+    assert b"Random content" in res.data

changedetectionio/tests/test_security.py

@@ -2,11 +2,9 @@ from flask import url_for
 from . util import set_original_response, set_modified_response, live_server_setup
 import time
 
-def test_setup(live_server):
+
+def test_bad_access(client, live_server):
     live_server_setup(live_server)
-
-def test_file_access(client, live_server):
-
     res = client.post(
         url_for("import_page"),
         data={"urls": 'https://localhost'},
@@ -19,18 +17,49 @@ def test_file_access(client, live_server):
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
-              "url": 'file:///etc/passwd',
+              "url": 'javascript:alert(document.domain)',
               "tag": "",
               "method": "GET",
               "fetch_backend": "html_requests",
               "body": ""},
         follow_redirects=True
     )
-    time.sleep(3)
 
-    res = client.get(
-        url_for("index", uuid="first"),
+    assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
+
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": '            javascript:alert(123)', "tag": ''},
         follow_redirects=True
     )
 
-    assert b'denied for security reasons' in res.data
+    assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
+
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": '%20%20%20javascript:alert(123)%20%20', "tag": ''},
+        follow_redirects=True
+    )
+
+    assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
+
+
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": ' source:javascript:alert(document.domain)', "tag": ''},
+        follow_redirects=True
+    )
+
+    assert b'Watch protocol is not permitted by SAFE_PROTOCOL_REGEX' in res.data
+
+    # file:// is permitted by default, but it will be caught by ALLOW_FILE_URI
+
+    client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": 'file:///tasty/disk/drive', "tag": ''},
+        follow_redirects=True
+    )
+    time.sleep(1)
+    res = client.get(url_for("index"))
+
+    assert b'file:// type access is denied for security reasons.' in res.data