Faster BrowserStep screenshot updates and enable gzip compression for all content replies in the UI

.github/test/Dockerfile-alpine

@@ -0,0 +1,31 @@
+# Taken from https://github.com/linuxserver/docker-changedetection.io/blob/main/Dockerfile
+# Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)
+# Some packages wont install via pypi because they dont have a wheel available under this architecture.
+
+FROM ghcr.io/linuxserver/baseimage-alpine:3.16
+ENV PYTHONUNBUFFERED=1
+
+COPY requirements.txt /requirements.txt
+
+RUN \
+  apk add --update --no-cache --virtual=build-dependencies \
+    cargo \
+    g++ \
+    gcc \
+    libc-dev \
+    libffi-dev \
+    libxslt-dev \
+    make \
+    openssl-dev \
+    py3-wheel \
+    python3-dev \
+    zlib-dev && \
+  apk add --update --no-cache \
+    libxslt \
+    python3 \
+    py3-pip && \
+  echo "**** pip3 install test of changedetection.io ****" && \
+  pip3 install -U pip wheel setuptools && \
+  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.16/ -r /requirements.txt && \
+  apk del --purge \
+    build-dependencies

.github/workflows/test-container-build.yml

@@ -0,0 +1,66 @@
+name: ChangeDetection.io Container Build Test
+
+# Triggers the workflow on push or pull request events
+
+# This line doesnt work, even tho it is the documented one
+#on: [push, pull_request]
+
+on:
+  push:
+    paths:
+      - requirements.txt
+      - Dockerfile
+
+  pull_request:
+    paths:
+      - requirements.txt
+      - Dockerfile
+
+  # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
+  # @todo: some kind of path filter for requirements.txt and Dockerfile
+jobs:
+  test-container-build:
+    runs-on: ubuntu-latest
+    steps:
+        - uses: actions/checkout@v2
+        - name: Set up Python 3.9
+          uses: actions/setup-python@v2
+          with:
+            python-version: 3.9
+
+        # Just test that the build works, some libraries won't compile on ARM/rPi etc
+        - name: Set up QEMU
+          uses: docker/setup-qemu-action@v1
+          with:
+            image: tonistiigi/binfmt:latest
+            platforms: all
+
+        - name: Set up Docker Buildx
+          id: buildx
+          uses: docker/setup-buildx-action@v1
+          with:
+            install: true
+            version: latest
+            driver-opts: image=moby/buildkit:master
+
+        # https://github.com/dgtlmoon/changedetection.io/pull/1067
+        # Check we can still build under alpine/musl
+        - name: Test that the docker containers can build (musl via alpine check)
+          id: docker_build_musl
+          uses: docker/build-push-action@v2
+          with:
+            context: ./
+            file: ./.github/test/Dockerfile-alpine
+            platforms: linux/amd64,linux/arm64
+
+        - name: Test that the docker containers can build
+          id: docker_build
+          uses: docker/build-push-action@v2
+          # https://github.com/docker/build-push-action#customizing
+          with:
+            context: ./
+            file: ./Dockerfile
+            platforms: linux/arm/v7,linux/arm/v6,linux/amd64,linux/arm64,
+            cache-from: type=local,src=/tmp/.buildx-cache
+            cache-to: type=local,dest=/tmp/.buildx-cache
+

.github/workflows/test-only.yml

@@ -1,28 +1,25 @@
-name: ChangeDetection.io Test
+name: ChangeDetection.io App Test
 
 # Triggers the workflow on push or pull request events
 on: [push, pull_request]
 
 jobs:
-  test-build:
+  test-application:
     runs-on: ubuntu-latest
     steps:
-
       - uses: actions/checkout@v2
       - name: Set up Python 3.9
         uses: actions/setup-python@v2
         with:
           python-version: 3.9
 
-      - name: Show env vars
-        run: set
-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
           if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or undefined names
@@ -39,7 +36,4 @@ jobs:
           # Each test is totally isolated and performs its own cleanup/reset
           cd changedetectionio; ./run_all_tests.sh
 
-      # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
-      # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
 
-      # https://github.com/docker/buildx/issues/495#issuecomment-918925854

.gitignore

@@ -8,6 +8,7 @@ __pycache__
 build
 dist
 venv
+test-datastore/*
 test-datastore
 *.egg-info*
 .vscode/settings.json

CONTRIBUTING.md

@@ -6,7 +6,7 @@ Otherwise, it's always best to PR into the `dev` branch.
 
 Please be sure that all new functionality has a matching test!
 
-Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notifications.py` for example
+Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notification.py` for example
 
 ```
 pip3 install -r requirements-dev

Dockerfile

@@ -5,13 +5,15 @@ FROM python:3.8-slim as builder
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    libssl-dev \
-    libffi-dev \
+    g++ \
     gcc \
     libc-dev \
+    libffi-dev \
+    libjpeg-dev \
+    libssl-dev \
     libxslt-dev \
-    zlib1g-dev \
-    g++
+    make \
+    zlib1g-dev
 
 RUN mkdir /install
 WORKDIR /install
@@ -22,7 +24,8 @@ RUN pip install --target=/dependencies -r /requirements.txt
 
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
-RUN pip install --target=/dependencies playwright~=1.26 \
+# https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
+RUN pip install --target=/dependencies playwright~=1.27.1 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
@@ -34,13 +37,14 @@ ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
 # Re #93, #73, excluding rustc (adds another 430Mb~)
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    libssl-dev \
-    libffi-dev \
+    g++ \
     gcc \
     libc-dev \
+    libffi-dev \
+    libjpeg-dev \
+    libssl-dev \
     libxslt-dev \
-    zlib1g-dev \
-    g++
+    zlib1g-dev
 
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1
@@ -58,6 +62,7 @@ EXPOSE 5000
 
 # The actual flask app
 COPY changedetectionio /app/changedetectionio
+
 # The eventlet server wrapper
 COPY changedetection.py /app/changedetection.py
 

MANIFEST.in

@@ -2,7 +2,12 @@ recursive-include changedetectionio/api *
 recursive-include changedetectionio/templates *
 recursive-include changedetectionio/static *
 recursive-include changedetectionio/model *
+recursive-include changedetectionio/tests *
+recursive-include changedetectionio/res *
+prune changedetectionio/static/package-lock.json
+prune changedetectionio/static/styles/node_modules
+prune changedetectionio/static/styles/package-lock.json
 include changedetection.py
 global-exclude *.pyc
 global-exclude node_modules
-global-exclude venv
+global-exclude venv

README.md

@@ -1,6 +1,7 @@
 ## Web Site Change Detection, Monitoring and Notification.
 
-Live your data-life pro-actively, track website content changes and receive notifications via Discord, Email, Slack, Telegram and 70+ more
+_Live your data-life pro-actively, Detect website changes and perform meaningful actions, trigger notifications via Discord, Email, Slack, Telegram, API calls and many more._
+
 
 [<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start?src=github)
 
@@ -8,21 +9,37 @@ Live your data-life pro-actively, track website content changes and receive noti
 
 ![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 
-Know when important content changes, we support notifications via Discord, Telegram, Home-Assistant, Slack, Email and 70+ more
-
 [**Don't have time? Let us host it for you! try our $6.99/month subscription - use our proxies and support!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
 
 - Chrome browser included.
 - Super fast, no registration needed setup.
-- Start watching and receiving change notifications instantly.
+- Get started watching and receiving website change notifications straight away.
 
 
-Easily see what changed, examine by word, line, or individual character.
+### Target specific parts of the webpage using the Visual Selector tool.
 
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
+Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://lemonade.changedetection.io/start?src=github)
+
+### Easily see what changed, examine by word, line, or individual character.
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://lemonade.changedetection.io/start?src=github)
 
 
-#### Example use cases
+### Perform interactive browser steps
+
+Fill in text boxes, click buttons and more, setup your changedetection scenario. 
+
+Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
+
+[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Website change detection with interactive browser steps, login, cookies etc" />](https://lemonade.changedetection.io/start?src=github)
+
+After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
+Requires Playwright to be enabled.
+
+
+### Example use cases
 
 - Products and services have a change in pricing
 - _Out of stock notification_ and _Back In stock notification_
@@ -41,6 +58,8 @@ Easily see what changed, examine by word, line, or individual character.
 - Create RSS feeds based on changes in web content
 - Monitor HTML source code for unexpected changes, strengthen your PCI compliance
 - You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
+- Get notified when certain keywords appear in Twitter search results
+- Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 
@@ -54,19 +73,12 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 - Override Request Headers, Specify `POST` or `GET` and other methods
 - Use the "Visual Selector" to help target specific elements
 - Configurable [proxy per watch](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration)
+- Send a screenshot with the notification when a change is detected in the web page
 
 We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
 
-## Screenshots
-
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
-### Filter by elements using the Visual Selector tool.
-
-Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
-
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
-
 ## Installation
 
 ### Docker
@@ -106,8 +118,8 @@ _Now with per-site configurable support for using a fast built in HTTP fetcher o
 ### Docker
 ```
 docker pull dgtlmoon/changedetection.io
-docker kill $(docker ps -a|grep changedetection.io|awk '{print $1}')
-docker rm $(docker ps -a|grep changedetection.io|awk '{print $1}')
+docker kill $(docker ps -a -f name=changedetection.io -q)
+docker rm $(docker ps -a -f name=changedetection.io -q)
 docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```
 
@@ -121,8 +133,8 @@ See the wiki for more information https://github.com/dgtlmoon/changedetection.io
 
 
 ## Filters
-XPath, JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools.
 
+XPath, JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. 
 (We support LXML `re:test`, `re:math` and `re:replace`.)
 
 ## Notifications
@@ -161,46 +173,11 @@ This will re-parse the JSON and apply formatting to the text, making it super ea
 
 ### JSONPath or jq?
 
-For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more information on jq.
+For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more specifc information on jq.
 
-The example below adds the price in dollars to each item in the JSON data, and then filters to only show items that are greater than 10.
+One big advantage of `jq` is that you can use logic in your JSON filter, such as filters to only show items that have a value greater than/less than etc.
 
-#### Sample input data from API
-```
-{
-    "items": [
-        {
-           "name": "Product A",
-           "priceInCents": 2500
-        },
-        {
-           "name": "Product B",
-           "priceInCents": 500
-        },
-        {
-           "name": "Product C",
-           "priceInCents": 2000
-        }
-    ]
-}
-```
-
-#### Sample jq
-`jq:.items[] | . + { "priceInDollars": (.priceInCents / 100) } | select(.priceInDollars > 10)`
-
-#### Sample output data
-```
-{
-  "name": "Product A",
-  "priceInCents": 2500,
-  "priceInDollars": 25
-}
-{
-  "name": "Product C",
-  "priceInCents": 2000,
-  "priceInDollars": 20
-}
-```
+See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/JSON-Selector-Filter-help for more information and examples
 
 ### Parse JSON embedded in HTML!
 
@@ -216,9 +193,9 @@ When you enable a `json:` or `jq:` filter, you can even automatically extract an
 
 `json:$.price` or `jq:.price` would give `23.50`, or you can extract the whole structure
 
-## Proxy configuration
+## Proxy Configuration
 
-See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration
+See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [BrightData proxy services where possible]( https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support)
 
 ## Raspberry Pi support?
 

changedetectionio/__init__.py

@@ -1,18 +1,20 @@
 #!/usr/bin/python3
 
 import datetime
+import flask_login
+import logging
 import os
+import pytz
 import queue
 import threading
 import time
+import timeago
+
 from copy import deepcopy
+from distutils.util import strtobool
+from feedgen.feed import FeedGenerator
 from threading import Event
 
-import flask_login
-import logging
-import pytz
-import timeago
-from feedgen.feed import FeedGenerator
 from flask import (
     Flask,
     abort,
@@ -25,15 +27,15 @@ from flask import (
     session,
     url_for,
 )
+from flask_compress import Compress as FlaskCompress
 from flask_login import login_required
 from flask_restful import abort, Api
-
 from flask_wtf import CSRFProtect
 
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.39.20'
+__version__ = '0.39.22.1'
 
 datastore = None
 
@@ -44,13 +46,16 @@ ticker_thread = None
 extra_stylesheets = []
 
 update_q = queue.PriorityQueue()
-
 notification_q = queue.Queue()
 
 app = Flask(__name__,
             static_url_path="",
             static_folder="static",
             template_folder="templates")
+from flask_compress import Compress
+
+# Super handy for compressing large BrowserSteps responses and others
+FlaskCompress(app)
 
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
@@ -97,7 +102,7 @@ def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
     # Worker thread tells us which UUID it is currently processing.
     for t in running_update_threads:
         if t.current_uuid == watch_obj['uuid']:
-            return '<span class="loader"></span><span> Checking now</span>'
+            return '<span class="spinner"></span><span> Checking now</span>'
 
     if watch_obj['last_checked'] == 0:
         return 'Not yet'
@@ -194,7 +199,8 @@ def changedetection_app(config=None, datastore_o=None):
     watch_api.add_resource(api_v1.Watch, '/api/v1/watch/<string:uuid>',
                            resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
 
-
+    watch_api.add_resource(api_v1.SystemInfo, '/api/v1/systeminfo',
+                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
 
 
 
@@ -524,6 +530,7 @@ def changedetection_app(config=None, datastore_o=None):
 
     def edit_page(uuid):
         from changedetectionio import forms
+        from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
 
         using_default_check_time = True
         # More for testing, possible to return the first/only
@@ -557,6 +564,8 @@ def changedetection_app(config=None, datastore_o=None):
                                data=default,
                                )
 
+        # form.browser_steps[0] can be assumed that we 'goto url' first
+
         if datastore.proxy_list is None:
             # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
             del form.proxy
@@ -598,7 +607,7 @@ def changedetection_app(config=None, datastore_o=None):
                     extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
             # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
-            if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']:
+            if form.include_filters.data != datastore.data['watching'][uuid].get('include_filters', []):
                 if len(datastore.data['watching'][uuid].history):
                     extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
@@ -636,20 +645,33 @@ def changedetection_app(config=None, datastore_o=None):
             # Only works reliably with Playwright
             visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and default['fetch_backend'] == 'html_webdriver'
 
+            # JQ is difficult to install on windows and must be manually added (outside requirements.txt)
+            jq_support = True
+            try:
+                import jq
+            except ModuleNotFoundError:
+                jq_support = False
+
+            watch = datastore.data['watching'].get(uuid)
+            system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+            is_html_webdriver = True if watch.get('fetch_backend') == 'html_webdriver' or (
+                    watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
 
             output = render_template("edit.html",
-                                     uuid=uuid,
-                                     watch=datastore.data['watching'][uuid],
-                                     form=form,
-                                     has_empty_checktime=using_default_check_time,
-                                     has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
-                                     using_global_webdriver_wait=default['webdriver_delay'] is None,
+                                     browser_steps_config=browser_step_ui_config,
                                      current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
+                                     form=form,
+                                     has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
+                                     has_empty_checktime=using_default_check_time,
+                                     is_html_webdriver=is_html_webdriver,
+                                     jq_support=jq_support,
+                                     playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
                                      settings_application=datastore.data['settings']['application'],
-                                     visualselector_data_is_ready=visualselector_data_is_ready,
+                                     using_global_webdriver_wait=default['webdriver_delay'] is None,
+                                     uuid=uuid,
                                      visualselector_enabled=visualselector_enabled,
-                                     playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False)
+                                     watch=watch
                                      )
 
         return output
@@ -809,8 +831,10 @@ def changedetection_app(config=None, datastore_o=None):
 
         newest_file = history[dates[-1]]
 
+        # Read as binary and force decode as UTF-8
+        # Windows may fail decode in python if we just use 'r' mode (chardet decode exception)
         try:
-            with open(newest_file, 'r') as f:
+            with open(newest_file, 'r', encoding='utf-8', errors='ignore') as f:
                 newest_version_file_contents = f.read()
         except Exception as e:
             newest_version_file_contents = "Unable to read {}.\n".format(newest_file)
@@ -823,7 +847,7 @@ def changedetection_app(config=None, datastore_o=None):
             previous_file = history[dates[-2]]
 
         try:
-            with open(previous_file, 'r') as f:
+            with open(previous_file, 'r', encoding='utf-8', errors='ignore') as f:
                 previous_version_file_contents = f.read()
         except Exception as e:
             previous_version_file_contents = "Unable to read {}.\n".format(previous_file)
@@ -900,7 +924,7 @@ def changedetection_app(config=None, datastore_o=None):
         timestamp = list(watch.history.keys())[-1]
         filename = watch.history[timestamp]
         try:
-            with open(filename, 'r') as f:
+            with open(filename, 'r', encoding='utf-8', errors='ignore') as f:
                 tmp = f.readlines()
 
                 # Get what needs to be highlighted
@@ -975,9 +999,6 @@ def changedetection_app(config=None, datastore_o=None):
 
         # create a ZipFile object
         backupname = "changedetection-backup-{}.zip".format(int(time.time()))
-
-        # We only care about UUIDS from the current index file
-        uuids = list(datastore.data['watching'].keys())
         backup_filepath = os.path.join(datastore_o.datastore_path, backupname)
 
         with zipfile.ZipFile(backup_filepath, "w",
@@ -993,12 +1014,12 @@ def changedetection_app(config=None, datastore_o=None):
             # Add the flask app secret
             zipObj.write(os.path.join(datastore_o.datastore_path, "secret.txt"), arcname="secret.txt")
 
-            # Add any snapshot data we find, use the full path to access the file, but make the file 'relative' in the Zip.
-            for txt_file_path in Path(datastore_o.datastore_path).rglob('*.txt'):
-                parent_p = txt_file_path.parent
-                if parent_p.name in uuids:
-                    zipObj.write(txt_file_path,
-                                 arcname=str(txt_file_path).replace(datastore_o.datastore_path, ''),
+            # Add any data in the watch data directory.
+            for uuid, w in datastore.data['watching'].items():
+                for f in Path(w.watch_data_dir).glob('*'):
+                    zipObj.write(f,
+                                 # Use the full path to access the file, but make the file 'relative' in the Zip.
+                                 arcname=os.path.join(f.parts[-2], f.parts[-1]),
                                  compress_type=zipfile.ZIP_DEFLATED,
                                  compresslevel=8)
 
@@ -1177,7 +1198,6 @@ def changedetection_app(config=None, datastore_o=None):
         else:
             # No tag, no uuid, add everything.
             for watch_uuid, watch in datastore.data['watching'].items():
-
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                     update_q.put((1, watch_uuid))
                     i += 1
@@ -1295,13 +1315,15 @@ def changedetection_app(config=None, datastore_o=None):
         # paste in etc
         return redirect(url_for('index'))
 
+    import changedetectionio.blueprint.browser_steps as browser_steps
+    app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
+
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
-
     threading.Thread(target=notification_runner).start()
 
-    # Check for new release version, but not when running in test/build
-    if not os.getenv("GITHUB_REF", False):
+    # Check for new release version, but not when running in test/build or pytest
+    if not os.getenv("GITHUB_REF", False) and not config.get('disable_checkver') == True:
         threading.Thread(target=check_for_new_version).start()
 
     return app
@@ -1361,7 +1383,7 @@ def notification_runner():
                 # UUID wont be present when we submit a 'test' from the global settings
                 if 'uuid' in n_object:
                     datastore.update_watch(uuid=n_object['uuid'],
-                                           update_obj={'last_notification_error': "Notification error detected, please see logs."})
+                                           update_obj={'last_notification_error': "Notification error detected, goto notification log."})
 
                 log_lines = str(e).splitlines()
                 notification_debug_log += log_lines

changedetectionio/api/api_v1.py

@@ -122,3 +122,37 @@ class CreateWatch(Resource):
             return {'status': "OK"}, 200
 
         return list, 200
+
+class SystemInfo(Resource):
+    def __init__(self, **kwargs):
+        # datastore is a black box dependency
+        self.datastore = kwargs['datastore']
+        self.update_q = kwargs['update_q']
+
+    @auth.check_token
+    def get(self):
+        import time
+        overdue_watches = []
+
+        # Check all watches and report which have not been checked but should have been
+
+        for uuid, watch in self.datastore.data.get('watching', {}).items():
+            # see if now - last_checked is greater than the time that should have been
+            # this is not super accurate (maybe they just edited it) but better than nothing
+            t = watch.threshold_seconds()
+            if not t:
+                # Use the system wide default
+                t = self.datastore.threshold_seconds
+
+            time_since_check = time.time() - watch.get('last_checked')
+
+            # Allow 5 minutes of grace time before we decide it's overdue
+            if time_since_check - (5 * 60) > t:
+                overdue_watches.append(uuid)
+
+        return {
+                   'queue_size': self.update_q.qsize(),
+                   'overdue_watches': overdue_watches,
+                   'uptime': round(time.time() - self.datastore.start_time, 2),
+                   'watch_count': len(self.datastore.data.get('watching', {}))
+               }, 200

changedetectionio/blueprint/browser_steps/__init__.py

@@ -0,0 +1,239 @@
+
+# HORRIBLE HACK BUT WORKS :-) PR anyone?
+#
+# Why?
+# `browsersteps_playwright_browser_interface.chromium.connect_over_cdp()` will only run once without async()
+# - this flask app is not async()
+# - browserless has a single timeout/keepalive which applies to the session made at .connect_over_cdp()
+#
+# So it means that we must unfortunately for now just keep a single timer since .connect_over_cdp() was run
+# and know when that reaches timeout/keepalive :( when that time is up, restart the connection and tell the user
+# that their time is up, insert another coin. (reload)
+#
+# Bigger picture
+# - It's horrible that we have this click+wait deal, some nice socket.io solution using something similar
+# to what the browserless debug UI already gives us would be smarter..
+#
+# OR
+# - Some API call that should be hacked into browserless or playwright that we can "/api/bump-keepalive/{session_id}/60"
+# So we can tell it that we need more time (run this on each action)
+#
+# OR
+# - use multiprocessing to bump this over to its own process and add some transport layer (queue/pipes)
+
+
+
+
+from distutils.util import strtobool
+from flask import Blueprint, request, make_response
+from flask_login import login_required
+import os
+import logging
+from changedetectionio.store import ChangeDetectionStore
+
+browsersteps_live_ui_o = {}
+browsersteps_playwright_browser_interface = None
+browsersteps_playwright_browser_interface_start_time = None
+browsersteps_playwright_browser_interface_browser = None
+browsersteps_playwright_browser_interface_end_time = None
+
+
+def cleanup_playwright_session():
+    print("Cleaning up old playwright session because time was up")
+    global browsersteps_playwright_browser_interface
+    global browsersteps_live_ui_o
+    global browsersteps_playwright_browser_interface_browser
+    global browsersteps_playwright_browser_interface
+    global browsersteps_playwright_browser_interface_start_time
+    global browsersteps_playwright_browser_interface_end_time
+
+    import psutil
+
+    current_process = psutil.Process()
+    children = current_process.children(recursive=True)
+    for child in children:
+        print (child)
+        print('Child pid is {}'.format(child.pid))
+
+    # .stop() hangs sometimes if its called when there are no children to process
+    # but how do we know this is our child? dunno
+    if children:
+        browsersteps_playwright_browser_interface.stop()
+
+    browsersteps_live_ui_o = {}
+    browsersteps_playwright_browser_interface = None
+    browsersteps_playwright_browser_interface_start_time = None
+    browsersteps_playwright_browser_interface_browser = None
+    browsersteps_playwright_browser_interface_end_time = None
+    print ("Cleaning up old playwright session because time was up - done")
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+
+    browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
+
+    @login_required
+    @browser_steps_blueprint.route("/browsersteps_update", methods=['GET', 'POST'])
+    def browsersteps_ui_update():
+        import base64
+        import playwright._impl._api_types
+        import time
+
+        from changedetectionio.blueprint.browser_steps import browser_steps
+
+        global browsersteps_live_ui_o, browsersteps_playwright_browser_interface_end_time
+        global browsersteps_playwright_browser_interface_browser
+        global browsersteps_playwright_browser_interface
+        global browsersteps_playwright_browser_interface_start_time
+
+        step_n = None
+        remaining =0
+        uuid = request.args.get('uuid')
+
+        browsersteps_session_id = request.args.get('browsersteps_session_id')
+
+        if not browsersteps_session_id:
+            return make_response('No browsersteps_session_id specified', 500)
+
+        # Because we don't "really" run in a context manager ( we make the playwright interface global/long-living )
+        # We need to manage the shutdown when the time is up
+        if browsersteps_playwright_browser_interface_end_time:
+            remaining = browsersteps_playwright_browser_interface_end_time-time.time()
+            if browsersteps_playwright_browser_interface_end_time and remaining <= 0:
+
+
+                cleanup_playwright_session()
+
+                return make_response('Browser session expired, please reload the Browser Steps interface', 500)
+
+
+        # Actions - step/apply/etc, do the thing and return state
+        if request.method == 'POST':
+            # @todo - should always be an existing session
+            step_operation = request.form.get('operation')
+            step_selector = request.form.get('selector')
+            step_optional_value = request.form.get('optional_value')
+            step_n = int(request.form.get('step_n'))
+            is_last_step = strtobool(request.form.get('is_last_step'))
+
+            if step_operation == 'Goto site':
+                step_operation = 'goto_url'
+                step_optional_value = None
+                step_selector = datastore.data['watching'][uuid].get('url')
+
+            # @todo try.. accept.. nice errors not popups..
+            try:
+
+                this_session = browsersteps_live_ui_o.get(browsersteps_session_id)
+                if not this_session:
+                    print("Browser exited")
+                    return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+                this_session.call_action(action_name=step_operation,
+                                         selector=step_selector,
+                                         optional_value=step_optional_value)
+            except playwright._impl._api_types.TimeoutError as e:
+                print("Element wasnt found :-(", step_operation)
+                return make_response("Element was not found on page", 401)
+
+            except playwright._impl._api_types.Error as e:
+                # Browser/playwright level error
+                print("Browser error - got playwright._impl._api_types.Error, try reloading the session/browser")
+                print (str(e))
+
+                # Try to find something of value to give back to the user
+                for l in str(e).splitlines():
+                    if 'DOMException' in l:
+                        return make_response(l, 401)
+
+                return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+            # Get visual selector ready/update its data (also use the current filter info from the page?)
+            # When the last 'apply' button was pressed
+            # @todo this adds overhead because the xpath selection is happening twice
+            u = this_session.page.url
+            if is_last_step and u:
+                (screenshot, xpath_data) = this_session.request_visualselector_data()
+                datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
+                datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
+
+        # Setup interface
+        if request.method == 'GET':
+
+            if not browsersteps_playwright_browser_interface:
+                print("Starting connection with playwright")
+                logging.debug("browser_steps.py connecting")
+                from playwright.sync_api import sync_playwright
+
+                browsersteps_playwright_browser_interface = sync_playwright().start()
+
+
+                time.sleep(1)
+                # At 20 minutes, some other variable is closing it
+                # @todo find out what it is and set it
+                seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
+
+                # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
+                keepalive = "&timeout={}".format(((seconds_keepalive+3) * 1000))
+                try:
+                    browsersteps_playwright_browser_interface_browser = browsersteps_playwright_browser_interface.chromium.connect_over_cdp(
+                        os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
+                except Exception as e:
+                    if 'ECONNREFUSED' in str(e):
+                        return make_response('Unable to start the Playwright session properly, is it running?', 401)
+
+                browsersteps_playwright_browser_interface_end_time = time.time() + (seconds_keepalive-3)
+                print("Starting connection with playwright - done")
+
+            if not browsersteps_live_ui_o.get(browsersteps_session_id):
+                # Boot up a new session
+                proxy_id = datastore.get_preferred_proxy_for_watch(uuid=uuid)
+                proxy = None
+                if proxy_id:
+                    proxy_url = datastore.proxy_list.get(proxy_id).get('url')
+                    if proxy_url:
+                        proxy = {'server': proxy_url}
+                        print("Browser Steps: UUID {} Using proxy {}".format(uuid, proxy_url))
+
+                # Begin the new "Playwright Context" that re-uses the playwright interface
+                # Each session is a "Playwright Context" as a list, that uses the playwright interface
+                browsersteps_live_ui_o[browsersteps_session_id] = browser_steps.browsersteps_live_ui(
+                    playwright_browser=browsersteps_playwright_browser_interface_browser,
+                    proxy=proxy)
+                this_session = browsersteps_live_ui_o[browsersteps_session_id]
+
+        if not this_session.page:
+            cleanup_playwright_session()
+            return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+        try:
+            state = this_session.get_current_state()
+        except playwright._impl._api_types.Error as e:
+            return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
+
+        # Use send_file() which is way faster than read/write loop on bytes
+        import json
+        from tempfile import mkstemp
+        from flask import send_file
+        tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
+
+        output = json.dumps({'screenshot': "data:image/png;base64,{}".format(
+            base64.b64encode(state[0]).decode('ascii')),
+            'xpath_data': state[1],
+            'session_age_start': this_session.age_start,
+            'browser_time_remaining': round(remaining)
+        })
+
+        with os.fdopen(tmp_fd, 'w') as f:
+            f.write(output)
+
+        response = make_response(send_file(path_or_file=tmp_file,
+                                           mimetype='application/json; charset=UTF-8',
+                                           etag=True))
+        # No longer needed
+        os.unlink(tmp_file)
+
+        return response
+
+    return browser_steps_blueprint
+
+

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -0,0 +1,270 @@
+#!/usr/bin/python3
+
+import os
+import time
+import re
+from random import randint
+
+# Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
+# 0- off, 1- on
+browser_step_ui_config = {'Choose one': '0 0',
+                          #                 'Check checkbox': '1 0',
+                          #                 'Click button containing text': '0 1',
+                          #                 'Scroll to bottom': '0 0',
+                          #                 'Scroll to element': '1 0',
+                          #                 'Scroll to top': '0 0',
+                          #                 'Switch to iFrame by index number': '0 1'
+                          #                 'Uncheck checkbox': '1 0',
+                          # @todo
+                          'Check checkbox': '1 0',
+                          'Click X,Y': '0 1',
+                          'Click element if exists': '1 0',
+                          'Click element': '1 0',
+                          'Click element containing text': '0 1',
+                          'Enter text in field': '1 1',
+                          'Execute JS': '0 1',
+#                          'Extract text and use as filter': '1 0',
+                          'Goto site': '0 0',
+                          'Press Enter': '0 0',
+                          'Select by label': '1 1',
+                          'Scroll down': '0 0',
+                          'Uncheck checkbox': '1 0',
+                          'Wait for seconds': '0 1',
+                          'Wait for text': '0 1',
+                          #                          'Press Page Down': '0 0',
+                          #                          'Press Page Up': '0 0',
+                          # weird bug, come back to it later
+                          }
+
+
+# Good reference - https://playwright.dev/python/docs/input
+#                  https://pythonmana.com/2021/12/202112162236307035.html
+#
+# ONLY Works in Playwright because we need the fullscreen screenshot
+class steppable_browser_interface():
+    page = None
+
+    # Convert and perform "Click Button" for example
+    def call_action(self, action_name, selector=None, optional_value=None):
+        now = time.time()
+        call_action_name = re.sub('[^0-9a-zA-Z]+', '_', action_name.lower())
+        if call_action_name == 'choose_one':
+            return
+
+        print("> action calling", call_action_name)
+        # https://playwright.dev/python/docs/selectors#xpath-selectors
+        if selector.startswith('/') and not selector.startswith('//'):
+            selector = "xpath=" + selector
+
+        action_handler = getattr(self, "action_" + call_action_name)
+
+        # Support for Jinja2 variables in the value and selector
+        from jinja2 import Environment
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
+
+        if selector and ('{%' in selector or '{{' in selector):
+            selector = str(jinja2_env.from_string(selector).render())
+
+        if optional_value and ('{%' in optional_value or '{{' in optional_value):
+            optional_value = str(jinja2_env.from_string(optional_value).render())
+
+        action_handler(selector, optional_value)
+        self.page.wait_for_timeout(3 * 1000)
+        print("Call action done in", time.time() - now)
+
+    def action_goto_url(self, url, optional_value):
+        # self.page.set_viewport_size({"width": 1280, "height": 5000})
+        now = time.time()
+        response = self.page.goto(url, timeout=0, wait_until='domcontentloaded')
+        print("Time to goto URL", time.time() - now)
+
+        # Wait_until = commit
+        # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
+        # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
+        # This seemed to solve nearly all 'TimeoutErrors'
+        extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))
+        self.page.wait_for_timeout(extra_wait * 1000)
+
+    def action_click_element_containing_text(self, selector=None, value=''):
+        if not len(value.strip()):
+            return
+        elem = self.page.get_by_text(value)
+        if elem.count():
+            elem.first.click(delay=randint(200, 500))
+
+    def action_enter_text_in_field(self, selector, value):
+        if not len(selector.strip()):
+            return
+
+        self.page.fill(selector, value, timeout=10 * 1000)
+
+    def action_execute_js(self, selector, value):
+        self.page.evaluate(value)
+
+    def action_click_element(self, selector, value):
+        print("Clicking element")
+        if not len(selector.strip()):
+            return
+        self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
+
+    def action_click_element_if_exists(self, selector, value):
+        import playwright._impl._api_types as _api_types
+        print("Clicking element if exists")
+        if not len(selector.strip()):
+            return
+        try:
+            self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
+        except _api_types.TimeoutError as e:
+            return
+        except _api_types.Error as e:
+            # Element was there, but page redrew and now its long long gone
+            return
+
+    def action_click_x_y(self, selector, value):
+        x, y = value.strip().split(',')
+        x = int(float(x.strip()))
+        y = int(float(y.strip()))
+        self.page.mouse.click(x=x, y=y, delay=randint(200, 500))
+
+    def action_scroll_down(self, selector, value):
+        # Some sites this doesnt work on for some reason
+        self.page.mouse.wheel(0, 600)
+        self.page.wait_for_timeout(1000)
+
+    def action_wait_for_seconds(self, selector, value):
+        self.page.wait_for_timeout(int(value) * 1000)
+
+    # @todo - in the future make some popout interface to capture what needs to be set
+    # https://playwright.dev/python/docs/api/class-keyboard
+    def action_press_enter(self, selector, value):
+        self.page.keyboard.press("Enter", delay=randint(200, 500))
+
+    def action_press_page_up(self, selector, value):
+        self.page.keyboard.press("PageUp", delay=randint(200, 500))
+
+    def action_press_page_down(self, selector, value):
+        self.page.keyboard.press("PageDown", delay=randint(200, 500))
+
+    def action_check_checkbox(self, selector, value):
+        self.page.locator(selector).check()
+
+    def action_uncheck_checkbox(self, selector, value):
+        self.page.locator(selector).uncheck()
+
+
+# Responsible for maintaining a live 'context' with browserless
+# @todo - how long do contexts live for anyway?
+class browsersteps_live_ui(steppable_browser_interface):
+    context = None
+    page = None
+    render_extra_delay = 1
+    stale = False
+    # bump and kill this if idle after X sec
+    age_start = 0
+
+    # use a special driver, maybe locally etc
+    command_executor = os.getenv(
+        "PLAYWRIGHT_BROWSERSTEPS_DRIVER_URL"
+    )
+    # if not..
+    if not command_executor:
+        command_executor = os.getenv(
+            "PLAYWRIGHT_DRIVER_URL",
+            'ws://playwright-chrome:3000'
+        ).strip('"')
+
+    browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
+
+    def __init__(self, playwright_browser, proxy=None):
+        self.age_start = time.time()
+        self.playwright_browser = playwright_browser
+        if self.context is None:
+            self.connect(proxy=proxy)
+
+    # Connect and setup a new context
+    def connect(self, proxy=None):
+        # Should only get called once - test that
+        keep_open = 1000 * 60 * 5
+        now = time.time()
+
+        # @todo handle multiple contexts, bind a unique id from the browser on each req?
+        self.context = self.playwright_browser.new_context(
+            # @todo
+            #                user_agent=request_headers['User-Agent'] if request_headers.get('User-Agent') else 'Mozilla/5.0',
+            #               proxy=self.proxy,
+            # This is needed to enable JavaScript execution on GitHub and others
+            bypass_csp=True,
+            # Should never be needed
+            accept_downloads=False,
+            proxy=proxy
+        )
+
+        self.page = self.context.new_page()
+
+        # self.page.set_default_navigation_timeout(keep_open)
+        self.page.set_default_timeout(keep_open)
+        # @todo probably this doesnt work
+        self.page.on(
+            "close",
+            self.mark_as_closed,
+        )
+        # Listen for all console events and handle errors
+        self.page.on("console", lambda msg: print(f"Browser steps console - {msg.type}: {msg.text} {msg.args}"))
+
+        print("time to browser setup", time.time() - now)
+        self.page.wait_for_timeout(1 * 1000)
+
+    def mark_as_closed(self):
+        print("Page closed, cleaning up..")
+
+    @property
+    def has_expired(self):
+        if not self.page:
+            return True
+
+
+    def get_current_state(self):
+        """Return the screenshot and interactive elements mapping, generally always called after action_()"""
+        from pkg_resources import resource_string
+        xpath_element_js = resource_string(__name__, "../../res/xpath_element_scraper.js").decode('utf-8')
+        now = time.time()
+        self.page.wait_for_timeout(1 * 1000)
+
+        # The actual screenshot
+        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
+
+        self.page.evaluate("var include_filters=''")
+        # Go find the interactive elements
+        # @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers?
+        elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
+        xpath_element_js = xpath_element_js.replace('%ELEMENTS%', elements)
+        xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
+        # So the JS will find the smallest one first
+        xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
+        print("Time to complete get_current_state of browser", time.time() - now)
+        # except
+        # playwright._impl._api_types.Error: Browser closed.
+        # @todo show some countdown timer?
+        return (screenshot, xpath_data)
+
+    def request_visualselector_data(self):
+        """
+        Does the same that the playwright operation in content_fetcher does
+        This is used to just bump the VisualSelector data so it' ready to go if they click on the tab
+        @todo refactor and remove duplicate code, add include_filters
+        :param xpath_data:
+        :param screenshot:
+        :param current_include_filters:
+        :return:
+        """
+
+        self.page.evaluate("var include_filters=''")
+        from pkg_resources import resource_string
+        # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
+        xpath_element_js = resource_string(__name__, "../../res/xpath_element_scraper.js").decode('utf-8')
+        from changedetectionio.content_fetcher import visualselector_xpath_selectors
+        xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
+        xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
+        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
+
+        return (screenshot, xpath_data)

changedetectionio/changedetection.py

@@ -2,19 +2,20 @@
 
 # Launch as a eventlet.wsgi server instance.
 
+from distutils.util import strtobool
+import eventlet
+import eventlet.wsgi
 import getopt
 import os
 import signal
 import sys
 
-import eventlet
-import eventlet.wsgi
 from . import store, changedetection_app, content_fetcher
 from . import __version__
 
 # Only global so we can access it in the signal handler
-datastore = None
 app = None
+datastore = None
 
 def sigterm_handler(_signo, _stack_frame):
     global app
@@ -102,6 +103,15 @@ def main():
                     has_password=datastore.data['settings']['application']['password'] != False
                     )
 
+    # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.
+    # @Note: Incompatible with password login (and maybe other features) for now, submit a PR!
+    @app.after_request
+    def hide_referrer(response):
+        if strtobool(os.getenv("HIDE_REFERER", 'false')):
+            response.headers["Referrer-Policy"] = "no-referrer"
+
+        return response
+
     # Proxy sub-directory support
     # Set environment var USE_X_SETTINGS=1 on this script
     # And then in your proxy_pass settings

changedetectionio/content_fetcher.py

@@ -1,11 +1,13 @@
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 import chardet
 import json
+import logging
 import os
 import requests
-import time
 import sys
+import time
 
+visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary'
 
 class Non200ErrorCodeReceived(Exception):
     def __init__(self, status_code, url, screenshot=None, xpath_data=None, page_html=None):
@@ -30,6 +32,12 @@ class JSActionExceptions(Exception):
         self.message = message
         return
 
+class BrowserStepsStepTimout(Exception):
+    def __init__(self, step_n):
+        self.step_n = step_n
+        return
+
+
 class PageUnloadable(Exception):
     def __init__(self, status_code, url, screenshot=False, message=False):
         # Set this so we can use it in other parts of the app
@@ -70,134 +78,13 @@ class Fetcher():
     status_code = None
     content = None
     headers = None
+    browser_steps = None
+    browser_steps_screenshot_path = None
 
     fetcher_description = "No description"
     webdriver_js_execute_code = None
-    xpath_element_js = """               
-                // Include the getXpath script directly, easier than fetching
-                !function(e,n){"object"==typeof exports&&"undefined"!=typeof module?module.exports=n():"function"==typeof define&&define.amd?define(n):(e=e||self).getXPath=n()}(this,function(){return function(e){var n=e;if(n&&n.id)return'//*[@id="'+n.id+'"]';for(var o=[];n&&Node.ELEMENT_NODE===n.nodeType;){for(var i=0,r=!1,d=n.previousSibling;d;)d.nodeType!==Node.DOCUMENT_TYPE_NODE&&d.nodeName===n.nodeName&&i++,d=d.previousSibling;for(d=n.nextSibling;d;){if(d.nodeName===n.nodeName){r=!0;break}d=d.nextSibling}o.push((n.prefix?n.prefix+":":"")+n.localName+(i||r?"["+(i+1)+"]":"")),n=n.parentNode}return o.length?"/"+o.reverse().join("/"):""}});
+    xpath_element_js = ""
 
-
-                const findUpTag = (el) => {
-                  let r = el
-                  chained_css = [];
-                  depth=0;
-            
-                // Strategy 1: Keep going up until we hit an ID tag, imagine it's like  #list-widget div h4
-                  while (r.parentNode) {
-                    if(depth==5) {
-                      break;
-                    }
-                    if('' !==r.id) {
-                      chained_css.unshift("#"+CSS.escape(r.id));
-                      final_selector= chained_css.join(' > ');
-                      // Be sure theres only one, some sites have multiples of the same ID tag :-(
-                      if (window.document.querySelectorAll(final_selector).length ==1 ) {
-                        return final_selector;
-                        }
-                      return null;
-                    } else {
-                      chained_css.unshift(r.tagName.toLowerCase());
-                    }
-                    r=r.parentNode;
-                    depth+=1;
-                  }
-                  return null;
-                }
-
-
-                // @todo - if it's SVG or IMG, go into image diff mode
-                var elements = window.document.querySelectorAll("div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary");
-                var size_pos=[];
-                // after page fetch, inject this JS
-                // build a map of all elements and their positions (maybe that only include text?)
-                var bbox;
-                for (var i = 0; i < elements.length; i++) {   
-                 bbox = elements[i].getBoundingClientRect();
-
-                 // forget really small ones
-                 if (bbox['width'] <20 && bbox['height'] < 20 ) {
-                   continue;
-                 }
-
-                 // @todo the getXpath kind of sucks, it doesnt know when there is for example just one ID sometimes
-                 // it should not traverse when we know we can anchor off just an ID one level up etc..
-                 // maybe, get current class or id, keep traversing up looking for only class or id until there is just one match 
-
-                 // 1st primitive - if it has class, try joining it all and select, if theres only one.. well thats us.
-                 xpath_result=false;
-                 
-                 try {
-                   var d= findUpTag(elements[i]);
-                   if (d) {
-                     xpath_result =d;
-                   }                
-                 } catch (e) {
-                   console.log(e);
-                 }
-                 
-                 // You could swap it and default to getXpath and then try the smarter one
-                 // default back to the less intelligent one
-                 if (!xpath_result) {
-                    try {
-                       // I've seen on FB and eBay that this doesnt work
-                       // ReferenceError: getXPath is not defined at eval (eval at evaluate (:152:29), <anonymous>:67:20) at UtilityScript.evaluate (<anonymous>:159:18) at UtilityScript.<anonymous> (<anonymous>:1:44)
-                       xpath_result = getXPath(elements[i]);
-                     } catch (e) {
-                       console.log(e);
-                       continue;
-                     }            
-                 }
-                 
-                 if(window.getComputedStyle(elements[i]).visibility === "hidden") {
-                   continue;
-                 }
-
-                 size_pos.push({
-                   xpath: xpath_result,
-                   width: Math.round(bbox['width']), 
-                   height: Math.round(bbox['height']), 
-                   left: Math.floor(bbox['left']), 
-                   top: Math.floor(bbox['top']), 
-                   childCount: elements[i].childElementCount
-                 });                 
-                }
-
-
-                // inject the current one set in the css_filter, which may be a CSS rule
-                // used for displaying the current one in VisualSelector, where its not one we generated.
-                if (css_filter.length) {
-                   q=false;                   
-                   try {
-                       // is it xpath?
-                       if (css_filter.startsWith('/') || css_filter.startsWith('xpath:')) {
-                         q=document.evaluate(css_filter.replace('xpath:',''), document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
-                       } else {
-                         q=document.querySelector(css_filter);
-                       }                       
-                   } catch (e) {
-                    // Maybe catch DOMException and alert? 
-                     console.log(e);                       
-                   }
-                   bbox=false;
-                   if(q) {
-                     bbox = q.getBoundingClientRect();
-                   }
-                                   
-                   if (bbox && bbox['width'] >0 && bbox['height']>0) {                       
-                       size_pos.push({
-                           xpath: css_filter,
-                           width: bbox['width'], 
-                           height: bbox['height'],
-                           left: bbox['left'],
-                           top: bbox['top'],
-                           childCount: q.childElementCount
-                         });
-                     }
-                }
-                // Window.width required for proper scaling in the frontend
-                return {'size_pos':size_pos, 'browser_width': window.innerWidth};
-    """
     xpath_data = None
 
     # Will be needed in the future by the VisualSelector, always get this where possible.
@@ -208,6 +95,12 @@ class Fetcher():
     # Time ONTOP of the system defined env minimum time
     render_extract_delay = 0
 
+    def __init__(self):
+        from pkg_resources import resource_string
+        # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
+        self.xpath_element_js = resource_string(__name__, "res/xpath_element_scraper.js").decode('utf-8')
+
+
     @abstractmethod
     def get_error(self):
         return self.error
@@ -220,7 +113,7 @@ class Fetcher():
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_css_filter=None):
+            current_include_filters=None):
         # Should set self.error, self.status_code and self.content
         pass
 
@@ -232,11 +125,62 @@ class Fetcher():
     def get_last_status_code(self):
         return self.status_code
 
+    @abstractmethod
+    def screenshot_step(self, step_n):
+        return None
+
     @abstractmethod
     # Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
     def is_ready(self):
         return True
 
+    def iterate_browser_steps(self):
+        from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
+        from playwright._impl._api_types import TimeoutError
+        from jinja2 import Environment
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
+
+        step_n = 0
+
+        if self.browser_steps is not None and len(self.browser_steps):
+            interface = steppable_browser_interface()
+            interface.page = self.page
+
+            valid_steps = filter(lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'), self.browser_steps)
+
+            for step in valid_steps:
+                step_n += 1
+                print(">> Iterating check - browser Step n {} - {}...".format(step_n, step['operation']))
+                self.screenshot_step("before-"+str(step_n))
+                self.save_step_html("before-"+str(step_n))
+                try:
+                    optional_value = step['optional_value']
+                    selector = step['selector']
+                    # Support for jinja2 template in step values, with date module added
+                    if '{%' in step['optional_value'] or '{{' in step['optional_value']:
+                        optional_value = str(jinja2_env.from_string(step['optional_value']).render())
+                    if '{%' in step['selector'] or '{{' in step['selector']:
+                        selector = str(jinja2_env.from_string(step['selector']).render())
+
+                    getattr(interface, "call_action")(action_name=step['operation'],
+                                                      selector=selector,
+                                                      optional_value=optional_value)
+                    self.screenshot_step(step_n)
+                    self.save_step_html(step_n)
+                except TimeoutError:
+                    # Stop processing here
+                    raise BrowserStepsStepTimout(step_n=step_n)
+
+
+
+    # It's always good to reset these
+    def delete_browser_steps_screenshots(self):
+        import glob
+        if self.browser_steps_screenshot_path is not None:
+            dest = os.path.join(self.browser_steps_screenshot_path, 'step_*.jpeg')
+            files = glob.glob(dest)
+            for f in files:
+                os.unlink(f)
 
 #   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
 #   the current one would return javascript output (as we use JS to generate the diff)
@@ -255,7 +199,6 @@ def available_fetchers():
 
     return p
 
-
 class base_html_playwright(Fetcher):
     fetcher_description = "Playwright {}/Javascript".format(
         os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
@@ -273,7 +216,7 @@ class base_html_playwright(Fetcher):
     proxy = None
 
     def __init__(self, proxy_override=None):
-
+        super().__init__()
         # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
         self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
         self.command_executor = os.getenv(
@@ -293,15 +236,26 @@ class base_html_playwright(Fetcher):
 
         # allow per-watch proxy selection override
         if proxy_override:
-            # https://playwright.dev/docs/network#http-proxy
-            from urllib.parse import urlparse
-            parsed = urlparse(proxy_override)
-            proxy_url = "{}://{}:{}".format(parsed.scheme, parsed.hostname, parsed.port)
-            self.proxy = {'server': proxy_url}
-            if parsed.username:
-                self.proxy['username'] = parsed.username
-            if parsed.password:
-                self.proxy['password'] = parsed.password
+            self.proxy = {'server': proxy_override}
+
+    def screenshot_step(self, step_n=''):
+
+        # There's a bug where we need to do it twice or it doesnt take the whole page, dont know why.
+        self.page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024})
+        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=85)
+
+        if self.browser_steps_screenshot_path is not None:
+            destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
+            logging.debug("Saving step screenshot to {}".format(destination))
+            with open(destination, 'wb') as f:
+                f.write(screenshot)
+
+    def save_step_html(self, step_n):
+        content = self.page.content()
+        destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
+        logging.debug("Saving step HTML to {}".format(destination))
+        with open(destination, 'w') as f:
+            f.write(content)
 
     def run(self,
             url,
@@ -310,13 +264,13 @@ class base_html_playwright(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_css_filter=None):
+            current_include_filters=None):
 
         from playwright.sync_api import sync_playwright
         import playwright._impl._api_types
-        from playwright._impl._api_types import Error, TimeoutError
-        response = None
 
+        self.delete_browser_steps_screenshots()
+        response = None
         with sync_playwright() as p:
             browser_type = getattr(p, self.browser_type)
 
@@ -336,89 +290,86 @@ class base_html_playwright(Fetcher):
                 accept_downloads=False
             )
 
+            self.page = context.new_page()
             if len(request_headers):
                 context.set_extra_http_headers(request_headers)
 
-            page = context.new_page()
             try:
-                page.set_default_navigation_timeout(90000)
-                page.set_default_timeout(90000)
+                self.page.set_default_navigation_timeout(90000)
+                self.page.set_default_timeout(90000)
 
                 # Listen for all console events and handle errors
-                page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
+                self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
 
                 # Bug - never set viewport size BEFORE page.goto
 
+
                 # Waits for the next navigation. Using Python context manager
                 # prevents a race condition between clicking and waiting for a navigation.
-                with page.expect_navigation():
-                    response = page.goto(url, wait_until='load')
+                with self.page.expect_navigation():
+                    response = self.page.goto(url, wait_until='load')
+                # Wait_until = commit
+                # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
+                # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
+                # This seemed to solve nearly all 'TimeoutErrors'
+                extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+                self.page.wait_for_timeout(extra_wait * 1000)
 
+                if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
+                    self.page.evaluate(self.webdriver_js_execute_code)
 
             except playwright._impl._api_types.TimeoutError as e:
                 context.close()
                 browser.close()
                 # This can be ok, we will try to grab what we could retrieve
                 pass
-
             except Exception as e:
-                print("other exception when page.goto")
-                print(str(e))
+                print ("other exception when page.goto")
+                print (str(e))
                 context.close()
                 browser.close()
-                raise PageUnloadable(url=url, status_code=None, message=e.message)
+                raise PageUnloadable(url=url, status_code=None)
+
 
             if response is None:
                 context.close()
                 browser.close()
-                print("response object was none")
+                print ("response object was none")
                 raise EmptyReply(url=url, status_code=None)
 
+            # Bug 2(?) Set the viewport size AFTER loading the page
+            self.page.set_viewport_size({"width": 1280, "height": 1024})
+
+            # Run Browser Steps here
+            self.iterate_browser_steps()
 
-            # Removed browser-set-size, seemed to be needed to make screenshots work reliably in older playwright versions
-            # Was causing exceptions like 'waiting for page but content is changing' etc
-            # https://www.browserstack.com/docs/automate/playwright/change-browser-window-size 1280x720 should be the default
-                        
             extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
             time.sleep(extra_wait)
 
-            if self.webdriver_js_execute_code is not None:
-                try:
-                    page.evaluate(self.webdriver_js_execute_code)
-                except Exception as e:
-                    # Is it possible to get a screenshot?
-                    error_screenshot = False
-                    try:
-                        page.screenshot(type='jpeg',
-                                        clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024},
-                                        quality=1)
 
-                        # The actual screenshot
-                        error_screenshot = page.screenshot(type='jpeg',
-                                                           full_page=True,
-                                                           quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
-                    except Exception as s:
-                        pass
-
-                    raise JSActionExceptions(status_code=response.status, screenshot=error_screenshot, message=str(e), url=url)
-
-                else:
-                    # JS eval was run, now we also wait some time if possible to let the page settle
-                    if self.render_extract_delay:
-                        page.wait_for_timeout(self.render_extract_delay * 1000)
-
-            page.wait_for_timeout(500)
-
-            self.content = page.content()
+            self.content = self.page.content()
             self.status_code = response.status
+
+            if len(self.page.content().strip()) == 0:
+                context.close()
+                browser.close()
+                print ("Content was empty")
+                raise EmptyReply(url=url, status_code=None)
+
+            # Bug 2(?) Set the viewport size AFTER loading the page
+            self.page.set_viewport_size({"width": 1280, "height": 1024})
+
+            self.status_code = response.status
+            self.content = self.page.content()
             self.headers = response.all_headers()
 
-            if current_css_filter is not None:
-                page.evaluate("var css_filter={}".format(json.dumps(current_css_filter)))
+            # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
+            if current_include_filters is not None:
+                self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
             else:
-                page.evaluate("var css_filter=''")
+                self.page.evaluate("var include_filters=''")
 
-            self.xpath_data = page.evaluate("async () => {" + self.xpath_element_js + "}")
+            self.xpath_data = self.page.evaluate("async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
 
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
@@ -429,26 +380,17 @@ class base_html_playwright(Fetcher):
             # acceptable screenshot quality here
             try:
                 # Quality set to 1 because it's not used, just used as a work-around for a bug, no need to change this.
-                page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024}, quality=1)
+                self.page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024}, quality=1)
                 # The actual screenshot
-                self.screenshot = page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
+                self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
             except Exception as e:
                 context.close()
                 browser.close()
                 raise ScreenshotUnavailable(url=url, status_code=None)
 
-            if len(self.content.strip()) == 0:
-                context.close()
-                browser.close()
-                print("Content was empty")
-                raise EmptyReply(url=url, status_code=None, screenshot=self.screenshot)
-
             context.close()
             browser.close()
 
-            if not ignore_status_codes and self.status_code!=200:
-                raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, page_html=self.content, screenshot=self.screenshot)
-
 class base_html_webdriver(Fetcher):
     if os.getenv("WEBDRIVER_URL"):
         fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))
@@ -465,6 +407,7 @@ class base_html_webdriver(Fetcher):
     proxy = None
 
     def __init__(self, proxy_override=None):
+        super().__init__()
         from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
 
         # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
@@ -497,7 +440,7 @@ class base_html_webdriver(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_css_filter=None):
+            current_include_filters=None):
 
         from selenium import webdriver
         from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
@@ -541,7 +484,6 @@ class base_html_webdriver(Fetcher):
     def is_ready(self):
         from selenium import webdriver
         from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
-        from selenium.common.exceptions import WebDriverException
 
         self.driver = webdriver.Remote(
             command_executor=self.command_executor,
@@ -573,7 +515,7 @@ class html_requests(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_css_filter=None):
+            current_include_filters=None):
 
         # Make requests use a more modern looking user-agent
         if not 'User-Agent' in request_headers:

changedetectionio/fetch_site_status.py

@@ -10,6 +10,11 @@ from changedetectionio import content_fetcher, html_tools
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 
+class FilterNotFoundInResponse(ValueError):
+    def __init__(self, msg):
+        ValueError.__init__(self, msg)
+
+
 # Some common stuff here that can be moved to a base class
 # (set_proxy_from_list)
 class perform_site_check():
@@ -33,18 +38,20 @@ class perform_site_check():
 
         return regex
 
-
     def run(self, uuid):
+        from copy import deepcopy
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
 
-        watch = self.datastore.data['watching'].get(uuid)
+        # DeepCopy so we can be sure we don't accidently change anything by reference
+        watch = deepcopy(self.datastore.data['watching'].get(uuid))
+
         if not watch:
             return
 
         # Protect against file:// access
-        if re.search(r'^file', watch['url'], re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
+        if re.search(r'^file', watch.get('url', ''), re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
             raise Exception(
                 "file:// type access is denied for security reasons."
             )
@@ -52,10 +59,10 @@ class perform_site_check():
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
 
-        extra_headers =self.datastore.data['watching'][uuid].get('headers')
+        extra_headers = watch.get('headers', [])
 
         # Tweak the base config with the per-watch ones
-        request_headers = self.datastore.data['settings']['headers'].copy()
+        request_headers = deepcopy(self.datastore.data['settings']['headers'])
         request_headers.update(extra_headers)
 
         # https://github.com/psf/requests/issues/4525
@@ -65,7 +72,9 @@ class perform_site_check():
             request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
 
         timeout = self.datastore.data['settings']['requests'].get('timeout')
-        url = watch.get('url')
+
+        url = watch.link
+
         request_body = self.datastore.data['watching'][uuid].get('body')
         request_method = self.datastore.data['watching'][uuid].get('method')
         ignore_status_codes = self.datastore.data['watching'][uuid].get('ignore_status_codes', False)
@@ -77,7 +86,7 @@ class perform_site_check():
             is_source = True
 
         # Pluggable content fetcher
-        prefer_backend = watch['fetch_backend']
+        prefer_backend = watch.get('fetch_backend')
         if hasattr(content_fetcher, prefer_backend):
             klass = getattr(content_fetcher, prefer_backend)
         else:
@@ -88,21 +97,26 @@ class perform_site_check():
         proxy_url = None
         if proxy_id:
             proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')
-            print ("UUID {} Using proxy {}".format(uuid, proxy_url))
+            print("UUID {} Using proxy {}".format(uuid, proxy_url))
 
         fetcher = klass(proxy_override=proxy_url)
 
         # Configurable per-watch or global extra delay before extracting text (for webDriver types)
         system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)
         if watch['webdriver_delay'] is not None:
-            fetcher.render_extract_delay = watch['webdriver_delay']
+            fetcher.render_extract_delay = watch.get('webdriver_delay')
         elif system_webdriver_delay is not None:
             fetcher.render_extract_delay = system_webdriver_delay
 
-        if watch['webdriver_js_execute_code'] is not None and watch['webdriver_js_execute_code'].strip():
-            fetcher.webdriver_js_execute_code = watch['webdriver_js_execute_code']
+        # Possible conflict
+        if prefer_backend == 'html_webdriver':
+            fetcher.browser_steps = watch.get('browser_steps', None)
+            fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, uuid)
 
-        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch['css_filter'])
+        if watch.get('webdriver_js_execute_code') is not None and watch.get('webdriver_js_execute_code').strip():
+            fetcher.webdriver_js_execute_code = watch.get('webdriver_js_execute_code')
+
+        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'))
         fetcher.quit()
 
         self.screenshot = fetcher.screenshot
@@ -126,28 +140,30 @@ class perform_site_check():
             is_html = False
             is_json = False
 
-        css_filter_rule = watch['css_filter']
+        include_filters_rule = watch.get('include_filters', [])
+        # include_filters_rule = watch['include_filters']
         subtractive_selectors = watch.get(
             "subtractive_selectors", []
         ) + self.datastore.data["settings"]["application"].get(
             "global_subtractive_selectors", []
         )
 
-        has_filter_rule = css_filter_rule and len(css_filter_rule.strip())
+        has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip())
         has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
 
         if is_json and not has_filter_rule:
-            css_filter_rule = "json:$"
+            include_filters_rule.append("json:$")
             has_filter_rule = True
 
         if has_filter_rule:
             json_filter_prefixes = ['json:', 'jq:']
-            if any(prefix in css_filter_rule for prefix in json_filter_prefixes):
-                stripped_text_from_html = html_tools.extract_json_as_string(content=fetcher.content, json_filter=css_filter_rule)
-                is_html = False
+            for filter in include_filters_rule:
+                if any(prefix in filter for prefix in json_filter_prefixes):
+                    stripped_text_from_html += html_tools.extract_json_as_string(content=fetcher.content, json_filter=filter)
+                    is_html = False
 
         if is_html or is_source:
-            
+
             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
             fetcher.content = html_tools.workarounds_for_obfuscations(fetcher.content)
             html_content = fetcher.content
@@ -159,33 +175,36 @@ class perform_site_check():
             else:
                 # Then we assume HTML
                 if has_filter_rule:
-                    # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
-                    if css_filter_rule[0] == '/' or css_filter_rule.startswith('xpath:'):
-                        html_content = html_tools.xpath_filter(xpath_filter=css_filter_rule.replace('xpath:', ''),
-                                                               html_content=fetcher.content)
-                    else:
-                        # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
-                        html_content = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
+                    html_content = ""
+                    for filter_rule in include_filters_rule:
+                        # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
+                        if filter_rule[0] == '/' or filter_rule.startswith('xpath:'):
+                            html_content += html_tools.xpath_filter(xpath_filter=filter_rule.replace('xpath:', ''),
+                                                                    html_content=fetcher.content,
+                                                                    append_pretty_line_formatting=not is_source)
+                        else:
+                            # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+                            html_content += html_tools.include_filters(include_filters=filter_rule,
+                                                                       html_content=fetcher.content,
+                                                                       append_pretty_line_formatting=not is_source)
+
+                    if not html_content.strip():
+                        raise FilterNotFoundInResponse(include_filters_rule)
 
                 if has_subtractive_selectors:
                     html_content = html_tools.element_removal(subtractive_selectors, html_content)
 
-                if not is_source:
+                if is_source:
+                    stripped_text_from_html = html_content
+                else:
                     # extract text
+                    do_anchor = self.datastore.data["settings"]["application"].get("render_anchor_tag_content", False)
                     stripped_text_from_html = \
                         html_tools.html_to_text(
                             html_content,
-                            render_anchor_tag_content=self.datastore.data["settings"][
-                                "application"].get(
-                                "render_anchor_tag_content", False)
+                            render_anchor_tag_content=do_anchor
                         )
 
-                elif is_source:
-                    stripped_text_from_html = html_content
-
-            # Re #340 - return the content before the 'ignore text' was applied
-            text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
-
         # Re #340 - return the content before the 'ignore text' was applied
         text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
@@ -218,7 +237,7 @@ class perform_site_check():
 
                 for l in result:
                     if type(l) is tuple:
-                        #@todo - some formatter option default (between groups)
+                        # @todo - some formatter option default (between groups)
                         regex_matched_output += list(l) + [b'\n']
                     else:
                         # @todo - some formatter option default (between each ungrouped result)
@@ -232,7 +251,6 @@ class perform_site_check():
                 stripped_text_from_html = b''.join(regex_matched_output)
                 text_content_before_ignored_filter = stripped_text_from_html
 
-
         # Re #133 - if we should strip whitespaces from triggering the change detected comparison
         if self.datastore.data['settings']['application'].get('ignore_whitespace', False):
             fetched_md5 = hashlib.md5(stripped_text_from_html.translate(None, b'\r\n\t ')).hexdigest()
@@ -242,29 +260,30 @@ class perform_site_check():
         ############ Blocking rules, after checksum #################
         blocked = False
 
-        if len(watch['trigger_text']):
+        trigger_text = watch.get('trigger_text', [])
+        if len(trigger_text):
             # Assume blocked
             blocked = True
             # Filter and trigger works the same, so reuse it
             # It should return the line numbers that match
             result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),
-                                                  wordlist=watch['trigger_text'],
+                                                  wordlist=trigger_text,
                                                   mode="line numbers")
             # Unblock if the trigger was found
             if result:
                 blocked = False
 
-
-        if len(watch['text_should_not_be_present']):
+        text_should_not_be_present = watch.get('text_should_not_be_present', [])
+        if len(text_should_not_be_present):
             # If anything matched, then we should block a change from happening
             result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),
-                                                  wordlist=watch['text_should_not_be_present'],
+                                                  wordlist=text_should_not_be_present,
                                                   mode="line numbers")
             if result:
                 blocked = True
 
         # The main thing that all this at the moment comes down to :)
-        if watch['previous_md5'] != fetched_md5:
+        if watch.get('previous_md5') != fetched_md5:
             changed_detected = True
 
         # Looks like something changed, but did it match all the rules?
@@ -273,7 +292,7 @@ class perform_site_check():
 
         # Extract title as title
         if is_html:
-            if self.datastore.data['settings']['application']['extract_title_as_title'] or watch['extract_title_as_title']:
+            if self.datastore.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
                 if not watch['title'] or not len(watch['title']):
                     update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
 

changedetectionio/forms.py

@@ -1,11 +1,10 @@
+import os
 import re
 
 from wtforms import (
     BooleanField,
-    Field,
     Form,
     IntegerField,
-    PasswordField,
     RadioField,
     SelectField,
     StringField,
@@ -13,15 +12,17 @@ from wtforms import (
     TextAreaField,
     fields,
     validators,
-    widgets,
+    widgets
 )
+from wtforms.fields import FieldList
 from wtforms.validators import ValidationError
 
+# default
+# each select <option data-enabled="enabled-0-0"
+from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
+
 from changedetectionio import content_fetcher
 from changedetectionio.notification import (
-    default_notification_body,
-    default_notification_format,
-    default_notification_title,
     valid_notification_formats,
 )
 
@@ -303,12 +304,16 @@ class ValidateCSSJSONXPATHInput(object):
 
                 # Re #265 - maybe in the future fetch the page and offer a
                 # warning/notice that its possible the rule doesnt yet match anything?
-
-            if 'jq:' in line:
                 if not self.allow_json:
                     raise ValidationError("jq not permitted in this field!")
 
-                import jq
+            if 'jq:' in line:
+                try:
+                    import jq
+                except ModuleNotFoundError:
+                    # `jq` requires full compilation in windows and so isn't generally available
+                    raise ValidationError("jq not support not found")
+
                 input = line.replace('jq:', '')
 
                 try:
@@ -319,7 +324,6 @@ class ValidateCSSJSONXPATHInput(object):
                 except:
                     raise ValidationError("A system-error occurred when validating your jq expression")
 
-
 class quickWatchForm(Form):
     url = fields.URLField('URL', validators=[validateURL()])
     tag = StringField('Group tag', [validators.Optional()])
@@ -338,6 +342,17 @@ class commonSettingsForm(Form):
     webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1,
                                                                                                                                     message="Should contain one or more seconds")])
 
+class SingleBrowserStep(Form):
+
+    operation = SelectField('Operation', [validators.Optional()], choices=browser_step_ui_config.keys())
+
+    # maybe better to set some <script>var..
+    selector = StringField('Selector', [validators.Optional()], render_kw={"placeholder": "CSS or xPath selector"})
+    optional_value = StringField('value', [validators.Optional()], render_kw={"placeholder": "Value"})
+#   @todo move to JS? ajax fetch new field?
+#    remove_button = SubmitField('-', render_kw={"type": "button", "class": "pure-button pure-button-primary", 'title': 'Remove'})
+#    add_button = SubmitField('+', render_kw={"type": "button", "class": "pure-button pure-button-primary", 'title': 'Add new step after'})
+
 class watchForm(commonSettingsForm):
 
     url = fields.URLField('URL', validators=[validateURL()])
@@ -345,7 +360,7 @@ class watchForm(commonSettingsForm):
 
     time_between_check = FormField(TimeBetweenCheckForm)
 
-    css_filter = StringField('CSS/JSON/XPATH Filter', [ValidateCSSJSONXPATHInput()], default='')
+    include_filters = StringListField('CSS/JSONPath/JQ/XPath Filters', [ValidateCSSJSONXPATHInput()], default='')
 
     subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
 
@@ -360,8 +375,9 @@ class watchForm(commonSettingsForm):
     ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
     check_unique_lines = BooleanField('Only trigger when new lines appear', default=False)
     trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
+    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
+        browser_steps = FieldList(FormField(SingleBrowserStep), min_entries=10)
     text_should_not_be_present = StringListField('Block change-detection if text matches', [validators.Optional(), ValidateListRegex()])
-
     webdriver_js_execute_code = TextAreaField('Execute JavaScript before change detection', render_kw={"rows": "5"}, validators=[validators.Optional()])
 
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
@@ -371,6 +387,7 @@ class watchForm(commonSettingsForm):
         'Send a notification when the filter can no longer be found on the page', default=False)
 
     notification_muted = BooleanField('Notifications Muted / Off', default=False)
+    notification_screenshot = BooleanField('Attach screenshot to notification (where possible)', default=False)
 
     def validate(self, **kwargs):
         if not super().validate():
@@ -383,6 +400,15 @@ class watchForm(commonSettingsForm):
             self.body.errors.append('Body must be empty when Request Method is set to GET')
             result = False
 
+        # Attempt to validate jinja2 templates in the URL
+        from jinja2 import Environment
+        # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
+        try:
+            ready_url = str(jinja2_env.from_string(self.url.data).render())
+        except Exception as e:
+            self.url.errors.append('Invalid template syntax')
+            result = False
         return result
 
 

changedetectionio/html_tools.py

@@ -1,33 +1,36 @@
-import json
-from typing import List
 
 from bs4 import BeautifulSoup
-from jsonpath_ng.ext import parse
-import jq
-import re
 from inscriptis import get_text
 from inscriptis.model.config import ParserConfig
+from jsonpath_ng.ext import parse
+from typing import List
+import json
+import re
 
-class FilterNotFoundInResponse(ValueError):
-    def __init__(self, msg):
-        ValueError.__init__(self, msg)
+# HTML added to be sure each result matching a filter (.example) gets converted to a new line by Inscriptis
+TEXT_FILTER_LIST_LINE_SUFFIX = "<br/>"
 
 class JSONNotFound(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
-
-
+        
 # Given a CSS Rule, and a blob of HTML, return the blob of HTML that matches
-def css_filter(css_filter, html_content):
+def include_filters(include_filters, html_content, append_pretty_line_formatting=False):
     soup = BeautifulSoup(html_content, "html.parser")
     html_block = ""
-    r = soup.select(css_filter, separator="")
-    if len(html_content) > 0 and len(r) == 0:
-        raise FilterNotFoundInResponse(css_filter)
-    for item in r:
-        html_block += str(item)
+    r = soup.select(include_filters, separator="")
 
-    return html_block + "\n"
+    for element in r:
+        # When there's more than 1 match, then add the suffix to separate each line
+        # And where the matched result doesn't include something that will cause Inscriptis to add a newline
+        # (This way each 'match' reliably has a new-line in the diff)
+        # Divs are converted to 4 whitespaces by inscriptis
+        if append_pretty_line_formatting and len(html_block) and not element.name in (['br', 'hr', 'div', 'p']):
+            html_block += TEXT_FILTER_LIST_LINE_SUFFIX
+
+        html_block += str(element)
+
+    return html_block
 
 def subtractive_css_selector(css_selector, html_content):
     soup = BeautifulSoup(html_content, "html.parser")
@@ -43,25 +46,29 @@ def element_removal(selectors: List[str], html_content):
 
 
 # Return str Utf-8 of matched rules
-def xpath_filter(xpath_filter, html_content):
+def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False):
     from lxml import etree, html
 
     tree = html.fromstring(bytes(html_content, encoding='utf-8'))
     html_block = ""
 
     r = tree.xpath(xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'})
-    if len(html_content) > 0 and len(r) == 0:
-        raise FilterNotFoundInResponse(xpath_filter)
-
     #@note: //title/text() wont work where <title>CDATA..
 
     for element in r:
+        # When there's more than 1 match, then add the suffix to separate each line
+        # And where the matched result doesn't include something that will cause Inscriptis to add a newline
+        # (This way each 'match' reliably has a new-line in the diff)
+        # Divs are converted to 4 whitespaces by inscriptis
+        if append_pretty_line_formatting and len(html_block) and (not hasattr( element, 'tag' ) or not element.tag in (['br', 'hr', 'div', 'p'])):
+            html_block += TEXT_FILTER_LIST_LINE_SUFFIX
+
         if type(element) == etree._ElementStringResult:
-            html_block += str(element) + "<br/>"
+            html_block += str(element)
         elif type(element) == etree._ElementUnicodeResult:
-            html_block += str(element) + "<br/>"
+            html_block += str(element)
         else:
-            html_block += etree.tostring(element, pretty_print=True).decode('utf-8') + "<br/>"
+            html_block += etree.tostring(element, pretty_print=True).decode('utf-8')
 
     return html_block
 
@@ -85,9 +92,18 @@ def _parse_json(json_data, json_filter):
         jsonpath_expression = parse(json_filter.replace('json:', ''))
         match = jsonpath_expression.find(json_data)
         return _get_stripped_text_from_json_match(match)
+
     if 'jq:' in json_filter:
+
+        try:
+            import jq
+        except ModuleNotFoundError:
+            # `jq` requires full compilation in windows and so isn't generally available
+            raise Exception("jq not support not found")
+
         jq_expression = jq.compile(json_filter.replace('jq:', ''))
         match = jq_expression.input(json_data).all()
+
         return _get_stripped_text_from_json_match(match)
 
 def _get_stripped_text_from_json_match(match):

changedetectionio/importer.py

@@ -103,12 +103,12 @@ class import_distill_io_json(Importer):
                     pass
                 except IndexError:
                     pass
-
+                extras['include_filters'] = []
                 try:
-                    extras['css_filter'] = d_config['selections'][0]['frames'][0]['includes'][0]['expr']
                     if d_config['selections'][0]['frames'][0]['includes'][0]['type'] == 'xpath':
-                        extras['css_filter'] = 'xpath:' + extras['css_filter']
-
+                        extras['include_filters'].append('xpath:' + d_config['selections'][0]['frames'][0]['includes'][0]['expr'])
+                    else:
+                        extras['include_filters'].append(d_config['selections'][0]['frames'][0]['includes'][0]['expr'])
                 except KeyError:
                     pass
                 except IndexError:

changedetectionio/model/Watch.py

@@ -1,6 +1,8 @@
-import os
-import uuid as uuid_builder
 from distutils.util import strtobool
+import logging
+import os
+import time
+import uuid
 
 minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
 mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
@@ -14,42 +16,44 @@ class model(dict):
     __newest_history_key = None
     __history_n=0
     __base_config = {
-            'url': None,
-            'tag': None,
-            'last_checked': 0,
-            'paused': False,
-            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
-            #'newest_history_key': 0,
-            'title': None,
-            'previous_md5': False,
-            'uuid': str(uuid_builder.uuid4()),
-            'headers': {},  # Extra headers to send
+            #'history': {},  # Dict of timestamp and output stripped filename (removed)
+            #'newest_history_key': 0, (removed, taken from history.txt index)
             'body': None,
-            'method': 'GET',
-            #'history': {},  # Dict of timestamp and output stripped filename
+            'check_unique_lines': False, # On change-detected, compare against all history if its something new
+            'check_count': 0,
+            'consecutive_filter_failures': 0, # Every time the CSS/xPath filter cannot be located, reset when all is fine.
+            'extract_text': [],  # Extract text by regex after filters
+            'extract_title_as_title': False,
+            'fetch_backend': None,
+            'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
+            'headers': {},  # Extra headers to send
             'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
-            # Custom notification content
-            'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
-            'notification_title': None,
+            'include_filters': [],
+            'last_checked': 0,
+            'last_error': False,
+            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
+            'method': 'GET',
+             # Custom notification content
             'notification_body': None,
             'notification_format': default_notification_format_for_watch,
             'notification_muted': False,
-            'css_filter': '',
-            'last_error': False,
-            'extract_text': [],  # Extract text by regex after filters
-            'subtractive_selectors': [],
-            'trigger_text': [],  # List of text or regex to wait for until a change is detected
-            'text_should_not_be_present': [], # Text that should not present
-            'fetch_backend': None,
-            'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
-            'consecutive_filter_failures': 0, # Every time the CSS/xPath filter cannot be located, reset when all is fine.
-            'extract_title_as_title': False,
-            'check_unique_lines': False, # On change-detected, compare against all history if its something new
+            'notification_title': None,
+            'notification_screenshot': False, # Include the latest screenshot if available and supported by the apprise URL
+            'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
+            'paused': False,
+            'previous_md5': False,
             'proxy': None, # Preferred proxy connection
+            'subtractive_selectors': [],
+            'tag': None,
+            'text_should_not_be_present': [], # Text that should not present
             # Re #110, so then if this is set to None, we know to use the default value instead
             # Requires setting to None on submit if it's the same as the default
             # Should be all None by default, so we use the system default in this case.
             'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
+            'title': None,
+            'trigger_text': [],  # List of text or regex to wait for until a change is detected
+            'url': None,
+            'uuid': str(uuid.uuid4()),
             'webdriver_delay': None,
             'webdriver_js_execute_code': None, # Run before change-detection
         }
@@ -60,7 +64,7 @@ class model(dict):
         self.update(self.__base_config)
         self.__datastore_path = kw['datastore_path']
 
-        self['uuid'] = str(uuid_builder.uuid4())
+        self['uuid'] = str(uuid.uuid4())
 
         del kw['datastore_path']
 
@@ -82,10 +86,30 @@ class model(dict):
         return False
 
     def ensure_data_dir_exists(self):
-        target_path = os.path.join(self.__datastore_path, self['uuid'])
-        if not os.path.isdir(target_path):
-            print ("> Creating data dir {}".format(target_path))
-            os.mkdir(target_path)
+        if not os.path.isdir(self.watch_data_dir):
+            print ("> Creating data dir {}".format(self.watch_data_dir))
+            os.mkdir(self.watch_data_dir)
+
+    @property
+    def link(self):
+        url = self.get('url', '')
+        ready_url = url
+        if '{%' in url or '{{' in url:
+            from jinja2 import Environment
+            # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
+            jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
+            try:
+                ready_url = str(jinja2_env.from_string(url).render())
+            except Exception as e:
+                from flask import (
+                    flash, Markup, url_for
+                )
+                message = Markup('<a href="{}#general">The URL {} is invalid and cannot be used, click to edit</a>'.format(
+                    url_for('edit_page', uuid=self.get('uuid')), self.get('url', '')))
+                flash(message, 'error')
+                return ''
+
+        return ready_url
 
     @property
     def label(self):
@@ -109,16 +133,40 @@ class model(dict):
 
     @property
     def history(self):
+        """History index is just a text file as a list
+            {watch-uuid}/history.txt
+
+            contains a list like
+
+            {epoch-time},{filename}\n
+
+            We read in this list as the history information
+
+        """
         tmp_history = {}
-        import logging
-        import time
 
         # Read the history file as a dict
-        fname = os.path.join(self.__datastore_path, self.get('uuid'), "history.txt")
+        fname = os.path.join(self.watch_data_dir, "history.txt")
         if os.path.isfile(fname):
             logging.debug("Reading history index " + str(time.time()))
             with open(fname, "r") as f:
-                tmp_history = dict(i.strip().split(',', 2) for i in f.readlines())
+                for i in f.readlines():
+                    if ',' in i:
+                        k, v = i.strip().split(',', 2)
+
+                        # The index history could contain a relative path, so we need to make the fullpath
+                        # so that python can read it
+                        if not '/' in v and not '\'' in v:
+                            v = os.path.join(self.watch_data_dir, v)
+                        else:
+                            # It's possible that they moved the datadir on older versions
+                            # So the snapshot exists but is in a different path
+                            snapshot_fname = v.split('/')[-1]
+                            proposed_new_path = os.path.join(self.watch_data_dir, snapshot_fname)
+                            if not os.path.exists(v) and os.path.exists(proposed_new_path):
+                                v = proposed_new_path
+
+                        tmp_history[k] = v
 
         if len(tmp_history):
             self.__newest_history_key = list(tmp_history.keys())[-1]
@@ -129,7 +177,7 @@ class model(dict):
 
     @property
     def has_history(self):
-        fname = os.path.join(self.__datastore_path, self.get('uuid'), "history.txt")
+        fname = os.path.join(self.watch_data_dir, "history.txt")
         return os.path.isfile(fname)
 
     # Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0.
@@ -148,31 +196,33 @@ class model(dict):
     # Save some text file to the appropriate path and bump the history
     # result_obj from fetch_site_status.run()
     def save_history_text(self, contents, timestamp):
-        import uuid
-        import logging
-
-        output_path = "{}/{}".format(self.__datastore_path, self['uuid'])
 
         self.ensure_data_dir_exists()
 
-        snapshot_fname = "{}/{}.stripped.txt".format(output_path, uuid.uuid4())
-        logging.debug("Saving history text {}".format(snapshot_fname))
+        # Small hack so that we sleep just enough to allow 1 second  between history snapshots
+        # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
+        if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
+            time.sleep(timestamp - self.__newest_history_key)
 
-        with open(snapshot_fname, 'wb') as f:
+        snapshot_fname = "{}.txt".format(str(uuid.uuid4()))
+
+        # in /diff/ and /preview/ we are going to assume for now that it's UTF-8 when reading
+        # most sites are utf-8 and some are even broken utf-8
+        with open(os.path.join(self.watch_data_dir, snapshot_fname), 'wb') as f:
             f.write(contents)
             f.close()
 
         # Append to index
         # @todo check last char was \n
-        index_fname = "{}/history.txt".format(output_path)
+        index_fname = os.path.join(self.watch_data_dir, "history.txt")
         with open(index_fname, 'a') as f:
             f.write("{},{}\n".format(timestamp, snapshot_fname))
             f.close()
 
         self.__newest_history_key = timestamp
-        self.__history_n+=1
+        self.__history_n += 1
 
-        #@todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status
+        # @todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status
         return snapshot_fname
 
     @property
@@ -205,14 +255,26 @@ class model(dict):
         return not local_lines.issubset(existing_history)
 
     def get_screenshot(self):
-        fname = os.path.join(self.__datastore_path, self['uuid'], "last-screenshot.png")
+        fname = os.path.join(self.watch_data_dir, "last-screenshot.png")
         if os.path.isfile(fname):
             return fname
 
-        return False
+        # False is not an option for AppRise, must be type None
+        return None
+
+    def get_screenshot_as_jpeg(self):
+
+        # Created by save_screenshot()
+        fname = os.path.join(self.watch_data_dir, "last-screenshot.jpg")
+        if os.path.isfile(fname):
+            return fname
+
+        # False is not an option for AppRise, must be type None
+        return None
+
 
     def __get_file_ctime(self, filename):
-        fname = os.path.join(self.__datastore_path, self['uuid'], filename)
+        fname = os.path.join(self.watch_data_dir, filename)
         if os.path.isfile(fname):
             return int(os.path.getmtime(fname))
         return False
@@ -237,9 +299,14 @@ class model(dict):
     def snapshot_error_screenshot_ctime(self):
         return self.__get_file_ctime('last-error-screenshot.png')
 
+    @property
+    def watch_data_dir(self):
+        # The base dir of the watch data
+        return os.path.join(self.__datastore_path, self['uuid'])
+    
     def get_error_text(self):
         """Return the text saved from a previous request that resulted in a non-200 error"""
-        fname = os.path.join(self.__datastore_path, self['uuid'], "last-error.txt")
+        fname = os.path.join(self.watch_data_dir, "last-error.txt")
         if os.path.isfile(fname):
             with open(fname, 'r') as f:
                 return f.read()
@@ -247,7 +314,7 @@ class model(dict):
 
     def get_error_snapshot(self):
         """Return path to the screenshot that resulted in a non-200 error"""
-        fname = os.path.join(self.__datastore_path, self['uuid'], "last-error-screenshot.png")
+        fname = os.path.join(self.watch_data_dir, "last-error-screenshot.png")
         if os.path.isfile(fname):
             return fname
         return False

changedetectionio/notification.py

@@ -101,7 +101,10 @@ def process_notification(n_object, datastore):
                 apobj.notify(
                     title=n_title,
                     body=n_body,
-                    body_format=n_format)
+                    body_format=n_format,
+                    # False is not an option for AppRise, must be type None
+                    attach=n_object.get('screenshot', None)
+                )
 
                 apobj.clear()
 

changedetectionio/res/xpath_element_scraper.js

@@ -0,0 +1,188 @@
+// @file Scrape the page looking for elements of concern (%ELEMENTS%)
+// http://matatk.agrip.org.uk/tests/position-and-width/
+// https://stackoverflow.com/questions/26813480/when-is-element-getboundingclientrect-guaranteed-to-be-updated-accurate
+//
+// Some pages like https://www.londonstockexchange.com/stock/NCCL/ncondezi-energy-limited/analysis
+// will automatically force a scroll somewhere, so include the position offset
+// Lets hope the position doesnt change while we iterate the bbox's, but this is better than nothing
+
+var scroll_y=+document.documentElement.scrollTop || document.body.scrollTop
+
+// Include the getXpath script directly, easier than fetching
+function getxpath(e) {
+        var n = e;
+        if (n && n.id) return '//*[@id="' + n.id + '"]';
+        for (var o = []; n && Node.ELEMENT_NODE === n.nodeType;) {
+            for (var i = 0, r = !1, d = n.previousSibling; d;) d.nodeType !== Node.DOCUMENT_TYPE_NODE && d.nodeName === n.nodeName && i++, d = d.previousSibling;
+            for (d = n.nextSibling; d;) {
+                if (d.nodeName === n.nodeName) {
+                    r = !0;
+                    break
+                }
+                d = d.nextSibling
+            }
+            o.push((n.prefix ? n.prefix + ":" : "") + n.localName + (i || r ? "[" + (i + 1) + "]" : "")), n = n.parentNode
+        }
+        return o.length ? "/" + o.reverse().join("/") : ""
+    }
+
+const findUpTag = (el) => {
+    let r = el
+    chained_css = [];
+    depth = 0;
+
+    //  Strategy 1: If it's an input, with name, and there's only one, prefer that
+    if (el.name !== undefined && el.name.length) {
+        var proposed = el.tagName + "[name=" + el.name + "]";
+        var proposed_element = window.document.querySelectorAll(proposed);
+        if(proposed_element.length) {
+            if (proposed_element.length === 1) {
+                return proposed;
+            } else {
+                // Some sites change ID but name= stays the same, we can hit it if we know the index
+                // Find all the elements that match and work out the input[n]
+                var n=Array.from(proposed_element).indexOf(el);
+                // Return a Playwright selector for nthinput[name=zipcode]
+                return proposed+" >> nth="+n;
+            }
+        }
+    }
+
+    // Strategy 2: Keep going up until we hit an ID tag, imagine it's like  #list-widget div h4
+    while (r.parentNode) {
+        if (depth == 5) {
+            break;
+        }
+        if ('' !== r.id) {
+            chained_css.unshift("#" + CSS.escape(r.id));
+            final_selector = chained_css.join(' > ');
+            // Be sure theres only one, some sites have multiples of the same ID tag :-(
+            if (window.document.querySelectorAll(final_selector).length == 1) {
+                return final_selector;
+            }
+            return null;
+        } else {
+            chained_css.unshift(r.tagName.toLowerCase());
+        }
+        r = r.parentNode;
+        depth += 1;
+    }
+    return null;
+}
+
+
+// @todo - if it's SVG or IMG, go into image diff mode
+// %ELEMENTS% replaced at injection time because different interfaces use it with different settings
+var elements = window.document.querySelectorAll("%ELEMENTS%");
+var size_pos = [];
+// after page fetch, inject this JS
+// build a map of all elements and their positions (maybe that only include text?)
+var bbox;
+for (var i = 0; i < elements.length; i++) {
+    bbox = elements[i].getBoundingClientRect();
+
+    // Forget really small ones
+    if (bbox['width'] < 10 && bbox['height'] < 10) {
+        continue;
+    }
+
+    // Don't include elements that are offset from canvas
+    if (bbox['top']+scroll_y < 0 || bbox['left'] < 0) {
+        continue;
+    }
+
+    // @todo the getXpath kind of sucks, it doesnt know when there is for example just one ID sometimes
+    // it should not traverse when we know we can anchor off just an ID one level up etc..
+    // maybe, get current class or id, keep traversing up looking for only class or id until there is just one match
+
+    // 1st primitive - if it has class, try joining it all and select, if theres only one.. well thats us.
+    xpath_result = false;
+
+    try {
+        var d = findUpTag(elements[i]);
+        if (d) {
+            xpath_result = d;
+        }
+    } catch (e) {
+        console.log(e);
+    }
+
+    // You could swap it and default to getXpath and then try the smarter one
+    // default back to the less intelligent one
+    if (!xpath_result) {
+        try {
+            // I've seen on FB and eBay that this doesnt work
+            // ReferenceError: getXPath is not defined at eval (eval at evaluate (:152:29), <anonymous>:67:20) at UtilityScript.evaluate (<anonymous>:159:18) at UtilityScript.<anonymous> (<anonymous>:1:44)
+            xpath_result = getxpath(elements[i]);
+        } catch (e) {
+            console.log(e);
+            continue;
+        }
+    }
+
+    if (window.getComputedStyle(elements[i]).visibility === "hidden") {
+        continue;
+    }
+
+    // @todo Possible to ONLY list where it's clickable to save JSON xfer size
+    size_pos.push({
+        xpath: xpath_result,
+        width: Math.round(bbox['width']),
+        height: Math.round(bbox['height']),
+        left: Math.floor(bbox['left']),
+        top: Math.floor(bbox['top'])+scroll_y,
+        tagName: (elements[i].tagName) ? elements[i].tagName.toLowerCase() : '',
+        tagtype: (elements[i].tagName == 'INPUT' && elements[i].type) ? elements[i].type.toLowerCase() : '',
+        isClickable: (elements[i].onclick) || window.getComputedStyle(elements[i]).cursor == "pointer"
+    });
+
+}
+
+
+// Inject the current one set in the include_filters, which may be a CSS rule
+// used for displaying the current one in VisualSelector, where its not one we generated.
+if (include_filters.length) {
+    // Foreach filter, go and find it on the page and add it to the results so we can visualise it again
+    for (const f of include_filters) {
+        bbox = false;
+        q = false;
+
+        if (!f.length) {
+            console.log("xpath_element_scraper: Empty filter, skipping");
+            continue;
+        }
+
+        try {
+            // is it xpath?
+            if (f.startsWith('/') || f.startsWith('xpath:')) {
+                q = document.evaluate(f.replace('xpath:', ''), document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+            } else {
+                q = document.querySelector(f);
+            }
+        } catch (e) {
+            // Maybe catch DOMException and alert?
+            console.log("xpath_element_scraper: Exception selecting element from filter "+f);
+            console.log(e);
+        }
+
+        if (q) {
+            bbox = q.getBoundingClientRect();
+            console.log("xpath_element_scraper: Got filter element, scroll from top was "+scroll_y)
+        } else {
+            console.log("xpath_element_scraper: filter element "+f+" was not found");
+        }
+
+        if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
+            size_pos.push({
+                xpath: f,
+                width: parseInt(bbox['width']),
+                height: parseInt(bbox['height']),
+                left: parseInt(bbox['left']),
+                top: parseInt(bbox['top'])+scroll_y
+            });
+        }
+    }
+}
+
+// Window.width required for proper scaling in the frontend
+return {'size_pos': size_pos, 'browser_width': window.innerWidth};

changedetectionio/run_all_tests.sh

@@ -9,6 +9,8 @@
 # exit when any command fails
 set -e
 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
 find tests/test_*py -type f|while read test_name
 do
   echo "TEST RUNNING $test_name"
@@ -23,6 +25,11 @@ export BASE_URL="https://really-unique-domain.io"
 pytest tests/test_notification.py
 
 
+# Re-run with HIDE_REFERER set - could affect login
+export HIDE_REFERER=True
+pytest tests/test_access_control.py
+
+
 # Now for the selenium and playwright/browserless fetchers
 # Note - this is not UI functional tests - just checking that each one can fetch the content
 
@@ -38,7 +45,9 @@ docker kill $$-test_selenium
 
 echo "TESTING WEBDRIVER FETCH > PLAYWRIGHT/BROWSERLESS..."
 # Not all platforms support playwright (not ARM/rPI), so it's not packaged in requirements.txt
-pip3 install playwright~=1.24
+PLAYWRIGHT_VERSION=$(grep -i -E "RUN pip install.+" "$SCRIPT_DIR/../Dockerfile" | grep --only-matching -i -E "playwright[=><~+]+[0-9\.]+")
+echo "using $PLAYWRIGHT_VERSION"
+pip3 install "$PLAYWRIGHT_VERSION"
 docker run -d --name $$-test_browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
 # takes a while to spin up
 sleep 5

changedetectionio/static/js/browser-steps.js

@@ -0,0 +1,425 @@
+$(document).ready(function () {
+
+    // duplicate
+    var csrftoken = $('input[name=csrf_token]').val();
+    $.ajaxSetup({
+        beforeSend: function (xhr, settings) {
+            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken)
+            }
+        }
+    })
+    var browsersteps_session_id;
+    var browserless_seconds_remaining=0;
+    var apply_buttons_disabled = false;
+    var include_text_elements = $("#include_text_elements");
+    var xpath_data;
+    var current_selected_i;
+    var state_clicked = false;
+    var c;
+
+    // redline highlight context
+    var ctx;
+    var last_click_xy = {'x': -1, 'y': -1}
+
+    $(window).resize(function () {
+        set_scale();
+    });
+
+    $('a#browsersteps-tab').click(function () {
+        start();
+    });
+
+    // Show seconds remaining until playwright/browserless needs to restart the session
+    // (See comment at the top of changedetectionio/blueprint/browser_steps/__init__.py )
+    setInterval(() => {
+        if (browserless_seconds_remaining >= 1) {
+            document.getElementById('browserless-seconds-remaining').innerText = browserless_seconds_remaining + " seconds remaining in session";
+            browserless_seconds_remaining -= 1;
+        }
+    }, "1000")
+
+
+    if (window.location.hash == '#browser-steps') {
+        start();
+    }
+
+    window.addEventListener('hashchange', function () {
+        if (window.location.hash == '#browser-steps') {
+            start();
+        }
+        // For when the page loads
+        if (!window.location.hash || window.location.hash != '#browser-steps') {
+            $("img#browsersteps-img").attr('src', '');
+            return;
+        }
+    });
+
+    function set_scale() {
+
+        // some things to check if the scaling doesnt work
+        // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
+        selector_image = $("img#browsersteps-img")[0];
+        selector_image_rect = selector_image.getBoundingClientRect();
+
+        // make the canvas and input steps the same size as the image
+        $('#browsersteps-selector-canvas').attr('height', selector_image_rect.height).attr('width', selector_image_rect.width);
+        //$('#browsersteps-selector-wrapper').attr('width', selector_image_rect.width);
+        $('#browser-steps-ui').attr('width', selector_image_rect.width);
+
+        x_scale = selector_image_rect.width / xpath_data['browser_width'];
+        y_scale = selector_image_rect.height / selector_image.naturalHeight;
+        ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
+        ctx.fillStyle = 'rgba(255,0,0, 0.1)';
+        ctx.lineWidth = 3;
+        console.log("scaling set  x: " + x_scale + " by y:" + y_scale);
+    }
+
+    // bootstrap it, this will trigger everything else
+    $('#browsersteps-img').bind('load', function () {
+        $('body').addClass('full-width');
+        console.log("Loaded background...");
+
+        document.getElementById("browsersteps-selector-canvas");
+        c = document.getElementById("browsersteps-selector-canvas");
+        // redline highlight context
+        ctx = c.getContext("2d");
+        // @todo is click better?
+        $('#browsersteps-selector-canvas').off("mousemove mousedown click");
+        // Undo disable_browsersteps_ui
+        $("#browser_steps select,input").removeAttr('disabled').css('opacity', '1.0');
+        $("#browser-steps-ui").css('opacity', '1.0');
+
+        // init
+        set_scale();
+
+        // @todo click ? some better library?
+        $('#browsersteps-selector-canvas').bind('click', function (e) {
+            // https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent
+            e.preventDefault()
+        });
+
+        $('#browsersteps-selector-canvas').bind('mousedown', function (e) {
+            // https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent
+            e.preventDefault()
+            console.log(e);
+            console.log("current xpath in index is "+current_selected_i);
+            last_click_xy = {'x': parseInt((1 / x_scale) * e.offsetX), 'y': parseInt((1 / y_scale) * e.offsetY)}
+            process_selected(current_selected_i);
+            current_selected_i = false;
+
+            // if process selected returned false, then best we can do is offer a x,y click :(
+            if (!found_something) {
+                var first_available = $("ul#browser_steps li.empty").first();
+                $('select', first_available).val('Click X,Y').change();
+                $('input[type=text]', first_available).first().val(last_click_xy['x'] + ',' + last_click_xy['y']);
+                draw_circle_on_canvas(e.offsetX, e.offsetY);
+            }
+        });
+
+        $('#browsersteps-selector-canvas').bind('mousemove', function (e) {
+            // checkbox if find elements is enabled
+            ctx.clearRect(0, 0, c.width, c.height);
+            ctx.fillStyle = 'rgba(255,0,0, 0.1)';
+            ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
+
+            // Add in offset
+            if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) {
+                var targetOffset = $(e.target).offset();
+                e.offsetX = e.pageX - targetOffset.left;
+                e.offsetY = e.pageY - targetOffset.top;
+            }
+            current_selected_i = false;
+            // Reverse order - the most specific one should be deeper/"laster"
+            // Basically, find the most 'deepest'
+            //$('#browsersteps-selector-canvas').css('cursor', 'pointer');
+            for (var i = xpath_data['size_pos'].length; i !== 0; i--) {
+                // draw all of them? let them choose somehow?
+                var sel = xpath_data['size_pos'][i - 1];
+                // If we are in a bounding-box
+                if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
+                    &&
+                    e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
+
+                ) {
+                    // Only highlight these interesting types
+                    if (1) {
+                        ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                        ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                        current_selected_i = i - 1;
+                        break;
+
+                        // find the smallest one at this x,y
+                        // does it mean sort the xpath list by size (w*h) i think so!
+                    } else {
+
+                        if ( include_text_elements[0].checked === true) {
+                        // blue one with background instead?
+                            ctx.fillStyle = 'rgba(0,0,255, 0.1)';
+                            ctx.strokeStyle = 'rgba(0,0,200, 0.7)';
+                            $('#browsersteps-selector-canvas').css('cursor', 'grab');
+                            ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                            ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                            current_selected_i = i - 1;
+                            break;
+                        }
+                    }
+                }
+            }
+
+        }.debounce(10));
+    });
+
+//    $("#browser-steps-fieldlist").bind('mouseover', function(e) {
+//        console.log(e.xpath_data_index);
+    // });
+
+
+
+    // callback for clicking on an xpath on the canvas
+    function process_selected(xpath_data_index) {
+        found_something = false;
+        var first_available = $("ul#browser_steps li.empty").first();
+
+
+        if (xpath_data_index !== false) {
+            // Nothing focused, so fill in a new one
+            // if inpt type button or <button>
+            // from the top, find the next not used one and use it
+            var x = xpath_data['size_pos'][xpath_data_index];
+            console.log(x);
+            if (x && first_available.length) {
+                // @todo will it let you click shit that has a layer ontop? probably not.
+                if (x['tagtype'] === 'text' || x['tagtype'] === 'email' || x['tagName'] === 'textarea' || x['tagtype'] === 'password' || x['tagtype'] === 'search' ) {
+                    $('select', first_available).val('Enter text in field').change();
+                    $('input[type=text]', first_available).first().val(x['xpath']);
+                    $('input[placeholder="Value"]', first_available).addClass('ok').click().focus();
+                    found_something = true;
+                } else {
+                    if (x['isClickable'] || x['tagName'].startsWith('h')|| x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit'|| x['tagtype'] === 'checkbox'|| x['tagtype'] === 'radio'|| x['tagtype'] === 'li') {
+                        $('select', first_available).val('Click element').change();
+                        $('input[type=text]', first_available).first().val(x['xpath']);
+                        found_something = true;
+                    }
+                }
+
+                first_available.xpath_data_index=xpath_data_index;
+
+                if (!found_something) {
+                    if ( include_text_elements[0].checked === true) {
+                        // Suggest that we use as filter?
+                        // @todo filters should always be in the last steps, nothing non-filter after it
+                        found_something = true;
+                        ctx.strokeStyle = 'rgba(0,0,255, 0.9)';
+                        ctx.fillStyle = 'rgba(0,0,255, 0.1)';
+                        $('select', first_available).val('Extract text and use as filter').change();
+                        $('input[type=text]', first_available).first().val(x['xpath']);
+                        include_text_elements[0].checked = false;
+                    }
+                }
+            }
+        }
+    }
+
+    function draw_circle_on_canvas(x, y) {
+        ctx.beginPath();
+        ctx.arc(x, y, 8, 0, 2 * Math.PI, false);
+        ctx.fillStyle = 'rgba(255,0,0, 0.6)';
+        ctx.fill();
+    }
+
+    function start() {
+        console.log("Starting browser-steps UI");
+        browsersteps_session_id=Date.now();
+        // @todo This setting of the first one should be done at the datalayer but wtforms doesnt wanna play nice
+        $('#browser_steps >li:first-child').removeClass('empty');
+        $('#browser_steps >li:first-child select').val('Goto site').attr('disabled', 'disabled');
+        $('#browser-steps-ui .loader').show();
+        $('.clear,.remove', $('#browser_steps >li:first-child')).hide();
+        $.ajax({
+            type: "GET",
+            url: browser_steps_sync_url+"&browsersteps_session_id="+browsersteps_session_id,
+            statusCode: {
+                400: function () {
+                    // More than likely the CSRF token was lost when the server restarted
+                    alert("There was a problem processing the request, please reload the page.");
+                }
+            }
+        }).done(function (data) {
+            xpath_data = data.xpath_data;
+            $('#browsersteps-img').attr('src', data.screenshot);
+            $("#loading-status-text").fadeIn();
+            // This should trigger 'Goto site'
+            $('#browser_steps >li:first-child .apply').click();
+            browserless_seconds_remaining = data.browser_time_remaining;
+        }).fail(function (data) {
+            console.log(data);
+            alert('There was an error communicating with the server.');
+        });
+
+    }
+
+    function disable_browsersteps_ui() {
+        $("#browser_steps select,input").attr('disabled', 'disabled').css('opacity', '0.5');
+        $("#browser-steps-ui").css('opacity', '0.3');
+        $('#browsersteps-selector-canvas').off("mousemove mousedown click");
+    }
+
+
+    ////////////////////////// STEPS UI ////////////////////
+    $('ul#browser_steps [type="text"]').keydown(function (e) {
+        if (e.keyCode === 13) {
+            // hitting [enter] in a browser-step input should trigger the 'Apply'
+            e.preventDefault();
+            $(".apply", $(this).closest('li')).click();
+            return false;
+        }
+    });
+
+    // Look up which step was selected, and enable or disable the related extra fields
+    // So that people using it dont' get confused
+    $('ul#browser_steps select').on("change", function () {
+        var config = browser_steps_config[$(this).val()].split(' ');
+        var elem_selector = $('tr:nth-child(2) input', $(this).closest('tbody'));
+        var elem_value = $('tr:nth-child(3) input', $(this).closest('tbody'));
+
+        if (config[0] == 0) {
+            $(elem_selector).fadeOut();
+        } else {
+            $(elem_selector).fadeIn();
+        }
+        if (config[1] == 0) {
+            $(elem_value).fadeOut();
+        } else {
+            $(elem_value).fadeIn();
+        }
+
+        if ($(this).val() === 'Click X,Y' && last_click_xy['x'] > 0 && $(elem_value).val().length === 0) {
+            // @todo handle scale
+            $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']);
+        }
+    }).change();
+
+    function set_greyed_state() {
+        $('ul#browser_steps select').not('option:selected[value="Choose one"]').closest('li').removeClass('empty');
+        $('ul#browser_steps select option:selected[value="Choose one"]').closest('li').addClass('empty');
+    }
+
+    // Add the extra buttons to the steps
+    $('ul#browser_steps li').each(function (i) {
+            $(this).append('<div class="control">' +
+                '<a data-step-index=' + i + ' class="pure-button button-secondary button-green button-xsmall apply" >Apply</a>&nbsp;' +
+                '<a data-step-index=' + i + ' class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;' +
+                '<a data-step-index=' + i + ' class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>' +
+                '</div>')
+        }
+    );
+
+    $('ul#browser_steps li .control .clear').click(function (element) {
+        $("select", $(this).closest('li')).val("Choose one").change();
+        $(":text", $(this).closest('li')).val('');
+    });
+
+
+    $('ul#browser_steps li .control .remove').click(function (element) {
+        // so you wanna remove the 2nd (3rd spot 0,1,2,...)
+        var p = $("#browser_steps li").index($(this).closest('li'));
+
+        var elem_to_remove = $("#browser_steps li")[p];
+        $('.clear', elem_to_remove).click();
+        $("#browser_steps li").slice(p, 10).each(function (index) {
+            // get the next one's value from where we clicked
+            var next = $("#browser_steps li")[p + index + 1];
+            if (next) {
+                // and set THIS ones value from the next one
+                var n = $('input', next);
+                $("select", $(this)).val($('select', next).val());
+                $('input', this)[0].value = $(n)[0].value;
+                $('input', this)[1].value = $(n)[1].value;
+                // Triggers reconfiguring the field based on the system config
+                $("select", $(this)).change();
+            }
+
+        });
+
+        // Reset their hidden/empty states
+        set_greyed_state();
+    });
+
+    $('ul#browser_steps li .control .apply').click(function (event) {
+        // sequential requests @todo refactor
+        if(apply_buttons_disabled) {
+            return;
+        }
+
+        var current_data = $(event.currentTarget).closest('li');
+        $('#browser-steps-ui .loader').fadeIn();
+        apply_buttons_disabled=true;
+        $('ul#browser_steps li .control .apply').css('opacity',0.5);
+        $("#browsersteps-img").css('opacity',0.65);
+
+        var is_last_step = 0;
+        var step_n = $(event.currentTarget).data('step-index');
+
+        // On the last step, we should also be getting data ready for the visual selector
+        $('ul#browser_steps li select').each(function (i) {
+            if ($(this).val() !== 'Choose one') {
+                is_last_step += 1;
+            }
+        });
+
+        if (is_last_step == (step_n+1)) {
+            is_last_step = true;
+        } else {
+            is_last_step = false;
+        }
+
+
+        // POST the currently clicked step form widget back and await response, redraw
+        $.ajax({
+            method: "POST",
+            url: browser_steps_sync_url+"&browsersteps_session_id="+browsersteps_session_id,
+            data: {
+                'operation': $("select[id$='operation']", current_data).first().val(),
+                'selector': $("input[id$='selector']", current_data).first().val(),
+                'optional_value': $("input[id$='optional_value']", current_data).first().val(),
+                'step_n': step_n,
+                'is_last_step': is_last_step
+            },
+            statusCode: {
+                400: function () {
+                    // More than likely the CSRF token was lost when the server restarted
+                    alert("There was a problem processing the request, please reload the page.");
+                    $("#loading-status-text").hide();
+                }
+            }
+        }).done(function (data) {
+            // it should return the new state (selectors available and screenshot)
+            xpath_data = data.xpath_data;
+            $('#browsersteps-img').attr('src', data.screenshot);
+            $('#browser-steps-ui .loader').fadeOut();
+            apply_buttons_disabled=false;
+            $("#browsersteps-img").css('opacity',1);
+            $('ul#browser_steps li .control .apply').css('opacity',1);
+            browserless_seconds_remaining = data.browser_time_remaining;
+            $("#loading-status-text").hide();
+        }).fail(function (data) {
+            console.log(data);
+            if (data.responseText.includes("Browser session expired")) {
+                disable_browsersteps_ui();
+            }
+            apply_buttons_disabled=false;
+            $("#loading-status-text").hide();
+            $('ul#browser_steps li .control .apply').css('opacity',1);
+            $("#browsersteps-img").css('opacity',1);
+            //$('#browsersteps-selector-wrapper .loader').fadeOut(2500);
+        });
+
+    });
+
+
+    $("ul#browser_steps select").change(function () {
+        set_greyed_state();
+    }).change();
+
+});

changedetectionio/static/js/diff-render.js

@@ -0,0 +1,112 @@
+var a = document.getElementById('a');
+var b = document.getElementById('b');
+var result = document.getElementById('result');
+
+function changed() {
+    // https://github.com/kpdecker/jsdiff/issues/389
+    // I would love to use `{ignoreWhitespace: true}` here but it breaks the formatting
+    options = {ignoreWhitespace: document.getElementById('ignoreWhitespace').checked};
+
+    var diff = Diff[window.diffType](a.textContent, b.textContent, options);
+    var fragment = document.createDocumentFragment();
+    for (var i = 0; i < diff.length; i++) {
+
+        if (diff[i].added && diff[i + 1] && diff[i + 1].removed) {
+            var swap = diff[i];
+            diff[i] = diff[i + 1];
+            diff[i + 1] = swap;
+        }
+
+        var node;
+        if (diff[i].removed) {
+            node = document.createElement('del');
+            node.classList.add("change");
+            node.appendChild(document.createTextNode(diff[i].value));
+
+        } else if (diff[i].added) {
+            node = document.createElement('ins');
+            node.classList.add("change");
+            node.appendChild(document.createTextNode(diff[i].value));
+        } else {
+            node = document.createTextNode(diff[i].value);
+        }
+        fragment.appendChild(node);
+    }
+
+    result.textContent = '';
+    result.appendChild(fragment);
+
+    // Jump at start
+    inputs.current = 0;
+    next_diff();
+}
+
+window.onload = function () {
+
+
+    /* Convert what is options from UTC time.time() to local browser time */
+    var diffList = document.getElementById("diff-version");
+    if (typeof (diffList) != 'undefined' && diffList != null) {
+        for (var option of diffList.options) {
+            var dateObject = new Date(option.value * 1000);
+            option.label = dateObject.toLocaleString();
+        }
+    }
+
+    /* Set current version date as local time in the browser also */
+    var current_v = document.getElementById("current-v-date");
+    var dateObject = new Date(newest_version_timestamp*1000);
+    current_v.innerHTML = dateObject.toLocaleString();
+    onDiffTypeChange(document.querySelector('#settings [name="diff_type"]:checked'));
+    changed();
+};
+
+a.onpaste = a.onchange =
+    b.onpaste = b.onchange = changed;
+
+if ('oninput' in a) {
+    a.oninput = b.oninput = changed;
+} else {
+    a.onkeyup = b.onkeyup = changed;
+}
+
+function onDiffTypeChange(radio) {
+    window.diffType = radio.value;
+// Not necessary
+//	document.title = "Diff " + radio.value.slice(4);
+}
+
+var radio = document.getElementsByName('diff_type');
+for (var i = 0; i < radio.length; i++) {
+    radio[i].onchange = function (e) {
+        onDiffTypeChange(e.target);
+        changed();
+    }
+}
+
+document.getElementById('ignoreWhitespace').onchange = function (e) {
+    changed();
+}
+
+
+var inputs = document.getElementsByClassName('change');
+inputs.current = 0;
+
+
+function next_diff() {
+
+    var element = inputs[inputs.current];
+    var headerOffset = 80;
+    var elementPosition = element.getBoundingClientRect().top;
+    var offsetPosition = elementPosition - headerOffset + window.scrollY;
+
+    window.scrollTo({
+        top: offsetPosition,
+        behavior: "smooth"
+    });
+
+    inputs.current++;
+    if (inputs.current >= inputs.length) {
+        inputs.current = 0;
+    }
+}

changedetectionio/static/js/stepper.js

@@ -0,0 +1,34 @@
+$(document).ready(function(){
+   checkUserVal();
+   $('#fetch_backend input').on('change', checkUserVal);
+});
+
+var checkUserVal = function(){
+    if($('#fetch_backend input:checked').val()=='html_requests') {
+      $('#request-override').show();
+      $('#webdriver-stepper').hide();
+    } else {
+      $('#request-override').hide();
+      $('#webdriver-stepper').show();
+    }
+};
+
+$('a.row-options').on('click', function(){
+    var row=$(this.closest('tr'));
+    switch($(this).data("action")) {
+      case 'remove':
+        $(row).remove();
+      break;
+      case 'add':
+        var new_row=$(row).clone(true).insertAfter($(row));
+        $('input', new_new).val("");
+      break;
+      case 'add':
+        var new_row=$(row).clone(true).insertAfter($(row));
+        $('input', new_new).val("");
+      break;
+      case 'resend-step':
+
+      break;
+    }
+});

changedetectionio/static/js/tabs.js

@@ -3,7 +3,8 @@
 window.addEventListener('hashchange', function () {
     var tabs = document.getElementsByClassName('active');
     while (tabs[0]) {
-        tabs[0].classList.remove('active')
+        tabs[0].classList.remove('active');
+        document.body.classList.remove('full-width');
     }
     set_active_tab();
 }, false);
@@ -20,6 +21,7 @@ if (!has_errors.length) {
 }
 
 function set_active_tab() {
+    document.body.classList.remove('full-width');
     var tab = document.querySelectorAll("a[href='" + location.hash + "']");
     if (tab.length) {
         tab[0].parentElement.className = "active";
@@ -45,4 +47,3 @@ function focus_error_tab() {
 
 
 
-

changedetectionio/static/js/visual-selector.js

@@ -1,10 +1,10 @@
 // Horrible proof of concept code :)
 // yes - this is really a hack, if you are a front-ender and want to help, please get in touch!
 
-$(document).ready(function() {
+$(document).ready(function () {
 
     var current_selected_i;
-    var state_clicked=false;
+    var state_clicked = false;
 
     var c;
 
@@ -13,9 +13,9 @@ $(document).ready(function() {
     // redline highlight context
     var ctx;
 
-    var current_default_xpath;
-    var x_scale=1;
-    var y_scale=1;
+    var current_default_xpath = [];
+    var x_scale = 1;
+    var y_scale = 1;
     var selector_image;
     var selector_image_rect;
     var selector_data;
@@ -27,204 +27,216 @@ $(document).ready(function() {
         bootstrap_visualselector();
     });
 
-    $(document).on('keydown', function(event) {
+    $(document).on('keydown', function (event) {
         if ($("img#selector-background").is(":visible")) {
             if (event.key == "Escape") {
-                state_clicked=false;
+                state_clicked = false;
                 ctx.clearRect(0, 0, c.width, c.height);
             }
         }
     });
 
     // For when the page loads
-    if(!window.location.hash || window.location.hash != '#visualselector') {
-        $("img#selector-background").attr('src','');
+    if (!window.location.hash || window.location.hash != '#visualselector') {
+        $("img#selector-background").attr('src', '');
         return;
     }
 
     // Handle clearing button/link
-    $('#clear-selector').on('click', function(event) {
-        if(!state_clicked) {
+    $('#clear-selector').on('click', function (event) {
+        if (!state_clicked) {
             alert('Oops, Nothing selected!');
         }
-        state_clicked=false;
+        state_clicked = false;
         ctx.clearRect(0, 0, c.width, c.height);
         xctx.clearRect(0, 0, c.width, c.height);
-        $("#css_filter").val('');
+        $("#include_filters").val('');
     });
 
 
     bootstrap_visualselector();
 
 
-
     function bootstrap_visualselector() {
-        if ( 1 ) {
+        if (1) {
             // bootstrap it, this will trigger everything else
             $("img#selector-background").bind('load', function () {
                 console.log("Loaded background...");
-               c = document.getElementById("selector-canvas");
+                c = document.getElementById("selector-canvas");
                 // greyed out fill context
-               xctx = c.getContext("2d");
+                xctx = c.getContext("2d");
                 // redline highlight context
-               ctx = c.getContext("2d");
-               current_default_xpath =$("#css_filter").val();
-               fetch_data();
-               $('#selector-canvas').off("mousemove mousedown");
-               // screenshot_url defined in the edit.html template
+                ctx = c.getContext("2d");
+                if ($("#include_filters").val().trim().length) {
+                    current_default_xpath = $("#include_filters").val().split(/\r?\n/g);
+                } else {
+                    current_default_xpath = [];
+                }
+                fetch_data();
+                $('#selector-canvas').off("mousemove mousedown");
+                // screenshot_url defined in the edit.html template
             }).attr("src", screenshot_url);
         }
+        // Tell visualSelector that the image should update
+        var s = $("img#selector-background").attr('src')+"?"+ new Date().getTime();
+        $("img#selector-background").attr('src',s)
     }
 
     function fetch_data() {
-      // Image is ready
-      $('.fetching-update-notice').html("Fetching element data..");
+        // Image is ready
+        $('.fetching-update-notice').html("Fetching element data..");
 
-      $.ajax({
-        url: watch_visual_selector_data_url,
-        context: document.body
-      }).done(function (data) {
-        $('.fetching-update-notice').html("Rendering..");
-        selector_data = data;
-        console.log("Reported browser width from backend: "+data['browser_width']);
-        state_clicked=false;
-        set_scale();
-        reflow_selector();
-        $('.fetching-update-notice').fadeOut();
-      });
+        $.ajax({
+            url: watch_visual_selector_data_url,
+            context: document.body
+        }).done(function (data) {
+            $('.fetching-update-notice').html("Rendering..");
+            selector_data = data;
+            console.log("Reported browser width from backend: " + data['browser_width']);
+            state_clicked = false;
+            set_scale();
+            reflow_selector();
+            $('.fetching-update-notice').fadeOut();
+        });
     };
 
 
-
     function set_scale() {
 
-      // some things to check if the scaling doesnt work
-      // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
-      selector_image = $("img#selector-background")[0];
-      selector_image_rect = selector_image.getBoundingClientRect();
+        // some things to check if the scaling doesnt work
+        // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
+        $("#selector-wrapper").show();
+        selector_image = $("img#selector-background")[0];
+        selector_image_rect = selector_image.getBoundingClientRect();
 
-      // make the canvas the same size as the image
-      $('#selector-canvas').attr('height', selector_image_rect.height);
-      $('#selector-canvas').attr('width', selector_image_rect.width);
-      $('#selector-wrapper').attr('width', selector_image_rect.width);
-      x_scale = selector_image_rect.width / selector_data['browser_width'];
-      y_scale = selector_image_rect.height / selector_image.naturalHeight;
-      ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
-      ctx.fillStyle = 'rgba(255,0,0, 0.1)';
-      ctx.lineWidth = 3;
-      console.log("scaling set  x: "+x_scale+" by y:"+y_scale);
-      $("#selector-current-xpath").css('max-width', selector_image_rect.width);
+        // make the canvas the same size as the image
+        $('#selector-canvas').attr('height', selector_image_rect.height);
+        $('#selector-canvas').attr('width', selector_image_rect.width);
+        $('#selector-wrapper').attr('width', selector_image_rect.width);
+        x_scale = selector_image_rect.width / selector_data['browser_width'];
+        y_scale = selector_image_rect.height / selector_image.naturalHeight;
+        ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
+        ctx.fillStyle = 'rgba(255,0,0, 0.1)';
+        ctx.lineWidth = 3;
+        console.log("scaling set  x: " + x_scale + " by y:" + y_scale);
+        $("#selector-current-xpath").css('max-width', selector_image_rect.width);
     }
 
     function reflow_selector() {
-        $(window).resize(function() {
+        $(window).resize(function () {
             set_scale();
             highlight_current_selected_i();
         });
-      var selector_currnt_xpath_text=$("#selector-current-xpath span");
+        var selector_currnt_xpath_text = $("#selector-current-xpath span");
 
-      set_scale();
+        set_scale();
 
-      console.log(selector_data['size_pos'].length + " selectors found");
+        console.log(selector_data['size_pos'].length + " selectors found");
 
-      // highlight the default one if we can find it in the xPath list
-      // or the xpath matches the default one
-      found = false;
-      if(current_default_xpath.length) {
-          for (var i = selector_data['size_pos'].length; i!==0; i--) {
-            var sel = selector_data['size_pos'][i-1];
-            if(selector_data['size_pos'][i - 1].xpath == current_default_xpath) {
-            console.log("highlighting "+current_default_xpath);
-              current_selected_i = i-1;
-              highlight_current_selected_i();
-              found = true;
-              break;
+        // highlight the default one if we can find it in the xPath list
+        // or the xpath matches the default one
+        found = false;
+        if (current_default_xpath.length) {
+            // Find the first one that matches
+            // @todo In the future paint all that match
+            for (const c of current_default_xpath) {
+                for (var i = selector_data['size_pos'].length; i !== 0; i--) {
+                    if (selector_data['size_pos'][i - 1].xpath === c) {
+                        console.log("highlighting " + c);
+                        current_selected_i = i - 1;
+                        highlight_current_selected_i();
+                        found = true;
+                        break;
+                    }
+                }
+                if (found) {
+                    break;
+                }
+            }
+            if (!found) {
+                alert("Unfortunately your existing CSS/xPath Filter was no longer found!");
             }
-          }
-        if(!found) {
-          alert("Unfortunately your existing CSS/xPath Filter was no longer found!");
-        }
-      }
-
-
-      $('#selector-canvas').bind('mousemove', function (e) {
-        if(state_clicked) {
-          return;
-        }
-        ctx.clearRect(0, 0, c.width, c.height);
-        current_selected_i=null;
-
-        // Add in offset
-        if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) {
-          var targetOffset = $(e.target).offset();
-          e.offsetX = e.pageX - targetOffset.left;
-          e.offsetY = e.pageY - targetOffset.top;
         }
 
-        // Reverse order - the most specific one should be deeper/"laster"
-        // Basically, find the most 'deepest'
-        var found=0;
-        ctx.fillStyle = 'rgba(205,0,0,0.35)';
-        for (var i = selector_data['size_pos'].length; i!==0; i--) {
-          // draw all of them? let them choose somehow?
-          var sel = selector_data['size_pos'][i-1];
-          // If we are in a bounding-box
-          if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
-              &&
-              e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
 
-          ) {
+        $('#selector-canvas').bind('mousemove', function (e) {
+            if (state_clicked) {
+                return;
+            }
+            ctx.clearRect(0, 0, c.width, c.height);
+            current_selected_i = null;
 
-            // FOUND ONE
+            // Add in offset
+            if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) {
+                var targetOffset = $(e.target).offset();
+                e.offsetX = e.pageX - targetOffset.left;
+                e.offsetY = e.pageY - targetOffset.top;
+            }
+
+            // Reverse order - the most specific one should be deeper/"laster"
+            // Basically, find the most 'deepest'
+            var found = 0;
+            ctx.fillStyle = 'rgba(205,0,0,0.35)';
+            for (var i = selector_data['size_pos'].length; i !== 0; i--) {
+                // draw all of them? let them choose somehow?
+                var sel = selector_data['size_pos'][i - 1];
+                // If we are in a bounding-box
+                if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
+                    &&
+                    e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
+
+                ) {
+
+                    // FOUND ONE
+                    set_current_selected_text(sel.xpath);
+                    ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                    ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+
+                    // no need to keep digging
+                    // @todo or, O to go out/up, I to go in
+                    // or double click to go up/out the selector?
+                    current_selected_i = i - 1;
+                    found += 1;
+                    break;
+                }
+            }
+
+        }.debounce(5));
+
+        function set_current_selected_text(s) {
+            selector_currnt_xpath_text[0].innerHTML = s;
+        }
+
+        function highlight_current_selected_i() {
+            if (state_clicked) {
+                state_clicked = false;
+                xctx.clearRect(0, 0, c.width, c.height);
+                return;
+            }
+
+            var sel = selector_data['size_pos'][current_selected_i];
+            if (sel[0] == '/') {
+                // @todo - not sure just checking / is right
+                $("#include_filters").val('xpath:' + sel.xpath);
+            } else {
+                $("#include_filters").val(sel.xpath);
+            }
+            xctx.fillStyle = 'rgba(205,205,205,0.95)';
+            xctx.strokeStyle = 'rgba(225,0,0,0.9)';
+            xctx.lineWidth = 3;
+            xctx.fillRect(0, 0, c.width, c.height);
+            // Clear out what only should be seen (make a clear/clean spot)
+            xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+            xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+            state_clicked = true;
             set_current_selected_text(sel.xpath);
-            ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-            ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
 
-            // no need to keep digging
-            // @todo or, O to go out/up, I to go in
-            // or double click to go up/out the selector?
-            current_selected_i=i-1;
-            found+=1;
-            break;
-          }
         }
 
-      }.debounce(5));
 
-      function set_current_selected_text(s) {
-        selector_currnt_xpath_text[0].innerHTML=s;
-      }
-
-      function highlight_current_selected_i() {
-        if(state_clicked) {
-          state_clicked=false;
-          xctx.clearRect(0,0,c.width, c.height);
-          return;
-        }
-
-        var sel = selector_data['size_pos'][current_selected_i];
-        if (sel[0] == '/') {
-        // @todo - not sure just checking / is right
-            $("#css_filter").val('xpath:'+sel.xpath);
-        } else {
-            $("#css_filter").val(sel.xpath);
-        }
-        xctx.fillStyle = 'rgba(205,205,205,0.95)';
-        xctx.strokeStyle = 'rgba(225,0,0,0.9)';
-        xctx.lineWidth = 3;
-        xctx.fillRect(0,0,c.width, c.height);
-        // Clear out what only should be seen (make a clear/clean spot)
-        xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-        xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-        state_clicked=true;
-        set_current_selected_text(sel.xpath);
-
-      }
-
-
-      $('#selector-canvas').bind('mousedown', function (e) {
-        highlight_current_selected_i();
-      });
+        $('#selector-canvas').bind('mousedown', function (e) {
+            highlight_current_selected_i();
+        });
     }
 
-});
+});

changedetectionio/static/styles/parts/browser-steps.scss

@@ -0,0 +1,81 @@
+
+#browser_steps {
+  /* convert rows to horizontal cells */
+  th {
+    display: none;
+  }
+
+  li {
+    list-style: decimal;
+    padding: 5px;
+    .control {
+      padding-left: 5px;
+      padding-right: 5px;
+      a {
+        font-size: 70%;
+      }
+    }
+    &.empty {
+      padding: 0px;
+      opacity: 0.35;
+      .control {
+        display: none;
+      }
+    }
+    &:hover {
+      background: #eee;
+    }
+    > label {
+      display: none;
+    }
+  }
+}
+
+#browser-steps-fieldlist {
+  height: 100%;
+  overflow-y: scroll;
+}
+
+#browser-steps .flex-wrapper {
+  display: flex;
+  flex-flow: row;
+  height: 600px; /*@todo make this dynamic */
+}
+
+/*  this is duplicate :( */
+#browsersteps-selector-wrapper {
+  height: 100%;
+  width: 100%;
+  overflow-y: scroll;
+  position: relative;
+  //width: 100%;
+  > img {
+    position: absolute;
+    max-width: 100%;
+  }
+
+  > canvas {
+    position: relative;
+    max-width: 100%;
+
+    &:hover {
+      cursor: pointer;
+    }
+  }
+
+  .loader {
+    position: absolute;
+    left: 50%;
+    top: 50%;
+    transform: translate(-50%, -50%);
+    margin-left: -40px;
+    z-index: 100;
+  }
+
+  /* nice tall skinny one */
+  .spinner, .spinner:after {
+    width: 80px;
+    height: 80px;
+    font-size: 3px;
+  }
+}

changedetectionio/static/styles/parts/spinners.scss

@@ -0,0 +1,44 @@
+
+/* spinner */
+.spinner,
+.spinner:after {
+  border-radius: 50%;
+  width: 10px;
+  height: 10px;
+}
+.spinner {
+  margin: 0px auto;
+  font-size: 3px;
+  vertical-align: middle;
+  display: inline-block;
+  text-indent: -9999em;
+  border-top: 1.1em solid rgba(38,104,237, 0.2);
+  border-right: 1.1em solid rgba(38,104,237, 0.2);
+  border-bottom: 1.1em solid rgba(38,104,237, 0.2);
+  border-left: 1.1em solid #2668ed;
+  -webkit-transform: translateZ(0);
+  -ms-transform: translateZ(0);
+  transform: translateZ(0);
+  -webkit-animation: load8 1.1s infinite linear;
+  animation: load8 1.1s infinite linear;
+}
+@-webkit-keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg);
+  }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg);
+  }
+}
+@keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg);
+  }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg);
+  }
+}

changedetectionio/static/styles/styles.css

@@ -1,9 +1,107 @@
 /*
  * -- BASE STYLES --
  * Most of these are inherited from Base, but I want to change a few.
- * nvm use v14.18.1 && npm install && npm run build
+nvm use v14.18.1 && npm install && npm run build
  * or npm run watch
  */
+/* spinner */
+.spinner,
+.spinner:after {
+  border-radius: 50%;
+  width: 10px;
+  height: 10px; }
+
+.spinner {
+  margin: 0px auto;
+  font-size: 3px;
+  vertical-align: middle;
+  display: inline-block;
+  text-indent: -9999em;
+  border-top: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-right: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-left: 1.1em solid #2668ed;
+  -webkit-transform: translateZ(0);
+  -ms-transform: translateZ(0);
+  transform: translateZ(0);
+  -webkit-animation: load8 1.1s infinite linear;
+  animation: load8 1.1s infinite linear; }
+
+@-webkit-keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg); }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg); } }
+
+@keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg); }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg); } }
+
+#browser_steps {
+  /* convert rows to horizontal cells */ }
+  #browser_steps th {
+    display: none; }
+  #browser_steps li {
+    list-style: decimal;
+    padding: 5px; }
+    #browser_steps li .control {
+      padding-left: 5px;
+      padding-right: 5px; }
+      #browser_steps li .control a {
+        font-size: 70%; }
+    #browser_steps li.empty {
+      padding: 0px;
+      opacity: 0.35; }
+      #browser_steps li.empty .control {
+        display: none; }
+    #browser_steps li:hover {
+      background: #eee; }
+    #browser_steps li > label {
+      display: none; }
+
+#browser-steps-fieldlist {
+  height: 100%;
+  overflow-y: scroll; }
+
+#browser-steps .flex-wrapper {
+  display: flex;
+  flex-flow: row;
+  height: 600px;
+  /*@todo make this dynamic */ }
+
+/*  this is duplicate :( */
+#browsersteps-selector-wrapper {
+  height: 100%;
+  width: 100%;
+  overflow-y: scroll;
+  position: relative;
+  /* nice tall skinny one */ }
+  #browsersteps-selector-wrapper > img {
+    position: absolute;
+    max-width: 100%; }
+  #browsersteps-selector-wrapper > canvas {
+    position: relative;
+    max-width: 100%; }
+    #browsersteps-selector-wrapper > canvas:hover {
+      cursor: pointer; }
+  #browsersteps-selector-wrapper .loader {
+    position: absolute;
+    left: 50%;
+    top: 50%;
+    transform: translate(-50%, -50%);
+    margin-left: -40px;
+    z-index: 100; }
+  #browsersteps-selector-wrapper .spinner, #browsersteps-selector-wrapper .spinner:after {
+    width: 80px;
+    height: 80px;
+    font-size: 3px; }
+
 .arrow {
   border: solid #1b98f8;
   border-width: 0 2px 2px 0;
@@ -132,7 +230,7 @@ body:after, body:before {
 
 .fetch-error {
   padding-top: 1em;
-  font-size: 60%;
+  font-size: 80%;
   max-width: 400px;
   display: block; }
 
@@ -252,12 +350,12 @@ footer {
 
 #top-right-menu {
   /*
-    position: absolute;
-    right: 0px;
-    background: linear-gradient(to right, #fff0, #fff 10%);
-    padding-left: 20px;
-    padding-right: 10px;
-    */ }
+      position: absolute;
+      right: 0px;
+      background: linear-gradient(to right, #fff0, #fff 10%);
+      padding-left: 20px;
+      padding-right: 10px;
+      */ }
 
 .sticky-tab {
   position: absolute;
@@ -356,10 +454,10 @@ footer {
   input[type='text'] {
     width: 100%; }
   /*
-Max width before this PARTICULAR table gets nasty
-This query will take effect for any screen smaller than 760px
-and also iPads specifically.
-*/
+  Max width before this PARTICULAR table gets nasty
+  This query will take effect for any screen smaller than 760px
+  and also iPads specifically.
+  */
   .watch-table {
     /* Force table to not be like tables anymore */
     /* Force table to not be like tables anymore */
@@ -446,7 +544,7 @@ and also iPads specifically.
   .tab-pane-inner:target {
     display: block; }
 
-#beta-logo {
+.beta-logo {
   height: 50px;
   right: -3px;
   top: -3px;
@@ -455,6 +553,9 @@ and also iPads specifically.
 #selector-header {
   padding-bottom: 1em; }
 
+body.full-width .edit-form {
+  width: 95%; }
+
 .edit-form {
   min-width: 70%;
   /* so it cant overflow */
@@ -481,7 +582,7 @@ ul {
     width: 5em; }
 
 #selector-wrapper {
-  height: 600px;
+  height: 100%;
   overflow-y: scroll;
   position: relative; }
   #selector-wrapper > img {
@@ -507,44 +608,24 @@ ul {
 #api-key-copy {
   color: #0078e7; }
 
-/* spinner */
-.loader,
-.loader:after {
-  border-radius: 50%;
-  width: 10px;
-  height: 10px; }
+.button-green {
+  background-color: #42dd53; }
 
-.loader {
-  margin: 0px auto;
-  font-size: 3px;
-  vertical-align: middle;
-  display: inline-block;
-  text-indent: -9999em;
-  border-top: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-right: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-left: 1.1em solid #2668ed;
-  -webkit-transform: translateZ(0);
-  -ms-transform: translateZ(0);
-  transform: translateZ(0);
-  -webkit-animation: load8 1.1s infinite linear;
-  animation: load8 1.1s infinite linear; }
+.button-red {
+  background-color: #dd4242; }
 
-@-webkit-keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg); }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg); } }
-
-@keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg); }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg); } }
+.noselect {
+  -webkit-touch-callout: none;
+  /* iOS Safari */
+  -webkit-user-select: none;
+  /* Safari */
+  -moz-user-select: none;
+  /* Old versions of Firefox */
+  -ms-user-select: none;
+  /* Internet Explorer/Edge */
+  user-select: none;
+  /* Non-prefixed version, currently
+                                   supported by Chrome, Edge, Opera and Firefox */ }
 
 .snapshot-age {
   padding: 4px;

changedetectionio/static/styles/styles.scss

@@ -1,15 +1,19 @@
 /*
  * -- BASE STYLES --
  * Most of these are inherited from Base, but I want to change a few.
- * nvm use v14.18.1 && npm install && npm run build
+nvm use v14.18.1 && npm install && npm run build
  * or npm run watch
  */
+
+@import "parts/spinners";
+@import "parts/browser-steps";
 @import "parts/_arrows.scss";
 
 body {
   color: #333;
   background: #262626;
 }
+
 .pure-table-even {
   background: #fff;
 }
@@ -156,7 +160,7 @@ body:after, body:before {
 
 .fetch-error {
   padding-top: 1em;
-  font-size: 60%;
+  font-size: 80%;
   max-width: 400px;
   display: block;
 }
@@ -209,46 +213,50 @@ body:after, body:before {
 }
 
 .messages {
-    li {
-        list-style: none;
-        padding: 1em;
-        border-radius: 10px;
-        color: #fff;
-        font-weight: bold;
-        &.message {
-            background: rgba(255, 255, 255, .2);
-        }
-        &.error {
-            background: rgba(255, 1, 1, .5);
-        }
-        &.notice {
-            background: rgba(255, 255, 255, .5);
-        }
+  li {
+    list-style: none;
+    padding: 1em;
+    border-radius: 10px;
+    color: #fff;
+    font-weight: bold;
+
+    &.message {
+      background: rgba(255, 255, 255, .2);
     }
-    &.with-share-link {
-     > *:hover {
-       cursor:pointer;
-     }
+
+    &.error {
+      background: rgba(255, 1, 1, .5);
     }
+
+    &.notice {
+      background: rgba(255, 255, 255, .5);
+    }
+  }
+
+  &.with-share-link {
+    > *:hover {
+      cursor: pointer;
+    }
+  }
 }
 
 #notification-customisation {
-    border: 1px solid #ccc;
-    padding: 0.5rem;
-    border-radius: 5px;
+  border: 1px solid #ccc;
+  padding: 0.5rem;
+  border-radius: 5px;
 }
 
 #notification-error-log {
-    border: 1px solid #ccc;
-    padding: 1rem;
-    border-radius: 5px;
-    overflow-wrap: break-word;
+  border: 1px solid #ccc;
+  padding: 1rem;
+  border-radius: 5px;
+  overflow-wrap: break-word;
 }
 
 #token-table {
-    &.pure-table td, &.pure-table th {
-        font-size: 80%;
-    }
+  &.pure-table td, &.pure-table th {
+    font-size: 80%;
+  }
 }
 
 #new-watch-form {
@@ -256,13 +264,16 @@ body:after, body:before {
   padding: 1em;
   border-radius: 10px;
   margin-bottom: 1em;
+
   input {
     display: inline-block;
     margin-bottom: 5px;
   }
+
   .label {
     display: none;
   }
+
   legend {
     color: #fff;
     font-weight: bold;
@@ -281,8 +292,6 @@ body:after, body:before {
 }
 
 
-
-
 #diff-col {
   padding-left: 40px;
 }
@@ -296,15 +305,16 @@ body:after, body:before {
   border-top-right-radius: 5px;
   border-bottom-right-radius: 5px;
   box-shadow: 5px 0 5px -2px #888;
-     a {
-      color: #1b98f8;
-      cursor: grabbing;
-      -moz-user-select: none;
-      -webkit-user-select: none;
-      -ms-user-select: none;
-      user-select: none;
-      -o-user-select: none;
-    }
+
+  a {
+    color: #1b98f8;
+    cursor: grabbing;
+    -moz-user-select: none;
+    -webkit-user-select: none;
+    -ms-user-select: none;
+    user-select: none;
+    -o-user-select: none;
+  }
 }
 
 footer {
@@ -319,14 +329,14 @@ footer {
 }
 
 #top-right-menu {
-// Just let flex overflow the x axis for now
-/*
-    position: absolute;
-    right: 0px;
-    background: linear-gradient(to right, #fff0, #fff 10%);
-    padding-left: 20px;
-    padding-right: 10px;
-    */
+  // Just let flex overflow the x axis for now
+  /*
+      position: absolute;
+      right: 0px;
+      background: linear-gradient(to right, #fff0, #fff 10%);
+      padding-left: 20px;
+      padding-right: 10px;
+      */
 }
 
 .sticky-tab {
@@ -335,12 +345,15 @@ footer {
   font-size: 65%;
   background: #fff;
   padding: 10px;
+
   &#left-sticky {
     left: 0px;
   }
+
   &#right-sticky {
     right: 0px;
   }
+
   &#hosted-sticky {
     right: 0px;
     top: 100px;
@@ -374,43 +387,49 @@ footer {
 }
 
 .monospaced-textarea {
-    textarea {
-        width: 100%;
-        font-family: monospace;
-        white-space: pre;
-        overflow-wrap: normal;
-        overflow-x: scroll;
-    }
+  textarea {
+    width: 100%;
+    font-family: monospace;
+    white-space: pre;
+    overflow-wrap: normal;
+    overflow-x: scroll;
+  }
 }
 
 
 .pure-form {
-    fieldset {
-        padding-top: 0px;
-        ul {
-            padding-bottom: 0px;
-            margin-bottom: 0px;
-        }
+  fieldset {
+    padding-top: 0px;
+
+    ul {
+      padding-bottom: 0px;
+      margin-bottom: 0px;
     }
-    .pure-control-group, .pure-group, .pure-controls {
-        padding-bottom: 1em;
-        div {
-            margin: 0px;
-        }
-        .checkbox {
-            > * {
-              display: inline;
-              vertical-align: middle;
-            }
-            > label {
-               padding-left: 5px;
-            }
-        }
+  }
+
+  .pure-control-group, .pure-group, .pure-controls {
+    padding-bottom: 1em;
+
+    div {
+      margin: 0px;
     }
+
+    .checkbox {
+      > * {
+        display: inline;
+        vertical-align: middle;
+      }
+
+      > label {
+        padding-left: 5px;
+      }
+    }
+  }
+
   /* The input fields with errors */
   .error {
     input {
-        background-color: #ffebeb;
+      background-color: #ffebeb;
     }
   }
 
@@ -422,9 +441,10 @@ footer {
     vertical-align: middle;
     -webkit-box-sizing: border-box;
     box-sizing: border-box;
+
     li {
-        margin-left: 1em;
-        color: #dd0000;
+      margin-left: 1em;
+      color: #dd0000;
     }
   }
 
@@ -435,19 +455,22 @@ footer {
   textarea {
     width: 100%;
   }
+
   .inline-radio {
-      ul {
-        margin: 0px;
-        list-style: none;
-        li {
-            > * {
-                display: inline-block;
-            }
+    ul {
+      margin: 0px;
+      list-style: none;
+
+      li {
+        > * {
+          display: inline-block;
         }
       }
+    }
   }
 }
 
+
 @media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
   .box {
     max-width: 95%
@@ -462,7 +485,6 @@ footer {
 }
 
 
-
 @media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 800px) {
 
   div.sticky-tab#hosted-sticky {
@@ -486,11 +508,11 @@ footer {
     width: 100%;
   }
 
-/*
-Max width before this PARTICULAR table gets nasty
-This query will take effect for any screen smaller than 760px
-and also iPads specifically.
-*/
+  /*
+  Max width before this PARTICULAR table gets nasty
+  This query will take effect for any screen smaller than 760px
+  and also iPads specifically.
+  */
   .watch-table {
     /* Force table to not be like tables anymore */
     thead, tbody, th, td, tr {
@@ -567,19 +589,19 @@ and also iPads specifically.
 - Rely always on width in CSS
 */
 @media only screen and (min-width: 761px) {
-/* m-d is medium-desktop */
-    .m-d {
-        min-width: 80%;
-    }
+  /* m-d is medium-desktop */
+  .m-d {
+    min-width: 80%;
+  }
 }
 
 
-
 .tabs {
   ul {
     margin: 0px;
     padding: 0px;
-    display:block;
+    display: block;
+
     li {
       margin-right: 3px;
       display: inline-block;
@@ -588,13 +610,15 @@ and also iPads specifically.
       border-top-right-radius: 5px;
       background-color: rgba(255, 255, 255, 0.2);
 
-      &.active,:target {
+      &.active, :target {
         background-color: #fff;
+
         a {
           color: #222;
           font-weight: bold;
         }
       }
+
       a {
         display: block;
         padding: 0.8em;
@@ -606,7 +630,7 @@ and also iPads specifically.
 
 $form-edge-padding: 20px;
 .pure-form-stacked {
-  >div:first-child {
+  > div:first-child {
     display: block;
   }
 }
@@ -620,39 +644,48 @@ $form-edge-padding: 20px;
 }
 
 .tab-pane-inner {
-    &:not(:target) {
-        display: none;
-    }
-    &:target {
-      display: block;
-    }
-    // doesnt need padding because theres another row of buttons/activity
-    padding: 0px;
+  &:not(:target) {
+    display: none;
+  }
+
+  &:target {
+    display: block;
+  }
+
+  // doesnt need padding because theres another row of buttons/activity
+  padding: 0px;
 }
 
-#beta-logo {
-    height: 50px;
-    // looks better when it's hanging off a little
-    right: -3px;
-    top: -3px;
-    position: absolute;
+.beta-logo {
+  height: 50px;
+  // looks better when it's hanging off a little
+  right: -3px;
+  top: -3px;
+  position: absolute;
 }
 
 #selector-header {
-    padding-bottom: 1em;
+  padding-bottom: 1em;
+}
+body.full-width {
+  .edit-form {
+    width: 95%;
+  }
 }
-
 .edit-form {
   min-width: 70%;
   /* so it cant overflow */
   max-width: 95%;
+
   .box-wrap {
     position: relative;
   }
+
   .inner {
     background: #fff;;
     padding: $form-edge-padding;
   }
+
   #actions {
     display: block;
     background: #fff;
@@ -664,38 +697,41 @@ $form-edge-padding: 20px;
 }
 
 ul {
-    padding-left: 1em;
-    padding-top: 0px;
-    margin-top: 4px;
+  padding-left: 1em;
+  padding-top: 0px;
+  margin-top: 4px;
 }
 
 .time-check-widget {
-    tr {
-        display: inline;
-        input[type="number"] {
-            width: 5em;
-        }
+  tr {
+    display: inline;
+
+    input[type="number"] {
+      width: 5em;
     }
+  }
 }
 
 #selector-wrapper {
- height: 600px;
- overflow-y: scroll;
- position: relative;
-    //width: 100%;
- > img {
+  height: 100%;
+  overflow-y: scroll;
+  position: relative;
+  //width: 100%;
+  > img {
     position: absolute;
     z-index: 4;
     max-width: 100%;
- }
- >canvas {
+  }
+
+  > canvas {
     position: relative;
     z-index: 5;
-     max-width: 100%;
-     &:hover {
-     cursor: pointer;
-     }
- }
+    max-width: 100%;
+
+    &:hover {
+      cursor: pointer;
+    }
+  }
 }
 
 #selector-current-xpath {
@@ -703,9 +739,9 @@ ul {
 }
 
 #webdriver-override-options {
-        input[type="number"] {
-            width: 5em;
-        }
+  input[type="number"] {
+    width: 5em;
+  }
 }
 
 #api-key {
@@ -718,48 +754,22 @@ ul {
   color: #0078e7;
 }
 
-/* spinner */
-.loader,
-.loader:after {
-  border-radius: 50%;
-  width: 10px;
-  height: 10px;
+.button-green {
+  background-color: #42dd53;
 }
-.loader {
-  margin: 0px auto;
-  font-size: 3px;
-  vertical-align: middle;
-  display: inline-block;
-  text-indent: -9999em;
-  border-top: 1.1em solid rgba(38,104,237, 0.2);
-  border-right: 1.1em solid rgba(38,104,237, 0.2);
-  border-bottom: 1.1em solid rgba(38,104,237, 0.2);
-  border-left: 1.1em solid #2668ed;
-  -webkit-transform: translateZ(0);
-  -ms-transform: translateZ(0);
-  transform: translateZ(0);
-  -webkit-animation: load8 1.1s infinite linear;
-  animation: load8 1.1s infinite linear;
+
+.button-red {
+  background-color: #dd4242;
 }
-@-webkit-keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg);
-  }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg);
-  }
-}
-@keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg);
-  }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg);
-  }
+
+.noselect {
+  -webkit-touch-callout: none; /* iOS Safari */
+  -webkit-user-select: none; /* Safari */
+  -moz-user-select: none; /* Old versions of Firefox */
+  -ms-user-select: none; /* Internet Explorer/Edge */
+  user-select: none;
+  /* Non-prefixed version, currently
+                                   supported by Chrome, Edge, Opera and Firefox */
 }
 
 .snapshot-age {
@@ -803,4 +813,4 @@ ul {
   padding: 0.5rem;
   border-radius: 5px;
   color: #ff3300;
-}
+}

changedetectionio/store.py

@@ -27,17 +27,18 @@ class ChangeDetectionStore:
     # For when we edit, we should write to disk
     needs_write_urgent = False
 
+    __version_check = True
+
     def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
         # Should only be active for docker
         # logging.basicConfig(filename='/dev/stdout', level=logging.INFO)
-        self.needs_write = False
+        self.__data = App.model()
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
+        self.needs_write = False
         self.proxy_list = None
+        self.start_time = time.time()
         self.stop_thread = False
-
-        self.__data = App.model()
-
         # Base definition for all watchers
         # deepcopy part of #569 - not sure why its needed exactly
         self.generic_definition = deepcopy(Watch.model(datastore_path = datastore_path, default={}))
@@ -81,8 +82,13 @@ class ChangeDetectionStore:
         except (FileNotFoundError, json.decoder.JSONDecodeError):
             if include_default_watches:
                 print("Creating JSON store at", self.datastore_path)
-                self.add_watch(url='https://news.ycombinator.com/', tag='Tech news')
-                self.add_watch(url='https://changedetection.io/CHANGELOG.txt', tag='changedetection.io')
+                self.add_watch(url='https://news.ycombinator.com/',
+                               tag='Tech news',
+                               extras={'fetch_backend': 'html_requests'})
+
+                self.add_watch(url='https://changedetection.io/CHANGELOG.txt',
+                               tag='changedetection.io',
+                               extras={'fetch_backend': 'html_requests'})
 
         self.__data['version_tag'] = version_tag
 
@@ -266,7 +272,7 @@ class ChangeDetectionStore:
             extras = {}
         # should always be str
         if tag is None or not tag:
-            tag=''
+            tag = ''
 
         # Incase these are copied across, assume it's a reference and deepcopy()
         apply_extras = deepcopy(extras)
@@ -281,17 +287,31 @@ class ChangeDetectionStore:
                 res = r.json()
 
                 # List of permissible attributes we accept from the wild internet
-                for k in ['url', 'tag',
-                          'paused', 'title',
-                          'previous_md5', 'headers',
-                          'body', 'method',
-                          'ignore_text', 'css_filter',
-                          'subtractive_selectors', 'trigger_text',
-                          'extract_title_as_title', 'extract_text',
-                          'text_should_not_be_present',
-                          'webdriver_js_execute_code']:
+                for k in [
+                    'body',
+                    'css_filter',
+                    'extract_text',
+                    'extract_title_as_title',
+                    'headers',
+                    'ignore_text',
+                    'include_filters',
+                    'method',
+                    'paused',
+                    'previous_md5',
+                    'subtractive_selectors',
+                    'tag',
+                    'text_should_not_be_present',
+                    'title',
+                    'trigger_text',
+                    'url',
+                    'webdriver_js_execute_code',
+                ]:
                     if res.get(k):
-                        apply_extras[k] = res[k]
+                        if k != 'css_filter':
+                            apply_extras[k] = res[k]
+                        else:
+                            # We renamed the field and made it a list
+                            apply_extras['include_filters'] = [res['css_filter']]
 
             except Exception as e:
                 logging.error("Error fetching metadata for shared watch link", url, str(e))
@@ -314,12 +334,13 @@ class ChangeDetectionStore:
                     del apply_extras[k]
 
             new_watch.update(apply_extras)
-            self.__data['watching'][new_uuid]=new_watch
+            self.__data['watching'][new_uuid] = new_watch
 
         self.__data['watching'][new_uuid].ensure_data_dir_exists()
 
         if write_to_disk_now:
             self.sync_to_json()
+
         return new_uuid
 
     def visualselector_data_is_ready(self, watch_uuid):
@@ -347,6 +368,12 @@ class ChangeDetectionStore:
             f.write(screenshot)
             f.close()
 
+        # Make a JPEG that's used in notifications (due to being a smaller size) available
+        from PIL import Image
+        im1 = Image.open(target_path)
+        im1.convert('RGB').save(target_path.replace('.png','.jpg'), quality=int(os.getenv("NOTIFICATION_SCREENSHOT_JPG_QUALITY", 75)))
+
+
     def save_error_text(self, watch_uuid, contents):
         if not self.data['watching'].get(watch_uuid):
             return
@@ -583,3 +610,14 @@ class ChangeDetectionStore:
         for v in ['User-Agent', 'Accept', 'Accept-Encoding', 'Accept-Language']:
             if self.data['settings']['headers'].get(v):
                 del self.data['settings']['headers'][v]
+
+    # Convert filters to a list of filters css_filter -> include_filters
+    def update_8(self):
+        for uuid, watch in self.data['watching'].items():
+            try:
+                existing_filter = watch.get('css_filter', '')
+                if existing_filter:
+                    watch['include_filters'] = [existing_filter]
+            except:
+                continue
+        return

changedetectionio/templates/diff.html

@@ -21,6 +21,9 @@
 
             <label for="diffChars" class="pure-checkbox">
                 <input type="radio" name="diff_type" id="diffChars" value="diffChars"/> Chars</label>
+            <!-- @todo - when mimetype is JSON, select this by default? -->
+            <label for="diffJson" class="pure-checkbox">
+                <input type="radio" name="diff_type" id="diffJson" value="diffJson" /> JSON</label>
 
             {% if versions|length >= 1 %}
             <label for="diff-version">Compare newest (<span id="current-v-date"></span>) with</label>
@@ -37,6 +40,11 @@
     </form>
     <del>Removed text</del>
     <ins>Inserted Text</ins>
+    <span>
+        <!-- https://github.com/kpdecker/jsdiff/issues/389 ? -->
+        <label for="ignoreWhitespace" class="pure-checkbox" id="label-diff-ignorewhitespace">
+            <input type="checkbox" id="ignoreWhitespace" name="ignoreWhitespace"/> Ignore Whitespace</label>
+    </span>
 </div>
 
 <div id="diff-jump">
@@ -102,122 +110,12 @@
      </div>
 </div>
 
-
-<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff.js')}}"></script>
-
-<script defer="">
-
-var a = document.getElementById('a');
-var b = document.getElementById('b');
-var result = document.getElementById('result');
-
-function changed() {
-	var diff = JsDiff[window.diffType](a.textContent, b.textContent);
-	var fragment = document.createDocumentFragment();
-	for (var i=0; i < diff.length; i++) {
-
-		if (diff[i].added && diff[i + 1] && diff[i + 1].removed) {
-			var swap = diff[i];
-			diff[i] = diff[i + 1];
-			diff[i + 1] = swap;
-		}
-
-		var node;
-		if (diff[i].removed) {
-			node = document.createElement('del');
-			node.classList.add("change");
-			node.appendChild(document.createTextNode(diff[i].value));
-
-		} else if (diff[i].added) {
-			node = document.createElement('ins');
-			node.classList.add("change");
-			node.appendChild(document.createTextNode(diff[i].value));
-		} else {
-			node = document.createTextNode(diff[i].value);
-		}
-		fragment.appendChild(node);
-	}
-
-	result.textContent = '';
-	result.appendChild(fragment);
-
-	// Jump at start
-	inputs.current=0;
-    next_diff();
-}
-
-window.onload = function() {
-
-
-    /* Convert what is options from UTC time.time() to local browser time */
-    var diffList=document.getElementById("diff-version");
-    if (typeof(diffList) != 'undefined' && diffList != null) {
-        for (var option of diffList.options) {
-          var dateObject = new Date(option.value*1000);
-          option.label=dateObject.toLocaleString();
-        }
-    }
-
-    /* Set current version date as local time in the browser also */
-    var current_v = document.getElementById("current-v-date");
-    var dateObject = new Date({{ newest_version_timestamp }}*1000);
-    current_v.innerHTML=dateObject.toLocaleString();
-
-
-	onDiffTypeChange(document.querySelector('#settings [name="diff_type"]:checked'));
-	changed();
-
-};
-
-a.onpaste = a.onchange =
-b.onpaste = b.onchange = changed;
-
-if ('oninput' in a) {
-	a.oninput = b.oninput = changed;
-} else {
-	a.onkeyup = b.onkeyup = changed;
-}
-
-function onDiffTypeChange(radio) {
-	window.diffType = radio.value;
-// Not necessary 
-//	document.title = "Diff " + radio.value.slice(4);
-}
-
-var radio = document.getElementsByName('diff_type');
-for (var i = 0; i < radio.length; i++) {
-	radio[i].onchange = function(e) {
-		onDiffTypeChange(e.target);
-		changed();
-	}
-}
-
-
-var inputs = document.getElementsByClassName('change');
-inputs.current=0;
-
-
-function next_diff() {
-
-    var element = inputs[inputs.current];
-    var headerOffset = 80;
-    var elementPosition = element.getBoundingClientRect().top;
-    var offsetPosition = elementPosition - headerOffset +  window.scrollY;
-
-    window.scrollTo({
-         top: offsetPosition,
-         behavior: "smooth"
-    });
-
-    inputs.current++;
-    if(inputs.current >= inputs.length) {
-      inputs.current=0;
-    }
-}
-
-
-
+<script>
+    const newest_version_timestamp = {{newest_version_timestamp}};
 </script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff.min.js')}}"></script>
+
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff-render.js')}}"></script>
 
 
 {% endblock %}

changedetectionio/templates/edit.html

@@ -13,11 +13,17 @@
     const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
 
+    const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
+    const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
 </script>
+
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='visual-selector.js')}}" defer></script>
-<script type="text/javascript" src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>
+{% if playwright_enabled %}
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
+{% endif %}
 
 <div class="edit-form monospaced-textarea">
 
@@ -25,6 +31,9 @@
         <ul>
             <li class="tab" id=""><a href="#general">General</a></li>
             <li class="tab"><a href="#request">Request</a></li>
+            {% if playwright_enabled %}
+            <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
+            {% endif %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
             <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
             <li class="tab"><a href="#notifications">Notifications</a></li>
@@ -40,7 +49,8 @@
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
-                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span>
+                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span><br/>
+                        <span class="pure-form-message-inline">You can use variables in the URL, perfect for inserting the current date and other logic, <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a></span><br/>
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.title, class="m-d") }}
@@ -134,12 +144,58 @@ User-Agent: wonderbra 1.0") }}
                     </div>
                 </fieldset>
             </div>
+            {% if playwright_enabled %}
+            <div class="tab-pane-inner" id="browser-steps">
+                <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}">
+                <fieldset>
+                    <div class="pure-control-group">
+                        <!--
+                        Too hard right now, better to just send the events to the fetcher for now and leave it in the final screenshot
+                        and/or report an error
+                        <a id="play-steps" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Play steps  ▶</a>
+                        -->
+
+                        <!---  Do this later -->
+                        <div class="checkbox" style="display: none;">
+                            <input type=checkbox id="include_text_elements" > <label for="include_text_elements">Turn on text finder</label>
+                        </div>
+
+                        <div id="loading-status-text" style="display: none;">Please wait, first browser step can take a little time to load..<div class="spinner"></div></div>
+                        <div class="flex-wrapper" >
+
+                            <div id="browser-steps-ui" class="noselect"  style="width: 100%; background-color: #eee; border-radius: 5px;">
+
+                                <div class="noselect"  id="browsersteps-selector-wrapper" style="width: 100%">
+                                    <span class="loader">
+                                        <div class="spinner"></div>
+                                    </span>
+                                    <img  class="noselect" id="browsersteps-img" src="" style="max-width: 100%; width: 100%;" />
+                                    <canvas  class="noselect" id="browsersteps-selector-canvas" style="max-width: 100%; width: 100%;"></canvas>
+                                </div>
+                            </div>
+                            <div id="browser-steps-fieldlist" style="padding-left: 1em;  width: 350px; font-size: 80%;" >
+                                <span id="browserless-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target=_new href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
+                                {{ render_field(form.browser_steps) }}
+                            </div>
+                        </div>
+                    </div>
+                </fieldset>
+            </div>
+            {% endif %}
 
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div  class="pure-control-group inline-radio">
                       {{ render_checkbox_field(form.notification_muted) }}
                     </div>
+                    {% if is_html_webdriver %}
+                    <div class="pure-control-group inline-radio">
+                      {{ render_checkbox_field(form.notification_screenshot) }}
+                        <span class="pure-form-message-inline">
+                            <strong>Use with caution!</strong> This will easily fill up your email storage quota or flood other storages.
+                        </span>
+                    </div>
+                    {% endif %}
                     <div class="field-group" id="notification-field-group">
                         {% if has_default_notification_urls %}
                         <div class="inline-warning">
@@ -173,21 +229,27 @@ User-Agent: wonderbra 1.0") }}
                         </div>
                     </fieldset>
                     <div class="pure-control-group">
-                        {% set field = render_field(form.css_filter,
-                            placeholder=".class-name or #some-id, or other CSS selector rule.",
+                        {% set field = render_field(form.include_filters,
+                            rows=5,
+                            placeholder="#example
+xpath://body/div/span[contains(@class, 'example-class')]",
                             class="m-d")
                         %}
                         {{ field }}
                         {% if '/text()' in  field %}
                           <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br/>
                         {% endif %}
-                        <span class="pure-form-message-inline">
+                        <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br/>
                     <ul>
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
-                        <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a>.
+                        <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
                             <ul>
                                 <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
+                                {% if jq_support %}
                                 <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
+                                {% else %}
+                                <li>jq support not installed</li>
+                                {% endif %}
                             </ul>
                         </li>
                         <li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
@@ -198,7 +260,7 @@ User-Agent: wonderbra 1.0") }}
                             </ul>
                             </li>
                     </ul>
-                    Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath, or jq selector rules before filing an issue on GitHub! <a
+                    Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
                                 href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
                 </span>
                     </div>
@@ -280,39 +342,32 @@ Unavailable") }}
             </div>
 
             <div class="tab-pane-inner visual-selector-ui" id="visualselector">
-                <img id="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}">
-                <strong>Pro-tip:</strong> This tool is only for limiting which elements will be included on a change-detection, not for interacting with browser directly.
+                <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}">
+
                 <fieldset>
                     <div class="pure-control-group">
                         {% if visualselector_enabled %}
-                            {% if visualselector_data_is_ready %}
-                                <div id="selector-header">
-                                    <a id="clear-selector" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Clear selection</a>
-                                    <i class="fetching-update-notice" style="font-size: 80%;">One moment, fetching screenshot and element information..</i>
-                                </div>
-                                <div id="selector-wrapper">
-                                    <!-- request the screenshot and get the element offset info ready -->
-                                    <!-- use img src ready load to know everything is ready to map out -->
-                                    <!-- @todo: maybe something interesting like a field to select 'elements that contain text... and their parents n' -->
-                                    <img id="selector-background" />
-                                    <canvas id="selector-canvas"></canvas>
-
-                                </div>
-                                <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
-
                             <span class="pure-form-message-inline">
-                                <p><span style="font-weight: bold">Beta!</span> The Visual Selector is new and there may be minor bugs, please report pages that dont work, help us to improve this software!</p>
+                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection &dash; after the <i>Browser Steps</i> has completed.<br/><br/>
                             </span>
 
-                            {% else %}
-                                <span class="pure-form-message-inline">Screenshot and element data is not available or not yet ready.</span>
-                            {% endif %}
+                            <div id="selector-header">
+                                <a id="clear-selector" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Clear selection</a>
+                                <i class="fetching-update-notice" style="font-size: 80%;">One moment, fetching screenshot and element information..</i>
+                            </div>
+                            <div id="selector-wrapper" style="display: none">
+                                <!-- request the screenshot and get the element offset info ready -->
+                                <!-- use img src ready load to know everything is ready to map out -->
+                                <!-- @todo: maybe something interesting like a field to select 'elements that contain text... and their parents n' -->
+                                <img id="selector-background" />
+                                <canvas id="selector-canvas"></canvas>
+                            </div>
+                            <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
                         {% else %}
                             <span class="pure-form-message-inline">
                                 <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
                                 <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
                                 <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
-
                             </span>
                         {% endif %}
                     </div>

changedetectionio/templates/watch-overview.html

@@ -87,7 +87,7 @@
                     <a class="state-{{'on' if watch.notification_muted}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications"/></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
-                    <a class="external" target="_blank" rel="noopener" href="{{ watch.url.replace('source:','') }}"></a>
+                    <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
                     <a href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
 
                     {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
@@ -96,7 +96,7 @@
                     <div class="fetch-error">{{ watch.last_error }}</div>
                     {% endif %}
                     {% if watch.last_notification_error is defined and watch.last_notification_error != False %}
-                    <div class="fetch-error notification-error">{{ watch.last_notification_error }}</div>
+                    <div class="fetch-error notification-error"><a href="{{url_for('notification_logs')}}">{{ watch.last_notification_error }}</a></div>
                     {% endif %}
                     {% if not active_tag %}
                     <span class="watch-tag-list">{{ watch.tag}}</span>

changedetectionio/tests/conftest.py

@@ -41,7 +41,7 @@ def app(request):
 
     cleanup(datastore_path)
 
-    app_config = {'datastore_path': datastore_path}
+    app_config = {'datastore_path': datastore_path, 'disable_checkver' : True}
     cleanup(app_config['datastore_path'])
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
     app = changedetection_app(app_config, datastore)

changedetectionio/tests/fetchers/test_content.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks
 import logging
 
-
+# Requires playwright to be installed
 def test_fetch_webdriver_content(client, live_server):
     live_server_setup(live_server)
 

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -24,7 +24,7 @@ def test_preferred_proxy(client, live_server):
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
-                "css_filter": "",
+                "include_filters": "",
                 "fetch_backend": "html_requests",
                 "headers": "",
                 "proxy": "proxy-two",

changedetectionio/tests/test_api.py

@@ -147,6 +147,16 @@ def test_api_simple(client, live_server):
     # @todo how to handle None/default global values?
     assert watch['history_n'] == 2, "Found replacement history section, which is in its own API"
 
+    # basic systeminfo check
+    res = client.get(
+        url_for("systeminfo"),
+        headers={'x-api-key': api_key},
+    )
+    info = json.loads(res.data)
+    assert info.get('watch_count') == 1
+    assert info.get('uptime') > 0.5
+
+
     # Finally delete the watch
     res = client.delete(
         url_for("watch", uuid=watch_uuid),

changedetectionio/tests/test_auth.py

@@ -19,17 +19,16 @@ def test_basic_auth(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
+    time.sleep(1)
 
     # Check form validation
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(1)
     res = client.get(
         url_for("preview_page", uuid="first"),

changedetectionio/tests/test_backend.py

@@ -3,7 +3,7 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 
 sleep_time_for_fetch_thread = 3
 
@@ -36,7 +36,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
         client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
-        time.sleep(sleep_time_for_fetch_thread)
+        wait_for_all_checks(client)
 
         # It should report nothing found (no new 'unviewed' class)
         res = client.get(url_for("index"))
@@ -69,7 +69,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches are queued for rechecking.' in res.data
 
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # Now something should be ready, indicated by having a 'unviewed' class
     res = client.get(url_for("index"))
@@ -98,14 +98,14 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'which has this one new line' in res.data
     assert b'Which is across multiple lines' not in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(2):
         client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
-        time.sleep(sleep_time_for_fetch_thread)
+        wait_for_all_checks(client)
 
         # It should report nothing found (no new 'unviewed' class)
         res = client.get(url_for("index"))
@@ -125,7 +125,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     )
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_backup.py

@@ -1,18 +1,31 @@
 #!/usr/bin/python3
 
-import time
+from .util import set_original_response, set_modified_response, live_server_setup
 from flask import url_for
 from urllib.request import urlopen
-from . util import set_original_response, set_modified_response, live_server_setup
+from zipfile import ZipFile
+import re
+import time
 
 
 def test_backup(client, live_server):
-
     live_server_setup(live_server)
 
+    set_original_response()
+
     # Give the endpoint time to spin up
     time.sleep(1)
 
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url_for('test_endpoint', _external=True)},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+    time.sleep(3)
+
     res = client.get(
         url_for("get_backup"),
         follow_redirects=True
@@ -20,6 +33,19 @@ def test_backup(client, live_server):
 
     # Should get the right zip content type
     assert res.content_type == "application/zip"
+
     # Should be PK/ZIP stream
     assert res.data.count(b'PK') >= 2
 
+    # ZipFile from buffer seems non-obvious, just save it instead
+    with open("download.zip", 'wb') as f:
+        f.write(res.data)
+
+    zip = ZipFile('download.zip')
+    l = zip.namelist()
+    uuid4hex = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.*txt', re.I)
+    newlist = list(filter(uuid4hex.match, l))  # Read Note below
+
+    # Should be two txt files in the archive (history and the snapshot)
+    assert len(newlist) == 2
+

changedetectionio/tests/test_css_selector.py

@@ -46,22 +46,23 @@ def set_modified_response():
 
 
 # Test that the CSS extraction works how we expect, important here is the right placing of new lines \n's
-def test_css_filter_output():
-    from changedetectionio import fetch_site_status
+def test_include_filters_output():
     from inscriptis import get_text
 
     # Check text with sub-parts renders correctly
     content = """<html> <body><div id="thingthing" >  Some really <b>bold</b> text  </div> </body> </html>"""
-    html_blob = css_filter(css_filter="#thingthing", html_content=content)
+    html_blob = include_filters(include_filters="#thingthing", html_content=content)
     text = get_text(html_blob)
     assert text == "  Some really bold text"
 
     content = """<html> <body>
     <p>foo bar blah</p>
-    <div class="parts">Block A</div> <div class="parts">Block B</div></body> 
+    <DIV class="parts">Block A</DiV> <div class="parts">Block B</DIV></body> 
     </html>
 """
-    html_blob = css_filter(css_filter=".parts", html_content=content)
+
+    # in xPath this would be //*[@class='parts']
+    html_blob = include_filters(include_filters=".parts", html_content=content)
     text = get_text(html_blob)
 
     # Divs are converted to 4 whitespaces by inscriptis
@@ -69,10 +70,10 @@ def test_css_filter_output():
 
 
 # Tests the whole stack works with the CSS Filter
-def test_check_markup_css_filter_restriction(client, live_server):
+def test_check_markup_include_filters_restriction(client, live_server):
     sleep_time_for_fetch_thread = 3
 
-    css_filter = "#sametext"
+    include_filters = "#sametext"
 
     set_original_response()
 
@@ -88,9 +89,6 @@ def test_check_markup_css_filter_restriction(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -98,19 +96,16 @@ def test_check_markup_css_filter_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-
+    time.sleep(1)
     # Check it saved
     res = client.get(
         url_for("edit_page", uuid="first"),
     )
-    assert bytes(css_filter.encode('utf-8')) in res.data
-
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    assert bytes(include_filters.encode('utf-8')) in res.data
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -126,3 +121,58 @@ def test_check_markup_css_filter_restriction(client, live_server):
     # Because it should be looking at only that 'sametext' id
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
+
+
+# Tests the whole stack works with the CSS Filter
+def test_check_multiple_filters(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    include_filters = "#blob-a\r\nxpath://*[contains(@id,'blob-b')]"
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("""<html><body>
+     <div id="blob-a">Blob A</div>
+     <div id="blob-b">Blob B</div>
+     <div id="blob-c">Blob C</div>
+     </body>
+     </html>
+    """)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    time.sleep(1)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"include_filters": include_filters,
+              "url": test_url,
+              "tag": "",
+              "headers": "",
+              'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Only the two blobs should be here
+    assert b"Blob A" in res.data # CSS was ok
+    assert b"Blob B" in res.data # xPath was ok
+    assert b"Blob C" not in res.data # Should not be included

changedetectionio/tests/test_encoding.py

@@ -70,9 +70,6 @@ def test_check_encoding_detection_missing_content_type_header(client, live_serve
         follow_redirects=True
     )
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(2)
 

changedetectionio/tests/test_extract_regex.py

@@ -88,7 +88,7 @@ def test_check_filter_multiline(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": '',
+        data={"include_filters": '',
               'extract_text': '/something.+?6 billion.+?lines/si',
               "url": test_url,
               "tag": "",
@@ -116,7 +116,7 @@ def test_check_filter_multiline(client, live_server):
 
 def test_check_filter_and_regex_extract(client, live_server):
     sleep_time_for_fetch_thread = 3
-    css_filter = ".changetext"
+    include_filters = ".changetext"
 
     set_original_response()
 
@@ -143,7 +143,7 @@ def test_check_filter_and_regex_extract(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": css_filter,
+        data={"include_filters": include_filters,
               'extract_text': '\d+ online\r\n\d+ guests\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i',
               "url": test_url,
               "tag": "",

changedetectionio/tests/test_filter_exist_changes.py

@@ -92,7 +92,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
         "tag": "my tag",
         "title": "my title",
         "headers": "",
-        "css_filter": '.ticket-available',
+        "include_filters": '.ticket-available',
         "fetch_backend": "html_requests"})
 
     res = client.post(

changedetectionio/tests/test_filter_failure_notification.py

@@ -76,7 +76,7 @@ def run_filter_test(client, content_filter):
         "title": "my title",
         "headers": "",
         "filter_failure_notification_send": 'y',
-        "css_filter": content_filter,
+        "include_filters": content_filter,
         "fetch_backend": "html_requests"})
 
     res = client.post(
@@ -95,7 +95,7 @@ def run_filter_test(client, content_filter):
         time.sleep(3)
 
     # We should see something in the frontend
-    assert b'Warning, filter' in res.data
+    assert b'Warning, no filters were found' in res.data
 
     # Now it should exist and contain our "filter not found" alert
     assert os.path.isfile("test-datastore/notification.txt")
@@ -131,7 +131,7 @@ def run_filter_test(client, content_filter):
 def test_setup(live_server):
     live_server_setup(live_server)
 
-def test_check_css_filter_failure_notification(client, live_server):
+def test_check_include_filters_failure_notification(client, live_server):
     set_original_response()
     time.sleep(1)
     run_filter_test(client, '#nope-doesnt-exist')

changedetectionio/tests/test_jinja2.py

@@ -0,0 +1,33 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from .util import live_server_setup
+
+
+# If there was only a change in the whitespacing, then we shouldnt have a change detected
+def test_jinja2_in_url_query(client, live_server):
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_return_query', _external=True)
+
+    # because url_for() will URL-encode the var, but we dont here
+    full_url = "{}?{}".format(test_url,
+                              "date={% now 'Europe/Berlin', '%Y' %}.{% now 'Europe/Berlin', '%m' %}.{% now 'Europe/Berlin', '%d' %}", )
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": full_url, "tag": "test"},
+        follow_redirects=True
+    )
+    assert b"Watch added" in res.data
+    time.sleep(3)
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b'date=2' in res.data

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -5,7 +5,12 @@ import time
 from flask import url_for, escape
 from . util import live_server_setup
 import pytest
+jq_support = True
 
+try:
+    import jq
+except ModuleNotFoundError:
+    jq_support = False
 
 def test_setup(live_server):
     live_server_setup(live_server)
@@ -40,22 +45,24 @@ and it can also be repeated
     assert text == "23.5"
 
     # also check for jq
-    text = html_tools.extract_json_as_string(content, "jq:.offers.price")
-    assert text == "23.5"
+    if jq_support:
+        text = html_tools.extract_json_as_string(content, "jq:.offers.price")
+        assert text == "23.5"
+
+        text = html_tools.extract_json_as_string('{"id":5}', "jq:.id")
+        assert text == "5"
 
     text = html_tools.extract_json_as_string('{"id":5}', "json:$.id")
     assert text == "5"
 
-    text = html_tools.extract_json_as_string('{"id":5}', "jq:.id")
-    assert text == "5"
-
     # When nothing at all is found, it should throw JSONNOTFound
     # Which is caught and shown to the user in the watch-overview table
     with pytest.raises(html_tools.JSONNotFound) as e_info:
         html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "json:$.id")
 
-    with pytest.raises(html_tools.JSONNotFound) as e_info:
-        html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jq:.id")
+    if jq_support:
+        with pytest.raises(html_tools.JSONNotFound) as e_info:
+            html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jq:.id")
 
 def set_original_ext_response():
     data = """
@@ -125,7 +132,7 @@ def set_original_response():
     return None
 
 
-def set_response_with_html():
+def set_json_response_with_html():
     test_return_data = """
     {
       "test": [
@@ -169,7 +176,7 @@ def set_modified_response():
 def test_check_json_without_filter(client, live_server):
     # Request a JSON document from a application/json source containing HTML
     # and be sure it doesn't get chewed up by instriptis
-    set_response_with_html()
+    set_json_response_with_html()
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -182,9 +189,6 @@ def test_check_json_without_filter(client, live_server):
         follow_redirects=True
     )
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(3)
 
@@ -193,6 +197,7 @@ def test_check_json_without_filter(client, live_server):
         follow_redirects=True
     )
 
+    # Should still see '"html": "<b>"'
     assert b'&#34;&lt;b&gt;' in res.data
     assert res.data.count(b'{\n') >= 2
 
@@ -214,9 +219,6 @@ def check_json_filter(json_filter, client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(3)
 
@@ -224,7 +226,7 @@ def check_json_filter(json_filter, client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": json_filter,
+        data={"include_filters": json_filter,
               "url": test_url,
               "tag": "",
               "headers": "",
@@ -240,9 +242,6 @@ def check_json_filter(json_filter, client, live_server):
     )
     assert bytes(escape(json_filter).encode('utf-8')) in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(3)
     #  Make a change
@@ -271,7 +270,8 @@ def test_check_jsonpath_filter(client, live_server):
     check_json_filter('json:boss.name', client, live_server)
 
 def test_check_jq_filter(client, live_server):
-    check_json_filter('jq:.boss.name', client, live_server)
+    if jq_support:
+        check_json_filter('jq:.boss.name', client, live_server)
 
 def check_json_filter_bool_val(json_filter, client, live_server):
     set_original_response()
@@ -293,7 +293,7 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": json_filter,
+        data={"include_filters": json_filter,
               "url": test_url,
               "tag": "",
               "headers": "",
@@ -303,11 +303,6 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     )
     assert b"Updated watch." in res.data
 
-    time.sleep(3)
-
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(3)
     #  Make a change
@@ -329,7 +324,8 @@ def test_check_jsonpath_filter_bool_val(client, live_server):
     check_json_filter_bool_val("json:$['available']", client, live_server)
 
 def test_check_jq_filter_bool_val(client, live_server):
-    check_json_filter_bool_val("jq:.available", client, live_server)
+    if jq_support:
+        check_json_filter_bool_val("jq:.available", client, live_server)
 
 # Re #265 - Extended JSON selector test
 # Stuff to consider here
@@ -351,9 +347,6 @@ def check_json_ext_filter(json_filter, client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(3)
 
@@ -361,7 +354,7 @@ def check_json_ext_filter(json_filter, client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": json_filter,
+        data={"include_filters": json_filter,
               "url": test_url,
               "tag": "",
               "headers": "",
@@ -377,9 +370,6 @@ def check_json_ext_filter(json_filter, client, live_server):
     )
     assert bytes(escape(json_filter).encode('utf-8')) in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(3)
     #  Make a change
@@ -408,4 +398,5 @@ def test_check_jsonpath_ext_filter(client, live_server):
     check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server)
 
 def test_check_jq_ext_filter(client, live_server):
-    check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
+    if jq_support:
+        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)

changedetectionio/tests/test_notification.py

@@ -1,9 +1,12 @@
+import json
 import os
 import time
 import re
 from flask import url_for
 from . util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup
+from . util import  extract_UUID_from_client
 import logging
+import base64
 
 from changedetectionio.notification import (
     default_notification_body,
@@ -18,7 +21,6 @@ def test_setup(live_server):
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_check_notification(client, live_server):
-
     set_original_response()
 
     # Give the endpoint time to spin up
@@ -68,6 +70,20 @@ def test_check_notification(client, live_server):
     # Give the thread time to pick up the first version
     time.sleep(3)
 
+    # We write the PNG to disk, but a JPEG should appear in the notification
+    # Write the last screenshot png
+    testimage_png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='
+    # This one is created when we save the screenshot from the webdriver/playwright session (converted from PNG)
+    testimage_jpg = '/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/wAALCAABAAEBAREA/8QAFAABAAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q=='
+
+
+    uuid = extract_UUID_from_client(client)
+    datastore = 'test-datastore'
+    with open(os.path.join(datastore, str(uuid), 'last-screenshot.png'), 'wb') as f:
+        f.write(base64.b64decode(testimage_png))
+    with open(os.path.join(datastore, str(uuid), 'last-screenshot.jpg'), 'wb') as f:
+        f.write(base64.b64decode(testimage_jpg))
+
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
 
@@ -86,6 +102,7 @@ def test_check_notification(client, live_server):
                                                    "Diff: {diff}\n"
                                                    "Diff Full: {diff_full}\n"
                                                    ":-)",
+                              "notification_screenshot": True,
                               "notification_format": "Text"}
 
     notification_form_data.update({
@@ -116,8 +133,6 @@ def test_check_notification(client, live_server):
     time.sleep(3)
     set_modified_response()
 
-    notification_submission = None
-
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(3)
@@ -143,6 +158,19 @@ def test_check_notification(client, live_server):
     assert ":-)" in notification_submission
     assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
 
+    # Check the attachment was added, and that it is a JPEG from the original PNG
+    notification_submission_object = json.loads(notification_submission)
+    assert notification_submission_object['attachments'][0]['filename'] == 'last-screenshot.jpg'
+    assert len(notification_submission_object['attachments'][0]['base64'])
+    assert notification_submission_object['attachments'][0]['mimetype'] == 'image/jpeg'
+    jpeg_in_attachment = base64.b64decode(notification_submission_object['attachments'][0]['base64'])
+    assert b'JFIF' in jpeg_in_attachment
+    assert testimage_png not in notification_submission
+    # Assert that the JPEG is readable (didn't get chewed up somewhere)
+    from PIL import Image
+    import io
+    assert Image.open(io.BytesIO(jpeg_in_attachment))
+
     if env_base_url:
         # Re #65 - did we see our BASE_URl ?
         logging.debug (">>> BASE_URL checking in notification: %s", env_base_url)

changedetectionio/tests/test_share_watch.py

@@ -14,7 +14,7 @@ def test_share_watch(client, live_server):
     live_server_setup(live_server)
 
     test_url = url_for('test_endpoint', _external=True)
-    css_filter = ".nice-filter"
+    include_filters = ".nice-filter"
 
     # Add our URL to the import page
     res = client.post(
@@ -29,7 +29,7 @@ def test_share_watch(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -37,7 +37,7 @@ def test_share_watch(client, live_server):
     res = client.get(
         url_for("edit_page", uuid="first"),
     )
-    assert bytes(css_filter.encode('utf-8')) in res.data
+    assert bytes(include_filters.encode('utf-8')) in res.data
 
     # click share the link
     res = client.get(
@@ -73,4 +73,8 @@ def test_share_watch(client, live_server):
     res = client.get(
         url_for("edit_page", uuid="first"),
     )
-    assert bytes(css_filter.encode('utf-8')) in res.data
+    assert bytes(include_filters.encode('utf-8')) in res.data
+
+    # Check it saved the URL
+    res = client.get(url_for("index"))
+    assert bytes(test_url.encode('utf-8')) in res.data

changedetectionio/tests/test_source.py

@@ -57,10 +57,9 @@ def test_check_basic_change_detection_functionality_source(client, live_server):
 
 
 
-
+# `subtractive_selectors` should still work in `source:` type requests
 def test_check_ignore_elements(client, live_server):
     set_original_response()
-
     time.sleep(2)
     test_url = 'source:'+url_for('test_endpoint', _external=True)
     # Add our URL to the import page
@@ -77,9 +76,9 @@ def test_check_ignore_elements(client, live_server):
     #####################
     # We want <span> and <p> ONLY, but ignore span with .foobar-detection
 
-    res = client.post(
+    client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": 'span,p', "url": test_url, "tag": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
+        data={"include_filters": 'span,p', "url": test_url, "tag": "", "subtractive_selectors": ".foobar-detection", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
 
@@ -89,7 +88,6 @@ def test_check_ignore_elements(client, live_server):
         url_for("preview_page", uuid="first"),
         follow_redirects=True
     )
-
     assert b'foobar-detection' not in res.data
     assert b'&lt;br' not in res.data
     assert b'&lt;p' in res.data

changedetectionio/tests/test_trigger_regex_with_filter.py

@@ -49,7 +49,7 @@ def test_trigger_regex_functionality_with_filter(client, live_server):
         url_for("edit_page", uuid="first"),
         data={"trigger_text": "/cool.stuff/",
               "url": test_url,
-              "css_filter": '#in-here',
+              "include_filters": '#in-here',
               "fetch_backend": "html_requests"},
         follow_redirects=True
     )

changedetectionio/tests/test_watch_fields_storage.py

@@ -22,7 +22,7 @@ def test_check_watch_field_storage(client, live_server):
         url_for("edit_page", uuid="first"),
         data={ "notification_urls": "json://127.0.0.1:30000\r\njson://128.0.0.1\r\n",
                "time_between_check-minutes": 126,
-               "css_filter" : ".fooclass",
+               "include_filters" : ".fooclass",
                "title" : "My title",
                "ignore_text" : "ignore this",
                "url": test_url,

changedetectionio/tests/test_xpath_selector.py

@@ -89,7 +89,7 @@ def test_check_xpath_filter_utf8(client, live_server):
     time.sleep(1)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -143,7 +143,7 @@ def test_check_xpath_text_function_utf8(client, live_server):
     time.sleep(1)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -182,9 +182,6 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -192,7 +189,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -230,10 +227,11 @@ def test_xpath_validation(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
+    time.sleep(2)
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"is not a valid XPath expression" in res.data
@@ -242,7 +240,7 @@ def test_xpath_validation(client, live_server):
 
 
 # actually only really used by the distll.io importer, but could be handy too
-def test_check_with_prefix_css_filter(client, live_server):
+def test_check_with_prefix_include_filters(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
@@ -263,7 +261,7 @@ def test_check_with_prefix_css_filter(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
 

changedetectionio/tests/util.py

@@ -86,6 +86,7 @@ def extract_UUID_from_client(client):
 def wait_for_all_checks(client):
     # Loop waiting until done..
     attempt=0
+    time.sleep(0.1)
     while attempt < 60:
         time.sleep(1)
         res = client.get(url_for("index"))
@@ -159,5 +160,10 @@ def live_server_setup(live_server):
         ret = " ".join([auth.username, auth.password, auth.type])
         return ret
 
+    # Just return some GET var
+    @live_server.app.route('/test-return-query', methods=['GET'])
+    def test_return_query():
+        return request.query_string
+
     live_server.start()
 

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -10,12 +10,13 @@ def test_visual_selector_content_ready(client, live_server):
     import json
 
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
-    live_server_setup(live_server)
     time.sleep(1)
+    live_server_setup(live_server)
+
+
+    # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url
+    test_url = "https://changedetection.io/ci-test/test-runjs.html"
 
-    # Add our URL to the import page, maybe better to use something we control?
-    # We use an external URL because the docker container is too difficult to setup to connect back to the pytest socket
-    test_url = 'https://news.ycombinator.com'
     res = client.post(
         url_for("form_quick_watch_add"),
         data={"url": test_url, "tag": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
@@ -25,13 +26,26 @@ def test_visual_selector_content_ready(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first", unpause_on_save=1),
-        data={"css_filter": ".does-not-exist", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_webdriver"},
+        data={
+              "url": test_url,
+              "tag": "",
+              "headers": "",
+              'fetch_backend': "html_webdriver",
+              'webdriver_js_execute_code': 'document.querySelector("button[name=test-button]").click();'
+        },
         follow_redirects=True
     )
     assert b"unpaused" in res.data
-    time.sleep(1)
     wait_for_all_checks(client)
     uuid = extract_UUID_from_client(client)
+
+    # Check the JS execute code before extract worked
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b'I smell JavaScript' in res.data
+
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'last-screenshot.png')), "last-screenshot.png should exist"
     assert os.path.isfile(os.path.join('test-datastore', uuid, 'elements.json')), "xpath elements.json data should exist"
 

changedetectionio/update_worker.py

@@ -4,7 +4,7 @@ import queue
 import time
 
 from changedetectionio import content_fetcher
-from changedetectionio.html_tools import FilterNotFoundInResponse
+from changedetectionio.fetch_site_status import FilterNotFoundInResponse
 
 # A single update worker
 #
@@ -74,6 +74,7 @@ class update_worker(threading.Thread):
             n_object.update({
                 'watch_url': watch['url'],
                 'uuid': watch_uuid,
+                'screenshot': watch.get_screenshot_as_jpeg() if watch.get('notification_screenshot') else None,
                 'current_snapshot': snapshot_contents.decode('utf-8'),
                 'diff': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], line_feed_sep=line_feed_sep),
                 'diff_full': diff.render_diff(watch_history[dates[-2]], watch_history[dates[-1]], True, line_feed_sep=line_feed_sep)
@@ -91,8 +92,8 @@ class update_worker(threading.Thread):
             return
 
         n_object = {'notification_title': 'Changedetection.io - Alert - CSS/xPath filter was not present in the page',
-                    'notification_body': "Your configured CSS/xPath filter of '{}' for {{watch_url}} did not appear on the page after {} attempts, did the page change layout?\n\nLink: {{base_url}}/edit/{{watch_uuid}}\n\nThanks - Your omniscient changedetection.io installation :)\n".format(
-                        watch['css_filter'],
+                    'notification_body': "Your configured CSS/xPath filters of '{}' for {{watch_url}} did not appear on the page after {} attempts, did the page change layout?\n\nLink: {{base_url}}/edit/{{watch_uuid}}\n\nThanks - Your omniscient changedetection.io installation :)\n".format(
+                        ", ".join(watch['include_filters']),
                         threshold),
                     'notification_format': 'text'}
 
@@ -102,6 +103,34 @@ class update_worker(threading.Thread):
         elif len(self.datastore.data['settings']['application']['notification_urls']):
             n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
 
+        # Only prepare to notify if the rules above matched
+        if 'notification_urls' in n_object:
+            n_object.update({
+                'watch_url': watch['url'],
+                'uuid': watch_uuid,
+                'screenshot': None
+            })
+            self.notification_q.put(n_object)
+            print("Sent filter not found notification for {}".format(watch_uuid))
+
+    def send_step_failure_notification(self, watch_uuid, step_n):
+        watch = self.datastore.data['watching'].get(watch_uuid, False)
+        if not watch:
+            return
+        threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
+        n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1),
+                    'notification_body': "Your configured browser step at position {} for {{watch['url']}} "
+                                         "did not appear on the page after {} attempts, did the page change layout? "
+                                         "Does it need a delay added?\n\nLink: {{base_url}}/edit/{{watch_uuid}}\n\n"
+                                         "Thanks - Your omniscient changedetection.io installation :)\n".format(step_n+1, threshold),
+                    'notification_format': 'text'}
+
+        if len(watch['notification_urls']):
+            n_object['notification_urls'] = watch['notification_urls']
+
+        elif len(self.datastore.data['settings']['application']['notification_urls']):
+            n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
+
         # Only prepare to notify if the rules above matched
         if 'notification_urls' in n_object:
             n_object.update({
@@ -109,7 +138,8 @@ class update_worker(threading.Thread):
                 'uuid': watch_uuid
             })
             self.notification_q.put(n_object)
-            print("Sent filter not found notification for {}".format(watch_uuid))
+            print("Sent step not found notification for {}".format(watch_uuid))
+
 
     def cleanup_error_artifacts(self, uuid):
         # All went fine, remove error artifacts
@@ -189,7 +219,7 @@ class update_worker(threading.Thread):
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
-                        err_text = "Warning, filter '{}' not found".format(str(e))
+                        err_text = "Warning, no filters were found, no change detection ran."
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
                                                                            # So that we get a trigger when the content is added again
                                                                            'previous_md5': ''})
@@ -211,6 +241,32 @@ class update_worker(threading.Thread):
 
                         process_changedetection_results = True
 
+                    except content_fetcher.BrowserStepsStepTimout as e:
+
+                        if not self.datastore.data['watching'].get(uuid):
+                            continue
+
+                        err_text = "Warning, browser step at position {} could not run, target not found, check the watch, add a delay if necessary.".format(e.step_n+1)
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                                                                           # So that we get a trigger when the content is added again
+                                                                           'previous_md5': ''})
+
+
+                        if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
+                            c = self.datastore.data['watching'][uuid].get('consecutive_filter_failures', 5)
+                            c += 1
+                            # Send notification if we reached the threshold?
+                            threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
+                                                                                           0)
+                            print("Step for {} not found, consecutive_filter_failures: {}".format(uuid, c))
+                            if threshold > 0 and c >= threshold:
+                                if not self.datastore.data['watching'][uuid].get('notification_muted'):
+                                    self.send_step_failure_notification(watch_uuid=uuid, step_n=e.step_n)
+                                c = 0
+
+                            self.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
+                        process_changedetection_results = False
+
                     except content_fetcher.EmptyReply as e:
                         # Some kind of custom to-str handler in the exception handler that does this?
                         err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)
@@ -282,16 +338,19 @@ class update_worker(threading.Thread):
                             self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
                             self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
 
+                    if self.datastore.data['watching'].get(uuid):
+                        # Always record that we atleast tried
+                        count = self.datastore.data['watching'][uuid].get('check_count', 0) + 1
+                        self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
+                                                                           'last_checked': round(time.time()),
+                                                                           'check_count': count
+                                                                           })
 
-                    # Always record that we atleast tried
-                    self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
-                                                                       'last_checked': round(time.time())})
-
-                    # Always save the screenshot if it's available
-                    if update_handler.screenshot:
-                        self.datastore.save_screenshot(watch_uuid=uuid, screenshot=update_handler.screenshot)
-                    if update_handler.xpath_data:
-                        self.datastore.save_xpath_data(watch_uuid=uuid, data=update_handler.xpath_data)
+                        # Always save the screenshot if it's available
+                        if update_handler.screenshot:
+                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=update_handler.screenshot)
+                        if update_handler.xpath_data:
+                            self.datastore.save_xpath_data(watch_uuid=uuid, data=update_handler.xpath_data)
 
 
                 self.current_uuid = None  # Done

docker-compose.yml

@@ -45,6 +45,9 @@ services:
   #        Respect proxy_pass type settings, `proxy_set_header Host "localhost";` and `proxy_set_header X-Forwarded-Prefix /app;`
   #        More here https://github.com/dgtlmoon/changedetection.io/wiki/Running-changedetection.io-behind-a-reverse-proxy-sub-directory
   #      - USE_X_SETTINGS=1
+  #
+  #        Hides the `Referer` header so that monitored websites can't see the changedetection.io hostname.
+  #      - HIDE_REFERER=true
 
       # Comment out ports: when using behind a reverse proxy , enable networks: etc.
       ports:

requirements.txt

@@ -1,35 +1,37 @@
-flask~= 2.0
+flask~=2.0
 flask_wtf
+flask-compress
 eventlet>=0.31.0
 validators
-timeago ~=1.0
-inscriptis ~= 2.2
-feedgen ~= 0.9
-flask-login ~= 0.5
+timeago~=1.0
+inscriptis~=2.2
+feedgen~=0.9
+flask-login~=0.5
 flask_restful
 pytz
 
 # Set these versions together to avoid a RequestsDependencyWarning
 # >= 2.26 also adds Brotli support if brotli is installed
-brotli ~= 1.0
-requests[socks] ~= 2.28
+brotli~=1.0
+requests[socks] ~=2.28
 
-urllib3 > 1.26
-chardet > 2.3.0
+urllib3>1.26
+chardet>2.3.0
 
-wtforms ~= 3.0
-jsonpath-ng ~= 1.5.3
-jq ~= 1.3.0
+wtforms~=3.0
+jsonpath-ng~=1.5.3
+
+# jq not available on Windows so must be installed manually
 
 # Notification library
-apprise ~= 1.1.0
+apprise~=1.2.0
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
 
 # Pinned version of cryptography otherwise
 # ERROR: Could not build wheels for cryptography which use PEP 517 and cannot be installed directly
-cryptography ~= 3.4
+cryptography~=3.4
 
 # Used for CSS filtering
 bs4
@@ -38,11 +40,24 @@ bs4
 lxml
 
 # 3.141 was missing socksVersion, 3.150 was not in pypi, so we try 4.1.0
-selenium ~= 4.1.0
+selenium~=4.1.0
 
 # https://stackoverflow.com/questions/71652965/importerror-cannot-import-name-safe-str-cmp-from-werkzeug-security/71653849#71653849
 # ImportError: cannot import name 'safe_str_cmp' from 'werkzeug.security'
 # need to revisit flask login versions
-werkzeug ~= 2.0.0
+werkzeug~=2.0.0
 
+# Templating, so far just in the URLs but in the future can be for the notifications also
+jinja2~=3.1
+jinja2-time
+
+# https://peps.python.org/pep-0508/#environment-markers
+# https://github.com/dgtlmoon/changedetection.io/pull/1009
+jq~=1.3 ;python_version >= "3.8" and sys_platform == "linux"
+
+# Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future
+pillow
 # playwright is installed at Dockerfile build time because it's not available on all platforms
+
+# For shutting down playwright BrowserSteps nicely
+psutil