memory GC workaround

.github/test/Dockerfile-alpine

@@ -1,31 +0,0 @@
-# Taken from https://github.com/linuxserver/docker-changedetection.io/blob/main/Dockerfile
-# Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)
-# Some packages wont install via pypi because they dont have a wheel available under this architecture.
-
-FROM ghcr.io/linuxserver/baseimage-alpine:3.16
-ENV PYTHONUNBUFFERED=1
-
-COPY requirements.txt /requirements.txt
-
-RUN \
-  apk add --update --no-cache --virtual=build-dependencies \
-    cargo \
-    g++ \
-    gcc \
-    libc-dev \
-    libffi-dev \
-    libxslt-dev \
-    make \
-    openssl-dev \
-    py3-wheel \
-    python3-dev \
-    zlib-dev && \
-  apk add --update --no-cache \
-    libxslt \
-    python3 \
-    py3-pip && \
-  echo "**** pip3 install test of changedetection.io ****" && \
-  pip3 install -U pip wheel setuptools && \
-  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.16/ -r /requirements.txt && \
-  apk del --purge \
-    build-dependencies

.github/workflows/test-container-build.yml

@@ -1,66 +0,0 @@
-name: ChangeDetection.io Container Build Test
-
-# Triggers the workflow on push or pull request events
-
-# This line doesnt work, even tho it is the documented one
-#on: [push, pull_request]
-
-on:
-  push:
-    paths:
-      - requirements.txt
-      - Dockerfile
-
-  pull_request:
-    paths:
-      - requirements.txt
-      - Dockerfile
-
-  # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
-  # @todo: some kind of path filter for requirements.txt and Dockerfile
-jobs:
-  test-container-build:
-    runs-on: ubuntu-latest
-    steps:
-        - uses: actions/checkout@v2
-        - name: Set up Python 3.9
-          uses: actions/setup-python@v2
-          with:
-            python-version: 3.9
-
-        # Just test that the build works, some libraries won't compile on ARM/rPi etc
-        - name: Set up QEMU
-          uses: docker/setup-qemu-action@v1
-          with:
-            image: tonistiigi/binfmt:latest
-            platforms: all
-
-        - name: Set up Docker Buildx
-          id: buildx
-          uses: docker/setup-buildx-action@v1
-          with:
-            install: true
-            version: latest
-            driver-opts: image=moby/buildkit:master
-
-        # https://github.com/dgtlmoon/changedetection.io/pull/1067
-        # Check we can still build under alpine/musl
-        - name: Test that the docker containers can build (musl via alpine check)
-          id: docker_build_musl
-          uses: docker/build-push-action@v2
-          with:
-            context: ./
-            file: ./.github/test/Dockerfile-alpine
-            platforms: linux/amd64,linux/arm64
-
-        - name: Test that the docker containers can build
-          id: docker_build
-          uses: docker/build-push-action@v2
-          # https://github.com/docker/build-push-action#customizing
-          with:
-            context: ./
-            file: ./Dockerfile
-            platforms: linux/arm/v7,linux/arm/v6,linux/amd64,linux/arm64,
-            cache-from: type=local,src=/tmp/.buildx-cache
-            cache-to: type=local,dest=/tmp/.buildx-cache
-

.github/workflows/test-only.yml

@@ -1,25 +1,28 @@
-name: ChangeDetection.io App Test
+name: ChangeDetection.io Test
 
 # Triggers the workflow on push or pull request events
 on: [push, pull_request]
 
 jobs:
-  test-application:
+  test-build:
     runs-on: ubuntu-latest
     steps:
+
       - uses: actions/checkout@v2
       - name: Set up Python 3.9
         uses: actions/setup-python@v2
         with:
           python-version: 3.9
 
+      - name: Show env vars
+        run: set
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
           if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or undefined names
@@ -36,4 +39,7 @@ jobs:
           # Each test is totally isolated and performs its own cleanup/reset
           cd changedetectionio; ./run_all_tests.sh
 
+      # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
+      # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
 
+      # https://github.com/docker/buildx/issues/495#issuecomment-918925854

.gitignore

@@ -8,7 +8,6 @@ __pycache__
 build
 dist
 venv
-test-datastore/*
 test-datastore
 *.egg-info*
 .vscode/settings.json

CONTRIBUTING.md

@@ -6,7 +6,7 @@ Otherwise, it's always best to PR into the `dev` branch.
 
 Please be sure that all new functionality has a matching test!
 
-Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notification.py` for example
+Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notifications.py` for example
 
 ```
 pip3 install -r requirements-dev

Dockerfile

@@ -5,15 +5,13 @@ FROM python:3.8-slim as builder
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    g++ \
+    libssl-dev \
+    libffi-dev \
     gcc \
     libc-dev \
-    libffi-dev \
-    libjpeg-dev \
-    libssl-dev \
     libxslt-dev \
-    make \
+    zlib1g-dev \
-    zlib1g-dev
+    g++
 
 RUN mkdir /install
 WORKDIR /install
@@ -24,8 +22,7 @@ RUN pip install --target=/dependencies -r /requirements.txt
 
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
-# https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
+RUN pip install --target=/dependencies playwright~=1.24 \
-RUN pip install --target=/dependencies playwright~=1.27.1 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
@@ -37,14 +34,13 @@ ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
 # Re #93, #73, excluding rustc (adds another 430Mb~)
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    g++ \
+    libssl-dev \
+    libffi-dev \
     gcc \
     libc-dev \
-    libffi-dev \
-    libjpeg-dev \
-    libssl-dev \
     libxslt-dev \
-    zlib1g-dev
+    zlib1g-dev \
+    g++
 
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1
@@ -62,7 +58,6 @@ EXPOSE 5000
 
 # The actual flask app
 COPY changedetectionio /app/changedetectionio
-
 # The eventlet server wrapper
 COPY changedetection.py /app/changedetection.py
 

MANIFEST.in

@@ -1,14 +1,8 @@
 recursive-include changedetectionio/api *
-recursive-include changedetectionio/blueprint *
-recursive-include changedetectionio/model *
-recursive-include changedetectionio/res *
-recursive-include changedetectionio/static *
 recursive-include changedetectionio/templates *
-recursive-include changedetectionio/tests *
+recursive-include changedetectionio/static *
-prune changedetectionio/static/package-lock.json
+recursive-include changedetectionio/model *
-prune changedetectionio/static/styles/node_modules
-prune changedetectionio/static/styles/package-lock.json
 include changedetection.py
 global-exclude *.pyc
 global-exclude node_modules
-global-exclude venv
+global-exclude venv

README-pip.md

@@ -1,48 +1,45 @@
-## Web Site Change Detection, Monitoring and Notification.
+#  changedetection.io
+![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
+<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
+  <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
+</a>
+<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
+  <img src="https://img.shields.io/github/v/release/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
+</a>
 
-Live your data-life pro-actively, track website content changes and receive notifications via Discord, Email, Slack, Telegram and 70+ more
+## Self-hosted open source change monitoring of web pages.
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start?src=pip)
+_Know when web pages change! Stay ontop of new information!_ 
+
+Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
 
 
-[**Don't have time? Let us host it for you! try our extremely affordable subscription use our proxies and support!**](https://lemonade.changedetection.io/start) 
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
+
+
+**Get your own private instance now! Let us host it for you!**
+
+[**Try our $6.99/month subscription - unlimited checks, watches and notifications!**](https://lemonade.changedetection.io/start), choose from different geographical locations, let us handle everything for you. 
+
 
 
 #### Example use cases
 
-- Products and services have a change in pricing
+Know when ...
-- _Out of stock notification_ and _Back In stock notification_
+
-- Governmental department updates (changes are often only on their websites)
+- Government department updates (changes are often only on their websites)
+- Local government news (changes are often only on their websites)
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
 - Realestate listing changes
-- Know when your favourite whiskey is on sale, or other special deals are announced before anyone else
 - COVID related news from government websites
-- University/organisation news from their website
 - Detect and monitor changes in JSON API responses 
-- JSON API monitoring and alerting
+- API monitoring and alerting
-- Changes in legal and other documents
-- Trigger API calls via notifications when text appears on a website
-- Glue together APIs using the JSON filter and JSON notifications
-- Create RSS feeds based on changes in web content
-- Monitor HTML source code for unexpected changes, strengthen your PCI compliance
-- You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
-
-_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
-
-#### Key Features
-
-- Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
-- Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
-- Switch between fast non-JS and Chrome JS based "fetchers"
-- Easily specify how often a site should be checked
-- Execute JS before extracting text (Good for logging in, see examples in the UI!)
-- Override Request Headers, Specify `POST` or `GET` and other methods
-- Use the "Visual Selector" to help target specific elements
 
+**Get monitoring now!**
 
 ```bash
-$ pip3 install changedetection.io
+$ pip3 install changedetection.io   
 ```
 
 Specify a target for the *datastore path* with `-d` (required) and a *listening port* with `-p` (defaults to `5000`)
@@ -54,5 +51,17 @@ $ changedetection.io -d /path/to/empty/data/dir -p 5000
 
 Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
 
+### Features
+- Website monitoring
+- Change detection of content and analyses
+- Filters on change (Select by CSS or JSON)
+- Triggers (Wait for text, wait for regex)
+- Notification support
+- JSON API Monitoring
+- Parse JSON embedded in HTML
+- (Reverse) Proxy support
+- Javascript support via WebDriver
+- RaspberriPi (arm v6/v7/64 support)
+
 See https://github.com/dgtlmoon/changedetection.io for more information.
 

README.md

@@ -1,45 +1,25 @@
 ## Web Site Change Detection, Monitoring and Notification.
 
-_Live your data-life pro-actively, Detect website changes and perform meaningful actions, trigger notifications via Discord, Email, Slack, Telegram, API calls and many more._
+Live your data-life pro-actively, track website content changes and receive notifications via Discord, Email, Slack, Telegram and 70+ more
 
-
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start)
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start?src=github)
 
 [![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)
 
 ![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 
+Know when important content changes, we support notifications via Discord, Telegram, Home-Assistant, Slack, Email and 70+ more
+
 [**Don't have time? Let us host it for you! try our $6.99/month subscription - use our proxies and support!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
 
-- Chrome browser included.
-- Super fast, no registration needed setup.
-- Get started watching and receiving website change notifications straight away.
 
 
-### Target specific parts of the webpage using the Visual Selector tool.
+- Automatic Updates, Automatic Backups, No Heroku "paused application", don't miss a change!
-
+- Javascript browser included
-Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
+- Unlimited checks and watches!
-
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://lemonade.changedetection.io/start?src=github)
-
-### Easily see what changed, examine by word, line, or individual character.
-
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://lemonade.changedetection.io/start?src=github)
 
 
-### Perform interactive browser steps
+#### Example use cases
-
-Fill in text boxes, click buttons and more, setup your changedetection scenario. 
-
-Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
-
-[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Website change detection with interactive browser steps, login, cookies etc" />](https://lemonade.changedetection.io/start?src=github)
-
-After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
-Requires Playwright to be enabled.
-
-
-### Example use cases
 
 - Products and services have a change in pricing
 - _Out of stock notification_ and _Back In stock notification_
@@ -58,27 +38,36 @@ Requires Playwright to be enabled.
 - Create RSS feeds based on changes in web content
 - Monitor HTML source code for unexpected changes, strengthen your PCI compliance
 - You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
-- Get notified when certain keywords appear in Twitter search results
-- Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 
 #### Key Features
 
 - Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
-- Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
+- Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JsonPath rules
 - Switch between fast non-JS and Chrome JS based "fetchers"
 - Easily specify how often a site should be checked
 - Execute JS before extracting text (Good for logging in, see examples in the UI!)
 - Override Request Headers, Specify `POST` or `GET` and other methods
 - Use the "Visual Selector" to help target specific elements
-- Configurable [proxy per watch](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration)
-- Send a screenshot with the notification when a change is detected in the web page
 
-We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
+
+## Screenshots
+
+### Examine differences in content.
+
+Easily see what changed, examine by word, line, or individual character.
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
+### Filter by elements using the Visual Selector tool.
+
+Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
+
 ## Installation
 
 ### Docker
@@ -118,8 +107,8 @@ _Now with per-site configurable support for using a fast built in HTTP fetcher o
 ### Docker
 ```
 docker pull dgtlmoon/changedetection.io
-docker kill $(docker ps -a -f name=changedetection.io -q)
+docker kill $(docker ps -a|grep changedetection.io|awk '{print $1}')
-docker rm $(docker ps -a -f name=changedetection.io -q)
+docker rm $(docker ps -a|grep changedetection.io|awk '{print $1}')
 docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```
 
@@ -133,8 +122,8 @@ See the wiki for more information https://github.com/dgtlmoon/changedetection.io
 
 
 ## Filters
+XPath, JSONPath and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools.
 
-XPath, JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. 
 (We support LXML `re:test`, `re:math` and `re:replace`.)
 
 ## Notifications
@@ -159,11 +148,11 @@ Just some examples
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
 
-Now you can also customise your notification content and use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2 templating</a> for their title and body!
+Now you can also customise your notification content!
 
 ## JSON API Monitoring
 
-Detect changes and monitor data in JSON API's by using either JSONPath or jq to filter, parse, and restructure JSON as needed.
+Detect changes and monitor data in JSON API's by using the built-in JSONPath selectors as a filter / selector.
 
 ![image](https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/json-filter-field-example.png)
 
@@ -171,17 +160,9 @@ This will re-parse the JSON and apply formatting to the text, making it super ea
 
 ![image](https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/json-diff-example.png)
 
-### JSONPath or jq?
-
-For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more specifc information on jq.
-
-One big advantage of `jq` is that you can use logic in your JSON filter, such as filters to only show items that have a value greater than/less than etc.
-
-See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/JSON-Selector-Filter-help for more information and examples
-
 ### Parse JSON embedded in HTML!
 
-When you enable a `json:` or `jq:` filter, you can even automatically extract and parse embedded JSON inside a HTML page! Amazingly handy for sites that build content based on JSON, such as many e-commerce websites. 
+When you enable a `json:` filter, you can even automatically extract and parse embedded JSON inside a HTML page! Amazingly handy for sites that build content based on JSON, such as many e-commerce websites. 
 
 ```
 <html>
@@ -191,11 +172,11 @@ When you enable a `json:` or `jq:` filter, you can even automatically extract an
 </script>
 ```  
 
-`json:$.price` or `jq:.price` would give `23.50`, or you can extract the whole structure
+`json:$.price` would give `23.50`, or you can extract the whole structure
 
-## Proxy Configuration
+## Proxy configuration
 
-See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [BrightData proxy services where possible]( https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support)
+See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration
 
 ## Raspberry Pi support?
 

changedetectionio/__init__.py

@@ -1,20 +1,29 @@
 #!/usr/bin/python3
 
+
+# @todo logging
+# @todo extra options for url like , verify=False etc.
+# @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option?
+# @todo option for interval day/6 hour/etc
+# @todo on change detected, config for calling some API
+# @todo fetch title into json
+# https://distill.io/features
+# proxy per check
+#  - flask_cors, itsdangerous,MarkupSafe
+
 import datetime
-import flask_login
-import logging
 import os
-import pytz
 import queue
 import threading
 import time
-import timeago
-
 from copy import deepcopy
-from distutils.util import strtobool
-from feedgen.feed import FeedGenerator
 from threading import Event
 
+import flask_login
+import logging
+import pytz
+import timeago
+from feedgen.feed import FeedGenerator
 from flask import (
     Flask,
     abort,
@@ -27,15 +36,15 @@ from flask import (
     session,
     url_for,
 )
-from flask_compress import Compress as FlaskCompress
 from flask_login import login_required
 from flask_restful import abort, Api
+
 from flask_wtf import CSRFProtect
 
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.40.0.1'
+__version__ = '0.39.18'
 
 datastore = None
 
@@ -46,16 +55,13 @@ ticker_thread = None
 extra_stylesheets = []
 
 update_q = queue.PriorityQueue()
+
 notification_q = queue.Queue()
 
 app = Flask(__name__,
             static_url_path="",
             static_folder="static",
             template_folder="templates")
-from flask_compress import Compress
-
-# Super handy for compressing large BrowserSteps responses and others
-FlaskCompress(app)
 
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
@@ -95,12 +101,6 @@ def init_app_secret(datastore_path):
 
     return secret
 
-
-@app.template_global()
-def get_darkmode_state():
-    css_dark_mode = request.cookies.get('css_dark_mode', 'false')
-    return 'true' if css_dark_mode and strtobool(css_dark_mode) else 'false'
-
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
 # running or something similar.
 @app.template_filter('format_last_checked_time')
@@ -108,7 +108,7 @@ def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
     # Worker thread tells us which UUID it is currently processing.
     for t in running_update_threads:
         if t.current_uuid == watch_obj['uuid']:
-            return '<span class="spinner"></span><span> Checking now</span>'
+            return '<span class="loader"></span><span> Checking now</span>'
 
     if watch_obj['last_checked'] == 0:
         return 'Not yet'
@@ -205,8 +205,9 @@ def changedetection_app(config=None, datastore_o=None):
     watch_api.add_resource(api_v1.Watch, '/api/v1/watch/<string:uuid>',
                            resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
 
-    watch_api.add_resource(api_v1.SystemInfo, '/api/v1/systeminfo',
+
-                           resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+
+
 
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
@@ -502,7 +503,7 @@ def changedetection_app(config=None, datastore_o=None):
         from changedetectionio import fetch_site_status
 
         # Get the most recent one
-        newest_history_key = datastore.data['watching'][uuid].get('newest_history_key')
+        newest_history_key = datastore.get_val(uuid, 'newest_history_key')
 
         # 0 means that theres only one, so that there should be no 'unviewed' history available
         if newest_history_key == 0:
@@ -534,7 +535,6 @@ def changedetection_app(config=None, datastore_o=None):
 
     def edit_page(uuid):
         from changedetectionio import forms
-        from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
 
         using_default_check_time = True
         # More for testing, possible to return the first/only
@@ -552,13 +552,16 @@ def changedetection_app(config=None, datastore_o=None):
         # be sure we update with a copy instead of accidently editing the live object by reference
         default = deepcopy(datastore.data['watching'][uuid])
 
+        # Show system wide default if nothing configured
+        if datastore.data['watching'][uuid]['fetch_backend'] is None:
+            default['fetch_backend'] = datastore.data['settings']['application']['fetch_backend']
+
         # Show system wide default if nothing configured
         if all(value == 0 or value == None for value in datastore.data['watching'][uuid]['time_between_check'].values()):
             default['time_between_check'] = deepcopy(datastore.data['settings']['requests']['time_between_check'])
 
         # Defaults for proxy choice
         if datastore.proxy_list is not None:  # When enabled
-            # @todo
             # Radio needs '' not None, or incase that the chosen one no longer exists
             if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list):
                 default['proxy'] = ''
@@ -568,16 +571,11 @@ def changedetection_app(config=None, datastore_o=None):
                                data=default,
                                )
 
-        # form.browser_steps[0] can be assumed that we 'goto url' first
-
         if datastore.proxy_list is None:
             # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
             del form.proxy
         else:
-            form.proxy.choices = [('', 'Default')]
+            form.proxy.choices = [('', 'Default')] + datastore.proxy_list
-            for p in datastore.proxy_list:
-                form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))
-
 
         if request.method == 'POST' and form.validate():
             extra_update_obj = {}
@@ -600,8 +598,10 @@ def changedetection_app(config=None, datastore_o=None):
             if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']:
                 extra_update_obj['fetch_backend'] = None
 
+            # Notification URLs
+            datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data
 
-             # Ignore text
+            # Ignore text
             form_ignore_text = form.ignore_text.data
             datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
 
@@ -611,7 +611,7 @@ def changedetection_app(config=None, datastore_o=None):
                     extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
             # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
-            if form.include_filters.data != datastore.data['watching'][uuid].get('include_filters', []):
+            if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']:
                 if len(datastore.data['watching'][uuid].history):
                     extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
@@ -649,33 +649,18 @@ def changedetection_app(config=None, datastore_o=None):
             # Only works reliably with Playwright
             visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and default['fetch_backend'] == 'html_webdriver'
 
-            # JQ is difficult to install on windows and must be manually added (outside requirements.txt)
-            jq_support = True
-            try:
-                import jq
-            except ModuleNotFoundError:
-                jq_support = False
-
-            watch = datastore.data['watching'].get(uuid)
-            system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
-            is_html_webdriver = True if watch.get('fetch_backend') == 'html_webdriver' or (
-                    watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
 
             output = render_template("edit.html",
-                                     browser_steps_config=browser_step_ui_config,
+                                     uuid=uuid,
+                                     watch=datastore.data['watching'][uuid],
+                                     form=form,
+                                     has_empty_checktime=using_default_check_time,
+                                     using_global_webdriver_wait=default['webdriver_delay'] is None,
                                      current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
-                                     form=form,
+                                     visualselector_data_is_ready=visualselector_data_is_ready,
-                                     has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
-                                     has_empty_checktime=using_default_check_time,
-                                     is_html_webdriver=is_html_webdriver,
-                                     jq_support=jq_support,
-                                     playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False),
-                                     settings_application=datastore.data['settings']['application'],
-                                     using_global_webdriver_wait=default['webdriver_delay'] is None,
-                                     uuid=uuid,
                                      visualselector_enabled=visualselector_enabled,
-                                     watch=watch
+                                     playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False)
                                      )
 
         return output
@@ -687,34 +672,26 @@ def changedetection_app(config=None, datastore_o=None):
 
         default = deepcopy(datastore.data['settings'])
         if datastore.proxy_list is not None:
-            available_proxies = list(datastore.proxy_list.keys())
             # When enabled
             system_proxy = datastore.data['settings']['requests']['proxy']
             # In the case it doesnt exist anymore
-            if not system_proxy in available_proxies:
+            if not any([system_proxy in tup for tup in datastore.proxy_list]):
                 system_proxy = None
 
-            default['requests']['proxy'] = system_proxy if system_proxy is not None else available_proxies[0]
+            default['requests']['proxy'] = system_proxy if system_proxy is not None else datastore.proxy_list[0][0]
             # Used by the form handler to keep or remove the proxy settings
-            default['proxy_list'] = available_proxies[0]
+            default['proxy_list'] = datastore.proxy_list
 
 
         # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status
         form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,
                                         data=default
                                         )
-
-        # Remove the last option 'System default'
-        form.application.form.notification_format.choices.pop()
-
         if datastore.proxy_list is None:
             # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
             del form.requests.form.proxy
         else:
-            form.requests.form.proxy.choices = []
+            form.requests.form.proxy.choices = datastore.proxy_list
-            for p in datastore.proxy_list:
-                form.requests.form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))
-
 
         if request.method == 'POST':
             # Password unset is a GET, but we can lock the session to a salted env password to always need the password
@@ -755,8 +732,7 @@ def changedetection_app(config=None, datastore_o=None):
                                  current_base_url = datastore.data['settings']['application']['base_url'],
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
                                  api_key=datastore.data['settings']['application'].get('api_access_token'),
-                                 emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
+                                 emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False))
-                                 settings_application=datastore.data['settings']['application'])
 
         return output
 
@@ -808,12 +784,10 @@ def changedetection_app(config=None, datastore_o=None):
 
         return redirect(url_for('index'))
 
-    @app.route("/diff/<string:uuid>", methods=['GET', 'POST'])
+    @app.route("/diff/<string:uuid>", methods=['GET'])
     @login_required
     def diff_history_page(uuid):
 
-        from changedetectionio import forms
-
         # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
@@ -825,28 +799,6 @@ def changedetection_app(config=None, datastore_o=None):
             flash("No history found for the specified link, bad link?", "error")
             return redirect(url_for('index'))
 
-        # For submission of requesting an extract
-        extract_form = forms.extractDataForm(request.form)
-        if request.method == 'POST':
-            if not extract_form.validate():
-                flash("An error occurred, please see below.", "error")
-
-            else:
-                extract_regex = request.form.get('extract_regex').strip()
-                output = watch.extract_regex_from_all_history(extract_regex)
-                if output:
-                    watch_dir = os.path.join(datastore_o.datastore_path, uuid)
-                    response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True))
-                    response.headers['Content-type'] = 'text/csv'
-                    response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
-                    response.headers['Pragma'] = 'no-cache'
-                    response.headers['Expires'] = 0
-                    return response
-
-
-                flash('Nothing matches that RegEx', 'error')
-                redirect(url_for('diff_history_page', uuid=uuid)+'#extract')
-
         history = watch.history
         dates = list(history.keys())
 
@@ -859,10 +811,8 @@ def changedetection_app(config=None, datastore_o=None):
 
         newest_file = history[dates[-1]]
 
-        # Read as binary and force decode as UTF-8
-        # Windows may fail decode in python if we just use 'r' mode (chardet decode exception)
         try:
-            with open(newest_file, 'r', encoding='utf-8', errors='ignore') as f:
+            with open(newest_file, 'r') as f:
                 newest_version_file_contents = f.read()
         except Exception as e:
             newest_version_file_contents = "Unable to read {}.\n".format(newest_file)
@@ -875,7 +825,7 @@ def changedetection_app(config=None, datastore_o=None):
             previous_file = history[dates[-2]]
 
         try:
-            with open(previous_file, 'r', encoding='utf-8', errors='ignore') as f:
+            with open(previous_file, 'r') as f:
                 previous_version_file_contents = f.read()
         except Exception as e:
             previous_version_file_contents = "Unable to read {}.\n".format(previous_file)
@@ -889,23 +839,22 @@ def changedetection_app(config=None, datastore_o=None):
                     watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
 
         output = render_template("diff.html",
-                                 current_diff_url=watch['url'],
+                                 watch_a=watch,
-                                 current_previous_version=str(previous_version),
+                                 newest=newest_version_file_contents,
+                                 previous=previous_version_file_contents,
                                  extra_stylesheets=extra_stylesheets,
+                                 versions=dates[:-1], # All except current/last
+                                 uuid=uuid,
+                                 newest_version_timestamp=dates[-1],
+                                 current_previous_version=str(previous_version),
+                                 current_diff_url=watch['url'],
                                  extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
-                                 extract_form=extract_form,
+                                 left_sticky=True,
+                                 screenshot=screenshot_url,
                                  is_html_webdriver=is_html_webdriver,
                                  last_error=watch['last_error'],
-                                 last_error_screenshot=watch.get_error_snapshot(),
                                  last_error_text=watch.get_error_text(),
-                                 left_sticky=True,
+                                 last_error_screenshot=watch.get_error_snapshot()
-                                 newest=newest_version_file_contents,
-                                 newest_version_timestamp=dates[-1],
-                                 previous=previous_version_file_contents,
-                                 screenshot=screenshot_url,
-                                 uuid=uuid,
-                                 versions=dates[:-1], # All except current/last
-                                 watch_a=watch
                                  )
 
         return output
@@ -953,7 +902,7 @@ def changedetection_app(config=None, datastore_o=None):
         timestamp = list(watch.history.keys())[-1]
         filename = watch.history[timestamp]
         try:
-            with open(filename, 'r', encoding='utf-8', errors='ignore') as f:
+            with open(filename, 'r') as f:
                 tmp = f.readlines()
 
                 # Get what needs to be highlighted
@@ -1009,6 +958,10 @@ def changedetection_app(config=None, datastore_o=None):
 
         return output
 
+    @app.route("/favicon.ico", methods=['GET'])
+    def favicon():
+        return send_from_directory("static/images", path="favicon.ico")
+
     # We're good but backups are even better!
     @app.route("/backup", methods=['GET'])
     @login_required
@@ -1024,6 +977,9 @@ def changedetection_app(config=None, datastore_o=None):
 
         # create a ZipFile object
         backupname = "changedetection-backup-{}.zip".format(int(time.time()))
+
+        # We only care about UUIDS from the current index file
+        uuids = list(datastore.data['watching'].keys())
         backup_filepath = os.path.join(datastore_o.datastore_path, backupname)
 
         with zipfile.ZipFile(backup_filepath, "w",
@@ -1039,12 +995,12 @@ def changedetection_app(config=None, datastore_o=None):
             # Add the flask app secret
             zipObj.write(os.path.join(datastore_o.datastore_path, "secret.txt"), arcname="secret.txt")
 
-            # Add any data in the watch data directory.
+            # Add any snapshot data we find, use the full path to access the file, but make the file 'relative' in the Zip.
-            for uuid, w in datastore.data['watching'].items():
+            for txt_file_path in Path(datastore_o.datastore_path).rglob('*.txt'):
-                for f in Path(w.watch_data_dir).glob('*'):
+                parent_p = txt_file_path.parent
-                    zipObj.write(f,
+                if parent_p.name in uuids:
-                                 # Use the full path to access the file, but make the file 'relative' in the Zip.
+                    zipObj.write(txt_file_path,
-                                 arcname=os.path.join(f.parts[-2], f.parts[-1]),
+                                 arcname=str(txt_file_path).replace(datastore_o.datastore_path, ''),
                                  compress_type=zipfile.ZIP_DEFLATED,
                                  compresslevel=8)
 
@@ -1223,6 +1179,7 @@ def changedetection_app(config=None, datastore_o=None):
         else:
             # No tag, no uuid, add everything.
             for watch_uuid, watch in datastore.data['watching'].items():
+
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                     update_q.put((1, watch_uuid))
                     i += 1
@@ -1242,7 +1199,7 @@ def changedetection_app(config=None, datastore_o=None):
                     datastore.delete(uuid.strip())
             flash("{} watches deleted".format(len(uuids)))
 
-        elif (op == 'pause'):
+        if (op == 'pause'):
             for uuid in uuids:
                 uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
@@ -1250,40 +1207,13 @@ def changedetection_app(config=None, datastore_o=None):
 
             flash("{} watches paused".format(len(uuids)))
 
-        elif (op == 'unpause'):
+        if (op == 'unpause'):
             for uuid in uuids:
                 uuid = uuid.strip()
                 if datastore.data['watching'].get(uuid):
                     datastore.data['watching'][uuid.strip()]['paused'] = False
             flash("{} watches unpaused".format(len(uuids)))
 
-        elif (op == 'mute'):
-            for uuid in uuids:
-                uuid = uuid.strip()
-                if datastore.data['watching'].get(uuid):
-                    datastore.data['watching'][uuid.strip()]['notification_muted'] = True
-            flash("{} watches muted".format(len(uuids)))
-
-        elif (op == 'unmute'):
-            for uuid in uuids:
-                uuid = uuid.strip()
-                if datastore.data['watching'].get(uuid):
-                    datastore.data['watching'][uuid.strip()]['notification_muted'] = False
-            flash("{} watches un-muted".format(len(uuids)))
-
-        elif (op == 'notification-default'):
-            from changedetectionio.notification import (
-                default_notification_format_for_watch
-            )
-            for uuid in uuids:
-                uuid = uuid.strip()
-                if datastore.data['watching'].get(uuid):
-                    datastore.data['watching'][uuid.strip()]['notification_title'] = None
-                    datastore.data['watching'][uuid.strip()]['notification_body'] = None
-                    datastore.data['watching'][uuid.strip()]['notification_urls'] = []
-                    datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch
-            flash("{} watches set to use default notification settings".format(len(uuids)))
-
         return redirect(url_for('index'))
 
     @app.route("/api/share-url", methods=['GET'])
@@ -1340,19 +1270,13 @@ def changedetection_app(config=None, datastore_o=None):
         # paste in etc
         return redirect(url_for('index'))
 
-    import changedetectionio.blueprint.browser_steps as browser_steps
-    app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
-
-    import changedetectionio.blueprint.price_data_follower as price_data_follower
-    app.register_blueprint(price_data_follower.construct_blueprint(datastore), url_prefix='/price_data_follower')
-
-
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
+
     threading.Thread(target=notification_runner).start()
 
-    # Check for new release version, but not when running in test/build or pytest
+    # Check for new release version, but not when running in test/build
-    if not os.getenv("GITHUB_REF", False) and not config.get('disable_checkver') == True:
+    if not os.getenv("GITHUB_REF", False):
         threading.Thread(target=check_for_new_version).start()
 
     return app
@@ -1412,7 +1336,7 @@ def notification_runner():
                 # UUID wont be present when we submit a 'test' from the global settings
                 if 'uuid' in n_object:
                     datastore.update_watch(uuid=n_object['uuid'],
-                                           update_obj={'last_notification_error': "Notification error detected, goto notification log."})
+                                           update_obj={'last_notification_error': "Notification error detected, please see logs."})
 
                 log_lines = str(e).splitlines()
                 notification_debug_log += log_lines
@@ -1427,8 +1351,6 @@ def ticker_thread_check_time_launch_checks():
     import random
     from changedetectionio import update_worker
 
-    proxy_last_called_time = {}
-
     recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 20))
     print("System env MINIMUM_SECONDS_RECHECK_TIME", recheck_time_minimum_seconds)
 
@@ -1489,30 +1411,10 @@ def ticker_thread_check_time_launch_checks():
                 if watch.jitter_seconds == 0:
                     watch.jitter_seconds = random.uniform(-abs(jitter), jitter)
 
-            seconds_since_last_recheck = now - watch['last_checked']
 
+            seconds_since_last_recheck = now - watch['last_checked']
             if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds:
                 if not uuid in running_uuids and uuid not in [q_uuid for p,q_uuid in update_q.queue]:
-
-                    # Proxies can be set to have a limit on seconds between which they can be called
-                    watch_proxy = datastore.get_preferred_proxy_for_watch(uuid=uuid)
-                    if watch_proxy and watch_proxy in list(datastore.proxy_list.keys()):
-                        # Proxy may also have some threshold minimum
-                        proxy_list_reuse_time_minimum = int(datastore.proxy_list.get(watch_proxy, {}).get('reuse_time_minimum', 0))
-                        if proxy_list_reuse_time_minimum:
-                            proxy_last_used_time = proxy_last_called_time.get(watch_proxy, 0)
-                            time_since_proxy_used = int(time.time() - proxy_last_used_time)
-                            if time_since_proxy_used < proxy_list_reuse_time_minimum:
-                                # Not enough time difference reached, skip this watch
-                                print("> Skipped UUID {} using proxy '{}', not enough time between proxy requests {}s/{}s".format(uuid,
-                                                                                                                         watch_proxy,
-                                                                                                                         time_since_proxy_used,
-                                                                                                                         proxy_list_reuse_time_minimum))
-                                continue
-                            else:
-                                # Record the last used time
-                                proxy_last_called_time[watch_proxy] = int(time.time())
-
                     # Use Epoch time as priority, so we get a "sorted" PriorityQueue, but we can still push a priority 1 into it.
                     priority = int(time.time())
                     print(

changedetectionio/api/api_v1.py

@@ -122,37 +122,3 @@ class CreateWatch(Resource):
             return {'status': "OK"}, 200
 
         return list, 200
-
-class SystemInfo(Resource):
-    def __init__(self, **kwargs):
-        # datastore is a black box dependency
-        self.datastore = kwargs['datastore']
-        self.update_q = kwargs['update_q']
-
-    @auth.check_token
-    def get(self):
-        import time
-        overdue_watches = []
-
-        # Check all watches and report which have not been checked but should have been
-
-        for uuid, watch in self.datastore.data.get('watching', {}).items():
-            # see if now - last_checked is greater than the time that should have been
-            # this is not super accurate (maybe they just edited it) but better than nothing
-            t = watch.threshold_seconds()
-            if not t:
-                # Use the system wide default
-                t = self.datastore.threshold_seconds
-
-            time_since_check = time.time() - watch.get('last_checked')
-
-            # Allow 5 minutes of grace time before we decide it's overdue
-            if time_since_check - (5 * 60) > t:
-                overdue_watches.append(uuid)
-
-        return {
-                   'queue_size': self.update_q.qsize(),
-                   'overdue_watches': overdue_watches,
-                   'uptime': round(time.time() - self.datastore.start_time, 2),
-                   'watch_count': len(self.datastore.data.get('watching', {}))
-               }, 200

changedetectionio/blueprint/browser_steps/__init__.py

@@ -1,231 +0,0 @@
-
-# HORRIBLE HACK BUT WORKS :-) PR anyone?
-#
-# Why?
-# `browsersteps_playwright_browser_interface.chromium.connect_over_cdp()` will only run once without async()
-# - this flask app is not async()
-# - browserless has a single timeout/keepalive which applies to the session made at .connect_over_cdp()
-#
-# So it means that we must unfortunately for now just keep a single timer since .connect_over_cdp() was run
-# and know when that reaches timeout/keepalive :( when that time is up, restart the connection and tell the user
-# that their time is up, insert another coin. (reload)
-#
-# Bigger picture
-# - It's horrible that we have this click+wait deal, some nice socket.io solution using something similar
-# to what the browserless debug UI already gives us would be smarter..
-#
-# OR
-# - Some API call that should be hacked into browserless or playwright that we can "/api/bump-keepalive/{session_id}/60"
-# So we can tell it that we need more time (run this on each action)
-#
-# OR
-# - use multiprocessing to bump this over to its own process and add some transport layer (queue/pipes)
-
-from distutils.util import strtobool
-from flask import Blueprint, request, make_response
-from flask_login import login_required
-import os
-import logging
-from changedetectionio.store import ChangeDetectionStore
-
-browsersteps_live_ui_o = {}
-browsersteps_playwright_browser_interface = None
-browsersteps_playwright_browser_interface_browser = None
-browsersteps_playwright_browser_interface_context = None
-browsersteps_playwright_browser_interface_end_time = None
-browsersteps_playwright_browser_interface_start_time = None
-
-def cleanup_playwright_session():
-
-    global browsersteps_live_ui_o
-    global browsersteps_playwright_browser_interface
-    global browsersteps_playwright_browser_interface_browser
-    global browsersteps_playwright_browser_interface_context
-    global browsersteps_playwright_browser_interface_end_time
-    global browsersteps_playwright_browser_interface_start_time
-
-    browsersteps_live_ui_o = {}
-    browsersteps_playwright_browser_interface = None
-    browsersteps_playwright_browser_interface_browser = None
-    browsersteps_playwright_browser_interface_end_time = None
-    browsersteps_playwright_browser_interface_start_time = None
-
-    print("Cleaning up old playwright session because time was up, calling .goodbye()")
-    try:
-        browsersteps_playwright_browser_interface_context.goodbye()
-    except Exception as e:
-        print ("Got exception in shutdown, probably OK")
-        print (str(e))
-
-    browsersteps_playwright_browser_interface_context = None
-
-    print ("Cleaning up old playwright session because time was up - done")
-
-def construct_blueprint(datastore: ChangeDetectionStore):
-
-    browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
-
-    @login_required
-    @browser_steps_blueprint.route("/browsersteps_update", methods=['GET', 'POST'])
-    def browsersteps_ui_update():
-        import base64
-        import playwright._impl._api_types
-        import time
-
-        from changedetectionio.blueprint.browser_steps import browser_steps
-
-        global browsersteps_live_ui_o, browsersteps_playwright_browser_interface_end_time
-        global browsersteps_playwright_browser_interface_browser
-        global browsersteps_playwright_browser_interface
-        global browsersteps_playwright_browser_interface_start_time
-
-        step_n = None
-        remaining =0
-        uuid = request.args.get('uuid')
-
-        browsersteps_session_id = request.args.get('browsersteps_session_id')
-
-        if not browsersteps_session_id:
-            return make_response('No browsersteps_session_id specified', 500)
-
-        # Because we don't "really" run in a context manager ( we make the playwright interface global/long-living )
-        # We need to manage the shutdown when the time is up
-        if browsersteps_playwright_browser_interface_end_time:
-            remaining = browsersteps_playwright_browser_interface_end_time-time.time()
-            if browsersteps_playwright_browser_interface_end_time and remaining <= 0:
-                cleanup_playwright_session()
-                return make_response('Browser session expired, please reload the Browser Steps interface', 401)
-
-        # Actions - step/apply/etc, do the thing and return state
-        if request.method == 'POST':
-            # @todo - should always be an existing session
-            step_operation = request.form.get('operation')
-            step_selector = request.form.get('selector')
-            step_optional_value = request.form.get('optional_value')
-            step_n = int(request.form.get('step_n'))
-            is_last_step = strtobool(request.form.get('is_last_step'))
-
-            if step_operation == 'Goto site':
-                step_operation = 'goto_url'
-                step_optional_value = None
-                step_selector = datastore.data['watching'][uuid].get('url')
-
-            # @todo try.. accept.. nice errors not popups..
-            try:
-
-                this_session = browsersteps_live_ui_o.get(browsersteps_session_id)
-                if not this_session:
-                    print("Browser exited")
-                    return make_response('Browser session ran out of time :( Please reload this page.', 401)
-
-                this_session.call_action(action_name=step_operation,
-                                         selector=step_selector,
-                                         optional_value=step_optional_value)
-
-            except Exception as e:
-                print("Exception when calling step operation", step_operation, str(e))
-                # Try to find something of value to give back to the user
-                return make_response(str(e).splitlines()[0], 401)
-
-            # Get visual selector ready/update its data (also use the current filter info from the page?)
-            # When the last 'apply' button was pressed
-            # @todo this adds overhead because the xpath selection is happening twice
-            u = this_session.page.url
-            if is_last_step and u:
-                (screenshot, xpath_data) = this_session.request_visualselector_data()
-                datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
-                datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
-
-        # Setup interface
-        if request.method == 'GET':
-
-            if not browsersteps_playwright_browser_interface:
-                print("Starting connection with playwright")
-                logging.debug("browser_steps.py connecting")
-
-                global browsersteps_playwright_browser_interface_context
-                from . import nonContext
-                browsersteps_playwright_browser_interface_context = nonContext.c_sync_playwright()
-                browsersteps_playwright_browser_interface = browsersteps_playwright_browser_interface_context.start()
-
-                time.sleep(1)
-                # At 20 minutes, some other variable is closing it
-                # @todo find out what it is and set it
-                seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
-
-                # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
-                keepalive = "&timeout={}".format(((seconds_keepalive+3) * 1000))
-                try:
-                    browsersteps_playwright_browser_interface_browser = browsersteps_playwright_browser_interface.chromium.connect_over_cdp(
-                        os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
-                except Exception as e:
-                    if 'ECONNREFUSED' in str(e):
-                        return make_response('Unable to start the Playwright session properly, is it running?', 401)
-
-                browsersteps_playwright_browser_interface_end_time = time.time() + (seconds_keepalive-3)
-                print("Starting connection with playwright - done")
-
-            if not browsersteps_live_ui_o.get(browsersteps_session_id):
-                # Boot up a new session
-                proxy_id = datastore.get_preferred_proxy_for_watch(uuid=uuid)
-                proxy = None
-                if proxy_id:
-                    proxy_url = datastore.proxy_list.get(proxy_id).get('url')
-                    if proxy_url:
-                        proxy = {'server': proxy_url}
-                        print("Browser Steps: UUID {} Using proxy {}".format(uuid, proxy_url))
-
-                # Begin the new "Playwright Context" that re-uses the playwright interface
-                # Each session is a "Playwright Context" as a list, that uses the playwright interface
-                browsersteps_live_ui_o[browsersteps_session_id] = browser_steps.browsersteps_live_ui(
-                    playwright_browser=browsersteps_playwright_browser_interface_browser,
-                    proxy=proxy)
-                this_session = browsersteps_live_ui_o[browsersteps_session_id]
-
-        if not this_session.page:
-            cleanup_playwright_session()
-            return make_response('Browser session ran out of time :( Please reload this page.', 401)
-
-        response = None
-
-        if request.method == 'POST':
-            # Screenshots and other info only needed on requesting a step (POST)
-            try:
-                state = this_session.get_current_state()
-            except playwright._impl._api_types.Error as e:
-                return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
-
-            # Use send_file() which is way faster than read/write loop on bytes
-            import json
-            from tempfile import mkstemp
-            from flask import send_file
-            tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
-
-            output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
-                base64.b64encode(state[0]).decode('ascii')),
-                'xpath_data': state[1],
-                'session_age_start': this_session.age_start,
-                'browser_time_remaining': round(remaining)
-            })
-
-            with os.fdopen(tmp_fd, 'w') as f:
-                f.write(output)
-
-            response = make_response(send_file(path_or_file=tmp_file,
-                                               mimetype='application/json; charset=UTF-8',
-                                               etag=True))
-            # No longer needed
-            os.unlink(tmp_file)
-
-        elif request.method == 'GET':
-            # Just enough to get the session rolling, it will call for goto-site via POST next
-            response = make_response({
-                'session_age_start': this_session.age_start,
-                'browser_time_remaining': round(remaining)
-            })
-
-        return response
-
-    return browser_steps_blueprint
-
-

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -1,270 +0,0 @@
-#!/usr/bin/python3
-
-import os
-import time
-import re
-from random import randint
-
-# Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
-# 0- off, 1- on
-browser_step_ui_config = {'Choose one': '0 0',
-                          #                 'Check checkbox': '1 0',
-                          #                 'Click button containing text': '0 1',
-                          #                 'Scroll to bottom': '0 0',
-                          #                 'Scroll to element': '1 0',
-                          #                 'Scroll to top': '0 0',
-                          #                 'Switch to iFrame by index number': '0 1'
-                          #                 'Uncheck checkbox': '1 0',
-                          # @todo
-                          'Check checkbox': '1 0',
-                          'Click X,Y': '0 1',
-                          'Click element if exists': '1 0',
-                          'Click element': '1 0',
-                          'Click element containing text': '0 1',
-                          'Enter text in field': '1 1',
-                          'Execute JS': '0 1',
-#                          'Extract text and use as filter': '1 0',
-                          'Goto site': '0 0',
-                          'Press Enter': '0 0',
-                          'Select by label': '1 1',
-                          'Scroll down': '0 0',
-                          'Uncheck checkbox': '1 0',
-                          'Wait for seconds': '0 1',
-                          'Wait for text': '0 1',
-                          #                          'Press Page Down': '0 0',
-                          #                          'Press Page Up': '0 0',
-                          # weird bug, come back to it later
-                          }
-
-
-# Good reference - https://playwright.dev/python/docs/input
-#                  https://pythonmana.com/2021/12/202112162236307035.html
-#
-# ONLY Works in Playwright because we need the fullscreen screenshot
-class steppable_browser_interface():
-    page = None
-
-    # Convert and perform "Click Button" for example
-    def call_action(self, action_name, selector=None, optional_value=None):
-        now = time.time()
-        call_action_name = re.sub('[^0-9a-zA-Z]+', '_', action_name.lower())
-        if call_action_name == 'choose_one':
-            return
-
-        print("> action calling", call_action_name)
-        # https://playwright.dev/python/docs/selectors#xpath-selectors
-        if selector.startswith('/') and not selector.startswith('//'):
-            selector = "xpath=" + selector
-
-        action_handler = getattr(self, "action_" + call_action_name)
-
-        # Support for Jinja2 variables in the value and selector
-        from jinja2 import Environment
-        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
-
-        if selector and ('{%' in selector or '{{' in selector):
-            selector = str(jinja2_env.from_string(selector).render())
-
-        if optional_value and ('{%' in optional_value or '{{' in optional_value):
-            optional_value = str(jinja2_env.from_string(optional_value).render())
-
-        action_handler(selector, optional_value)
-        self.page.wait_for_timeout(3 * 1000)
-        print("Call action done in", time.time() - now)
-
-    def action_goto_url(self, url, optional_value):
-        # self.page.set_viewport_size({"width": 1280, "height": 5000})
-        now = time.time()
-        response = self.page.goto(url, timeout=0, wait_until='domcontentloaded')
-        print("Time to goto URL", time.time() - now)
-
-        # Wait_until = commit
-        # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
-        # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
-        # This seemed to solve nearly all 'TimeoutErrors'
-        extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))
-        self.page.wait_for_timeout(extra_wait * 1000)
-
-    def action_click_element_containing_text(self, selector=None, value=''):
-        if not len(value.strip()):
-            return
-        elem = self.page.get_by_text(value)
-        if elem.count():
-            elem.first.click(delay=randint(200, 500), timeout=3000)
-
-    def action_enter_text_in_field(self, selector, value):
-        if not len(selector.strip()):
-            return
-
-        self.page.fill(selector, value, timeout=10 * 1000)
-
-    def action_execute_js(self, selector, value):
-        self.page.evaluate(value)
-
-    def action_click_element(self, selector, value):
-        print("Clicking element")
-        if not len(selector.strip()):
-            return
-        self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
-
-    def action_click_element_if_exists(self, selector, value):
-        import playwright._impl._api_types as _api_types
-        print("Clicking element if exists")
-        if not len(selector.strip()):
-            return
-        try:
-            self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
-        except _api_types.TimeoutError as e:
-            return
-        except _api_types.Error as e:
-            # Element was there, but page redrew and now its long long gone
-            return
-
-    def action_click_x_y(self, selector, value):
-        x, y = value.strip().split(',')
-        x = int(float(x.strip()))
-        y = int(float(y.strip()))
-        self.page.mouse.click(x=x, y=y, delay=randint(200, 500))
-
-    def action_scroll_down(self, selector, value):
-        # Some sites this doesnt work on for some reason
-        self.page.mouse.wheel(0, 600)
-        self.page.wait_for_timeout(1000)
-
-    def action_wait_for_seconds(self, selector, value):
-        self.page.wait_for_timeout(int(value) * 1000)
-
-    # @todo - in the future make some popout interface to capture what needs to be set
-    # https://playwright.dev/python/docs/api/class-keyboard
-    def action_press_enter(self, selector, value):
-        self.page.keyboard.press("Enter", delay=randint(200, 500))
-
-    def action_press_page_up(self, selector, value):
-        self.page.keyboard.press("PageUp", delay=randint(200, 500))
-
-    def action_press_page_down(self, selector, value):
-        self.page.keyboard.press("PageDown", delay=randint(200, 500))
-
-    def action_check_checkbox(self, selector, value):
-        self.page.locator(selector).check(timeout=1000)
-
-    def action_uncheck_checkbox(self, selector, value):
-        self.page.locator(selector, timeout=1000).uncheck(timeout=1000)
-
-
-# Responsible for maintaining a live 'context' with browserless
-# @todo - how long do contexts live for anyway?
-class browsersteps_live_ui(steppable_browser_interface):
-    context = None
-    page = None
-    render_extra_delay = 1
-    stale = False
-    # bump and kill this if idle after X sec
-    age_start = 0
-
-    # use a special driver, maybe locally etc
-    command_executor = os.getenv(
-        "PLAYWRIGHT_BROWSERSTEPS_DRIVER_URL"
-    )
-    # if not..
-    if not command_executor:
-        command_executor = os.getenv(
-            "PLAYWRIGHT_DRIVER_URL",
-            'ws://playwright-chrome:3000'
-        ).strip('"')
-
-    browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
-
-    def __init__(self, playwright_browser, proxy=None):
-        self.age_start = time.time()
-        self.playwright_browser = playwright_browser
-        if self.context is None:
-            self.connect(proxy=proxy)
-
-    # Connect and setup a new context
-    def connect(self, proxy=None):
-        # Should only get called once - test that
-        keep_open = 1000 * 60 * 5
-        now = time.time()
-
-        # @todo handle multiple contexts, bind a unique id from the browser on each req?
-        self.context = self.playwright_browser.new_context(
-            # @todo
-            #                user_agent=request_headers['User-Agent'] if request_headers.get('User-Agent') else 'Mozilla/5.0',
-            #               proxy=self.proxy,
-            # This is needed to enable JavaScript execution on GitHub and others
-            bypass_csp=True,
-            # Should never be needed
-            accept_downloads=False,
-            proxy=proxy
-        )
-
-        self.page = self.context.new_page()
-
-        # self.page.set_default_navigation_timeout(keep_open)
-        self.page.set_default_timeout(keep_open)
-        # @todo probably this doesnt work
-        self.page.on(
-            "close",
-            self.mark_as_closed,
-        )
-        # Listen for all console events and handle errors
-        self.page.on("console", lambda msg: print(f"Browser steps console - {msg.type}: {msg.text} {msg.args}"))
-
-        print("Time to browser setup", time.time() - now)
-        self.page.wait_for_timeout(1 * 1000)
-
-    def mark_as_closed(self):
-        print("Page closed, cleaning up..")
-
-    @property
-    def has_expired(self):
-        if not self.page:
-            return True
-
-
-    def get_current_state(self):
-        """Return the screenshot and interactive elements mapping, generally always called after action_()"""
-        from pkg_resources import resource_string
-        xpath_element_js = resource_string(__name__, "../../res/xpath_element_scraper.js").decode('utf-8')
-        now = time.time()
-        self.page.wait_for_timeout(1 * 1000)
-
-        # The actual screenshot
-        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=40)
-
-        self.page.evaluate("var include_filters=''")
-        # Go find the interactive elements
-        # @todo in the future, something smarter that can scan for elements with .click/focus etc event handlers?
-        elements = 'a,button,input,select,textarea,i,th,td,p,li,h1,h2,h3,h4,div,span'
-        xpath_element_js = xpath_element_js.replace('%ELEMENTS%', elements)
-        xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
-        # So the JS will find the smallest one first
-        xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
-        print("Time to complete get_current_state of browser", time.time() - now)
-        # except
-        # playwright._impl._api_types.Error: Browser closed.
-        # @todo show some countdown timer?
-        return (screenshot, xpath_data)
-
-    def request_visualselector_data(self):
-        """
-        Does the same that the playwright operation in content_fetcher does
-        This is used to just bump the VisualSelector data so it' ready to go if they click on the tab
-        @todo refactor and remove duplicate code, add include_filters
-        :param xpath_data:
-        :param screenshot:
-        :param current_include_filters:
-        :return:
-        """
-
-        self.page.evaluate("var include_filters=''")
-        from pkg_resources import resource_string
-        # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
-        xpath_element_js = resource_string(__name__, "../../res/xpath_element_scraper.js").decode('utf-8')
-        from changedetectionio.content_fetcher import visualselector_xpath_selectors
-        xpath_element_js = xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
-        xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
-        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
-
-        return (screenshot, xpath_data)

changedetectionio/blueprint/browser_steps/nonContext.py

@@ -1,18 +0,0 @@
-from playwright.sync_api import PlaywrightContextManager
-import asyncio
-
-# So playwright wants to run as a context manager, but we do something horrible and hacky
-# we are holding the session open for as long as possible, then shutting it down, and opening a new one
-# So it means we don't get to use PlaywrightContextManager' __enter__ __exit__
-# To work around this, make goodbye() act the same as the __exit__()
-#
-# But actually I think this is because the context is opened correctly with __enter__() but we timeout the connection
-# then theres some lock condition where we cant destroy it without it hanging
-
-class c_PlaywrightContextManager(PlaywrightContextManager):
-
-    def goodbye(self) -> None:
-        self.__exit__()
-
-def c_sync_playwright() -> PlaywrightContextManager:
-    return c_PlaywrightContextManager()

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -1,30 +0,0 @@
-
-from distutils.util import strtobool
-from flask import Blueprint, flash, redirect, url_for
-from flask_login import login_required
-from changedetectionio.store import ChangeDetectionStore
-
-PRICE_DATA_TRACK_ACCEPT = 'accepted'
-PRICE_DATA_TRACK_REJECT = 'rejected'
-
-def construct_blueprint(datastore: ChangeDetectionStore):
-
-    price_data_follower_blueprint = Blueprint('price_data_follower', __name__)
-
-    @login_required
-    @price_data_follower_blueprint.route("/<string:uuid>/accept", methods=['GET'])
-    def accept(uuid):
-        datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_ACCEPT
-        return redirect(url_for("form_watch_checknow", uuid=uuid))
-
-
-    @login_required
-    @price_data_follower_blueprint.route("/<string:uuid>/reject", methods=['GET'])
-    def reject(uuid):
-        datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_REJECT
-        return redirect(url_for("index"))
-
-
-    return price_data_follower_blueprint
-
-

changedetectionio/changedetection.py

@@ -2,20 +2,19 @@
 
 # Launch as a eventlet.wsgi server instance.
 
-from distutils.util import strtobool
-import eventlet
-import eventlet.wsgi
 import getopt
 import os
 import signal
 import sys
 
+import eventlet
+import eventlet.wsgi
 from . import store, changedetection_app, content_fetcher
 from . import __version__
 
 # Only global so we can access it in the signal handler
-app = None
 datastore = None
+app = None
 
 def sigterm_handler(_signo, _stack_frame):
     global app
@@ -103,15 +102,6 @@ def main():
                     has_password=datastore.data['settings']['application']['password'] != False
                     )
 
-    # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.
-    # @Note: Incompatible with password login (and maybe other features) for now, submit a PR!
-    @app.after_request
-    def hide_referrer(response):
-        if strtobool(os.getenv("HIDE_REFERER", 'false')):
-            response.headers["Referrer-Policy"] = "no-referrer"
-
-        return response
-
     # Proxy sub-directory support
     # Set environment var USE_X_SETTINGS=1 on this script
     # And then in your proxy_pass settings

changedetectionio/content_fetcher.py

@@ -1,13 +1,11 @@
-from abc import abstractmethod
+from abc import ABC, abstractmethod
 import chardet
 import json
-import logging
 import os
 import requests
-import sys
 import time
+import sys
 
-visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary'
 
 class Non200ErrorCodeReceived(Exception):
     def __init__(self, status_code, url, screenshot=None, xpath_data=None, page_html=None):
@@ -32,12 +30,6 @@ class JSActionExceptions(Exception):
         self.message = message
         return
 
-class BrowserStepsStepTimout(Exception):
-    def __init__(self, step_n):
-        self.step_n = step_n
-        return
-
-
 class PageUnloadable(Exception):
     def __init__(self, status_code, url, screenshot=False, message=False):
         # Set this so we can use it in other parts of the app
@@ -78,13 +70,134 @@ class Fetcher():
     status_code = None
     content = None
     headers = None
-    browser_steps = None
-    browser_steps_screenshot_path = None
 
     fetcher_description = "No description"
     webdriver_js_execute_code = None
-    xpath_element_js = ""
+    xpath_element_js = """               
+                // Include the getXpath script directly, easier than fetching
+                !function(e,n){"object"==typeof exports&&"undefined"!=typeof module?module.exports=n():"function"==typeof define&&define.amd?define(n):(e=e||self).getXPath=n()}(this,function(){return function(e){var n=e;if(n&&n.id)return'//*[@id="'+n.id+'"]';for(var o=[];n&&Node.ELEMENT_NODE===n.nodeType;){for(var i=0,r=!1,d=n.previousSibling;d;)d.nodeType!==Node.DOCUMENT_TYPE_NODE&&d.nodeName===n.nodeName&&i++,d=d.previousSibling;for(d=n.nextSibling;d;){if(d.nodeName===n.nodeName){r=!0;break}d=d.nextSibling}o.push((n.prefix?n.prefix+":":"")+n.localName+(i||r?"["+(i+1)+"]":"")),n=n.parentNode}return o.length?"/"+o.reverse().join("/"):""}});
 
+
+                const findUpTag = (el) => {
+                  let r = el
+                  chained_css = [];
+                  depth=0;
+            
+                // Strategy 1: Keep going up until we hit an ID tag, imagine it's like  #list-widget div h4
+                  while (r.parentNode) {
+                    if(depth==5) {
+                      break;
+                    }
+                    if('' !==r.id) {
+                      chained_css.unshift("#"+CSS.escape(r.id));
+                      final_selector= chained_css.join(' > ');
+                      // Be sure theres only one, some sites have multiples of the same ID tag :-(
+                      if (window.document.querySelectorAll(final_selector).length ==1 ) {
+                        return final_selector;
+                        }
+                      return null;
+                    } else {
+                      chained_css.unshift(r.tagName.toLowerCase());
+                    }
+                    r=r.parentNode;
+                    depth+=1;
+                  }
+                  return null;
+                }
+
+
+                // @todo - if it's SVG or IMG, go into image diff mode
+                var elements = window.document.querySelectorAll("div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary");
+                var size_pos=[];
+                // after page fetch, inject this JS
+                // build a map of all elements and their positions (maybe that only include text?)
+                var bbox;
+                for (var i = 0; i < elements.length; i++) {   
+                 bbox = elements[i].getBoundingClientRect();
+
+                 // forget really small ones
+                 if (bbox['width'] <20 && bbox['height'] < 20 ) {
+                   continue;
+                 }
+
+                 // @todo the getXpath kind of sucks, it doesnt know when there is for example just one ID sometimes
+                 // it should not traverse when we know we can anchor off just an ID one level up etc..
+                 // maybe, get current class or id, keep traversing up looking for only class or id until there is just one match 
+
+                 // 1st primitive - if it has class, try joining it all and select, if theres only one.. well thats us.
+                 xpath_result=false;
+                 
+                 try {
+                   var d= findUpTag(elements[i]);
+                   if (d) {
+                     xpath_result =d;
+                   }                
+                 } catch (e) {
+                   console.log(e);
+                 }
+                 
+                 // You could swap it and default to getXpath and then try the smarter one
+                 // default back to the less intelligent one
+                 if (!xpath_result) {
+                    try {
+                       // I've seen on FB and eBay that this doesnt work
+                       // ReferenceError: getXPath is not defined at eval (eval at evaluate (:152:29), <anonymous>:67:20) at UtilityScript.evaluate (<anonymous>:159:18) at UtilityScript.<anonymous> (<anonymous>:1:44)
+                       xpath_result = getXPath(elements[i]);
+                     } catch (e) {
+                       console.log(e);
+                       continue;
+                     }            
+                 }
+                 
+                 if(window.getComputedStyle(elements[i]).visibility === "hidden") {
+                   continue;
+                 }
+
+                 size_pos.push({
+                   xpath: xpath_result,
+                   width: Math.round(bbox['width']), 
+                   height: Math.round(bbox['height']), 
+                   left: Math.floor(bbox['left']), 
+                   top: Math.floor(bbox['top']), 
+                   childCount: elements[i].childElementCount
+                 });                 
+                }
+
+
+                // inject the current one set in the css_filter, which may be a CSS rule
+                // used for displaying the current one in VisualSelector, where its not one we generated.
+                if (css_filter.length) {
+                   q=false;                   
+                   try {
+                       // is it xpath?
+                       if (css_filter.startsWith('/') || css_filter.startsWith('xpath:')) {
+                         q=document.evaluate(css_filter.replace('xpath:',''), document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+                       } else {
+                         q=document.querySelector(css_filter);
+                       }                       
+                   } catch (e) {
+                    // Maybe catch DOMException and alert? 
+                     console.log(e);                       
+                   }
+                   bbox=false;
+                   if(q) {
+                     bbox = q.getBoundingClientRect();
+                   }
+                                   
+                   if (bbox && bbox['width'] >0 && bbox['height']>0) {                       
+                       size_pos.push({
+                           xpath: css_filter,
+                           width: bbox['width'], 
+                           height: bbox['height'],
+                           left: bbox['left'],
+                           top: bbox['top'],
+                           childCount: q.childElementCount
+                         });
+                     }
+                }
+                // Window.width required for proper scaling in the frontend
+                return {'size_pos':size_pos, 'browser_width': window.innerWidth};
+    """
     xpath_data = None
 
     # Will be needed in the future by the VisualSelector, always get this where possible.
@@ -95,12 +208,6 @@ class Fetcher():
     # Time ONTOP of the system defined env minimum time
     render_extract_delay = 0
 
-    def __init__(self):
-        from pkg_resources import resource_string
-        # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
-        self.xpath_element_js = resource_string(__name__, "res/xpath_element_scraper.js").decode('utf-8')
-
-
     @abstractmethod
     def get_error(self):
         return self.error
@@ -113,7 +220,7 @@ class Fetcher():
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None):
+            current_css_filter=None):
         # Should set self.error, self.status_code and self.content
         pass
 
@@ -125,62 +232,11 @@ class Fetcher():
     def get_last_status_code(self):
         return self.status_code
 
-    @abstractmethod
-    def screenshot_step(self, step_n):
-        return None
-
     @abstractmethod
     # Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
     def is_ready(self):
         return True
 
-    def iterate_browser_steps(self):
-        from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-        from playwright._impl._api_types import TimeoutError
-        from jinja2 import Environment
-        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
-
-        step_n = 0
-
-        if self.browser_steps is not None and len(self.browser_steps):
-            interface = steppable_browser_interface()
-            interface.page = self.page
-
-            valid_steps = filter(lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'), self.browser_steps)
-
-            for step in valid_steps:
-                step_n += 1
-                print(">> Iterating check - browser Step n {} - {}...".format(step_n, step['operation']))
-                self.screenshot_step("before-"+str(step_n))
-                self.save_step_html("before-"+str(step_n))
-                try:
-                    optional_value = step['optional_value']
-                    selector = step['selector']
-                    # Support for jinja2 template in step values, with date module added
-                    if '{%' in step['optional_value'] or '{{' in step['optional_value']:
-                        optional_value = str(jinja2_env.from_string(step['optional_value']).render())
-                    if '{%' in step['selector'] or '{{' in step['selector']:
-                        selector = str(jinja2_env.from_string(step['selector']).render())
-
-                    getattr(interface, "call_action")(action_name=step['operation'],
-                                                      selector=selector,
-                                                      optional_value=optional_value)
-                    self.screenshot_step(step_n)
-                    self.save_step_html(step_n)
-                except TimeoutError:
-                    # Stop processing here
-                    raise BrowserStepsStepTimout(step_n=step_n)
-
-
-
-    # It's always good to reset these
-    def delete_browser_steps_screenshots(self):
-        import glob
-        if self.browser_steps_screenshot_path is not None:
-            dest = os.path.join(self.browser_steps_screenshot_path, 'step_*.jpeg')
-            files = glob.glob(dest)
-            for f in files:
-                os.unlink(f)
 
 #   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
 #   the current one would return javascript output (as we use JS to generate the diff)
@@ -199,6 +255,7 @@ def available_fetchers():
 
     return p
 
+
 class base_html_playwright(Fetcher):
     fetcher_description = "Playwright {}/Javascript".format(
         os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
@@ -216,7 +273,7 @@ class base_html_playwright(Fetcher):
     proxy = None
 
     def __init__(self, proxy_override=None):
-        super().__init__()
+
         # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
         self.browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"')
         self.command_executor = os.getenv(
@@ -236,26 +293,15 @@ class base_html_playwright(Fetcher):
 
         # allow per-watch proxy selection override
         if proxy_override:
-            self.proxy = {'server': proxy_override}
+            # https://playwright.dev/docs/network#http-proxy
-
+            from urllib.parse import urlparse
-    def screenshot_step(self, step_n=''):
+            parsed = urlparse(proxy_override)
-
+            proxy_url = "{}://{}:{}".format(parsed.scheme, parsed.hostname, parsed.port)
-        # There's a bug where we need to do it twice or it doesnt take the whole page, dont know why.
+            self.proxy = {'server': proxy_url}
-        self.page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024})
+            if parsed.username:
-        screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=85)
+                self.proxy['username'] = parsed.username
-
+            if parsed.password:
-        if self.browser_steps_screenshot_path is not None:
+                self.proxy['password'] = parsed.password
-            destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
-            logging.debug("Saving step screenshot to {}".format(destination))
-            with open(destination, 'wb') as f:
-                f.write(screenshot)
-
-    def save_step_html(self, step_n):
-        content = self.page.content()
-        destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
-        logging.debug("Saving step HTML to {}".format(destination))
-        with open(destination, 'w') as f:
-            f.write(content)
 
     def run(self,
             url,
@@ -264,12 +310,11 @@ class base_html_playwright(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None):
+            current_css_filter=None):
 
         from playwright.sync_api import sync_playwright
         import playwright._impl._api_types
-
+        from playwright._impl._api_types import Error, TimeoutError
-        self.delete_browser_steps_screenshots()
         response = None
         with sync_playwright() as p:
             browser_type = getattr(p, self.browser_type)
@@ -290,86 +335,80 @@ class base_html_playwright(Fetcher):
                 accept_downloads=False
             )
 
-            self.page = context.new_page()
             if len(request_headers):
                 context.set_extra_http_headers(request_headers)
 
+            page = context.new_page()
             try:
-                self.page.set_default_navigation_timeout(90000)
+                page.set_default_navigation_timeout(90000)
-                self.page.set_default_timeout(90000)
+                page.set_default_timeout(90000)
 
                 # Listen for all console events and handle errors
-                self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
+                page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
 
                 # Bug - never set viewport size BEFORE page.goto
 
-
                 # Waits for the next navigation. Using Python context manager
                 # prevents a race condition between clicking and waiting for a navigation.
-                with self.page.expect_navigation():
+                with page.expect_navigation():
-                    response = self.page.goto(url, wait_until='load')
+                    response = page.goto(url, wait_until='load')
-                # Wait_until = commit
-                # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
-                # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
-                # This seemed to solve nearly all 'TimeoutErrors'
-                extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
-                self.page.wait_for_timeout(extra_wait * 1000)
 
-                if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
-                    self.page.evaluate(self.webdriver_js_execute_code)
 
             except playwright._impl._api_types.TimeoutError as e:
                 context.close()
                 browser.close()
                 # This can be ok, we will try to grab what we could retrieve
                 pass
+
             except Exception as e:
-                print ("other exception when page.goto")
+                print("other exception when page.goto")
-                print (str(e))
+                print(str(e))
                 context.close()
                 browser.close()
-                raise PageUnloadable(url=url, status_code=None)
+                raise PageUnloadable(url=url, status_code=None, message=e.message)
-
 
             if response is None:
                 context.close()
                 browser.close()
-                print ("response object was none")
+                print("response object was none")
                 raise EmptyReply(url=url, status_code=None)
 
             # Bug 2(?) Set the viewport size AFTER loading the page
-            self.page.set_viewport_size({"width": 1280, "height": 1024})
+            page.set_viewport_size({"width": 1280, "height": 1024})
-
-            # Run Browser Steps here
-            self.iterate_browser_steps()
-
             extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
             time.sleep(extra_wait)
 
+            if self.webdriver_js_execute_code is not None:
+                try:
+                    page.evaluate(self.webdriver_js_execute_code)
+                except Exception as e:
+                    # Is it possible to get a screenshot?
+                    error_screenshot = False
+                    try:
+                        page.screenshot(type='jpeg',
+                                        clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024},
+                                        quality=1)
 
-            self.content = self.page.content()
+                        # The actual screenshot
+                        error_screenshot = page.screenshot(type='jpeg',
+                                                           full_page=True,
+                                                           quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
+                    except Exception as s:
+                        pass
+
+                    raise JSActionExceptions(status_code=response.status, screenshot=error_screenshot, message=str(e), url=url)
+
+            self.content = page.content()
             self.status_code = response.status
-
-            if len(self.page.content().strip()) == 0:
-                context.close()
-                browser.close()
-                print ("Content was empty")
-                raise EmptyReply(url=url, status_code=None)
-
-            # Bug 2(?) Set the viewport size AFTER loading the page
-            self.page.set_viewport_size({"width": 1280, "height": 1024})
-
-            self.status_code = response.status
-            self.content = self.page.content()
             self.headers = response.all_headers()
 
-            # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
+            if current_css_filter is not None:
-            if current_include_filters is not None:
+                page.evaluate("var css_filter={}".format(json.dumps(current_css_filter)))
-                self.page.evaluate("var include_filters={}".format(json.dumps(current_include_filters)))
             else:
-                self.page.evaluate("var include_filters=''")
+                page.evaluate("var css_filter=''")
 
-            self.xpath_data = self.page.evaluate("async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
+            # str() here must create a dereferenced copy, which allows the GC to release correctly
+            self.xpath_data = str(page.evaluate("async () => {" + self.xpath_element_js + "}"))
 
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
@@ -380,17 +419,26 @@ class base_html_playwright(Fetcher):
             # acceptable screenshot quality here
             try:
                 # Quality set to 1 because it's not used, just used as a work-around for a bug, no need to change this.
-                self.page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024}, quality=1)
+                page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024}, quality=1)
                 # The actual screenshot
-                self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
+                self.screenshot = page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
             except Exception as e:
                 context.close()
                 browser.close()
                 raise ScreenshotUnavailable(url=url, status_code=None)
 
+            if len(self.content.strip()) == 0:
+                context.close()
+                browser.close()
+                print("Content was empty")
+                raise EmptyReply(url=url, status_code=None, screenshot=self.screenshot)
+
             context.close()
             browser.close()
 
+            if not ignore_status_codes and self.status_code!=200:
+                raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, page_html=self.content, screenshot=self.screenshot)
+
 class base_html_webdriver(Fetcher):
     if os.getenv("WEBDRIVER_URL"):
         fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))
@@ -407,7 +455,6 @@ class base_html_webdriver(Fetcher):
     proxy = None
 
     def __init__(self, proxy_override=None):
-        super().__init__()
         from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
 
         # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
@@ -440,7 +487,7 @@ class base_html_webdriver(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None):
+            current_css_filter=None):
 
         from selenium import webdriver
         from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
@@ -468,6 +515,8 @@ class base_html_webdriver(Fetcher):
             # Selenium doesn't automatically wait for actions as good as Playwright, so wait again
             self.driver.implicitly_wait(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
 
+        self.screenshot = self.driver.get_screenshot_as_png()
+
         # @todo - how to check this? is it possible?
         self.status_code = 200
         # @todo somehow we should try to get this working for WebDriver
@@ -478,12 +527,11 @@ class base_html_webdriver(Fetcher):
         self.content = self.driver.page_source
         self.headers = {}
 
-        self.screenshot = self.driver.get_screenshot_as_png()
-
     # Does the connection to the webdriver work? run a test connection.
     def is_ready(self):
         from selenium import webdriver
         from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+        from selenium.common.exceptions import WebDriverException
 
         self.driver = webdriver.Remote(
             command_executor=self.command_executor,
@@ -515,12 +563,7 @@ class html_requests(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None):
+            current_css_filter=None):
-
-        # Make requests use a more modern looking user-agent
-        if not 'User-Agent' in request_headers:
-            request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT",
-                                                      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36')
 
         proxies = {}
 

changedetectionio/fetch_site_status.py

@@ -2,19 +2,14 @@ import hashlib
 import logging
 import os
 import re
+import time
 import urllib3
 
 from changedetectionio import content_fetcher, html_tools
-from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 
-class FilterNotFoundInResponse(ValueError):
-    def __init__(self, msg):
-        ValueError.__init__(self, msg)
-
-
 # Some common stuff here that can be moved to a base class
 # (set_proxy_from_list)
 class perform_site_check():
@@ -25,6 +20,34 @@ class perform_site_check():
         super().__init__(*args, **kwargs)
         self.datastore = datastore
 
+    # If there was a proxy list enabled, figure out what proxy_args/which proxy to use
+    # if watch.proxy use that
+    # fetcher.proxy_override = watch.proxy or main config proxy
+    # Allows override the proxy on a per-request basis
+    # ALWAYS use the first one is nothing selected
+
+    def set_proxy_from_list(self, watch):
+        proxy_args = None
+        if self.datastore.proxy_list is None:
+            return None
+
+        # If its a valid one
+        if any([watch['proxy'] in p for p in self.datastore.proxy_list]):
+            proxy_args = watch['proxy']
+
+        # not valid (including None), try the system one
+        else:
+            system_proxy = self.datastore.data['settings']['requests']['proxy']
+            # Is not None and exists
+            if any([system_proxy in p for p in self.datastore.proxy_list]):
+                proxy_args = system_proxy
+
+        # Fallback - Did not resolve anything, use the first available
+        if proxy_args is None:
+            proxy_args = self.datastore.proxy_list[0][0]
+
+        return proxy_args
+
     # Doesn't look like python supports forward slash auto enclosure in re.findall
     # So convert it to inline flag "foobar(?i)" type configuration
     def forward_slash_enclosed_regex_to_options(self, regex):
@@ -38,20 +61,18 @@ class perform_site_check():
 
         return regex
 
+
     def run(self, uuid):
-        from copy import deepcopy
+        timestamp = int(time.time())  # used for storage etc too
+
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
 
-        # DeepCopy so we can be sure we don't accidently change anything by reference
+        watch = self.datastore.data['watching'][uuid]
-        watch = deepcopy(self.datastore.data['watching'].get(uuid))
-
-        if not watch:
-            return
 
         # Protect against file:// access
-        if re.search(r'^file', watch.get('url', ''), re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
+        if re.search(r'^file', watch['url'], re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
             raise Exception(
                 "file:// type access is denied for security reasons."
             )
@@ -59,10 +80,10 @@ class perform_site_check():
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
 
-        extra_headers = watch.get('headers', [])
+        extra_headers = self.datastore.get_val(uuid, 'headers')
 
         # Tweak the base config with the per-watch ones
-        request_headers = deepcopy(self.datastore.data['settings']['headers'])
+        request_headers = self.datastore.data['settings']['headers'].copy()
         request_headers.update(extra_headers)
 
         # https://github.com/psf/requests/issues/4525
@@ -71,12 +92,10 @@ class perform_site_check():
         if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
             request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
 
-        timeout = self.datastore.data['settings']['requests'].get('timeout')
+        timeout = self.datastore.data['settings']['requests']['timeout']
-
+        url = self.datastore.get_val(uuid, 'url')
-        url = watch.link
+        request_body = self.datastore.get_val(uuid, 'body')
-
+        request_method = self.datastore.get_val(uuid, 'method')
-        request_body = self.datastore.data['watching'][uuid].get('body')
-        request_method = self.datastore.data['watching'][uuid].get('method')
         ignore_status_codes = self.datastore.data['watching'][uuid].get('ignore_status_codes', False)
 
         # source: support
@@ -86,37 +105,28 @@ class perform_site_check():
             is_source = True
 
         # Pluggable content fetcher
-        prefer_backend = watch.get('fetch_backend')
+        prefer_backend = watch['fetch_backend']
         if hasattr(content_fetcher, prefer_backend):
             klass = getattr(content_fetcher, prefer_backend)
         else:
             # If the klass doesnt exist, just use a default
             klass = getattr(content_fetcher, "html_requests")
 
-        proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
-        proxy_url = None
-        if proxy_id:
-            proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')
-            print("UUID {} Using proxy {}".format(uuid, proxy_url))
 
-        fetcher = klass(proxy_override=proxy_url)
+        proxy_args = self.set_proxy_from_list(watch)
+        fetcher = klass(proxy_override=proxy_args)
 
         # Configurable per-watch or global extra delay before extracting text (for webDriver types)
         system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)
         if watch['webdriver_delay'] is not None:
-            fetcher.render_extract_delay = watch.get('webdriver_delay')
+            fetcher.render_extract_delay = watch['webdriver_delay']
         elif system_webdriver_delay is not None:
             fetcher.render_extract_delay = system_webdriver_delay
 
-        # Possible conflict
+        if watch['webdriver_js_execute_code'] is not None and watch['webdriver_js_execute_code'].strip():
-        if prefer_backend == 'html_webdriver':
+            fetcher.webdriver_js_execute_code = watch['webdriver_js_execute_code']
-            fetcher.browser_steps = watch.get('browser_steps', None)
-            fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, uuid)
 
-        if watch.get('webdriver_js_execute_code') is not None and watch.get('webdriver_js_execute_code').strip():
+        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch['css_filter'])
-            fetcher.webdriver_js_execute_code = watch.get('webdriver_js_execute_code')
-
-        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'))
         fetcher.quit()
 
         self.screenshot = fetcher.screenshot
@@ -140,34 +150,27 @@ class perform_site_check():
             is_html = False
             is_json = False
 
-        include_filters_rule = deepcopy(watch.get('include_filters', []))
+        css_filter_rule = watch['css_filter']
-        # include_filters_rule = watch['include_filters']
         subtractive_selectors = watch.get(
             "subtractive_selectors", []
         ) + self.datastore.data["settings"]["application"].get(
             "global_subtractive_selectors", []
         )
 
-        # Inject a virtual LD+JSON price tracker rule
+        has_filter_rule = css_filter_rule and len(css_filter_rule.strip())
-        if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
-            include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
-
-        has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip())
         has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
 
         if is_json and not has_filter_rule:
-            include_filters_rule.append("json:$")
+            css_filter_rule = "json:$"
             has_filter_rule = True
 
         if has_filter_rule:
-            json_filter_prefixes = ['json:', 'jq:']
+            if 'json:' in css_filter_rule:
-            for filter in include_filters_rule:
+                stripped_text_from_html = html_tools.extract_json_as_string(content=fetcher.content, jsonpath_filter=css_filter_rule)
-                if any(prefix in filter for prefix in json_filter_prefixes):
+                is_html = False
-                    stripped_text_from_html += html_tools.extract_json_as_string(content=fetcher.content, json_filter=filter)
-                    is_html = False
 
         if is_html or is_source:
-
+            
             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
             fetcher.content = html_tools.workarounds_for_obfuscations(fetcher.content)
             html_content = fetcher.content
@@ -177,42 +180,35 @@ class perform_site_check():
                 # Don't run get_text or xpath/css filters on plaintext
                 stripped_text_from_html = html_content
             else:
-                # Does it have some ld+json price data? used for easier monitoring
-                update_obj['has_ldjson_price_data'] = html_tools.has_ldjson_product_info(fetcher.content)
-
                 # Then we assume HTML
                 if has_filter_rule:
-                    html_content = ""
+                    # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
-
+                    if css_filter_rule[0] == '/' or css_filter_rule.startswith('xpath:'):
-                    for filter_rule in include_filters_rule:
+                        html_content = html_tools.xpath_filter(xpath_filter=css_filter_rule.replace('xpath:', ''),
-                        # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
+                                                               html_content=fetcher.content)
-                        if filter_rule[0] == '/' or filter_rule.startswith('xpath:'):
+                    else:
-                            html_content += html_tools.xpath_filter(xpath_filter=filter_rule.replace('xpath:', ''),
+                        # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
-                                                                    html_content=fetcher.content,
+                        html_content = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
-                                                                    append_pretty_line_formatting=not is_source)
-                        else:
-                            # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
-                            html_content += html_tools.include_filters(include_filters=filter_rule,
-                                                                       html_content=fetcher.content,
-                                                                       append_pretty_line_formatting=not is_source)
-
-                    if not html_content.strip():
-                        raise FilterNotFoundInResponse(include_filters_rule)
 
                 if has_subtractive_selectors:
                     html_content = html_tools.element_removal(subtractive_selectors, html_content)
 
-                if is_source:
+                if not is_source:
-                    stripped_text_from_html = html_content
-                else:
                     # extract text
-                    do_anchor = self.datastore.data["settings"]["application"].get("render_anchor_tag_content", False)
                     stripped_text_from_html = \
                         html_tools.html_to_text(
                             html_content,
-                            render_anchor_tag_content=do_anchor
+                            render_anchor_tag_content=self.datastore.data["settings"][
+                                "application"].get(
+                                "render_anchor_tag_content", False)
                         )
 
+                elif is_source:
+                    stripped_text_from_html = html_content
+
+            # Re #340 - return the content before the 'ignore text' was applied
+            text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
+
         # Re #340 - return the content before the 'ignore text' was applied
         text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
@@ -245,7 +241,7 @@ class perform_site_check():
 
                 for l in result:
                     if type(l) is tuple:
-                        # @todo - some formatter option default (between groups)
+                        #@todo - some formatter option default (between groups)
                         regex_matched_output += list(l) + [b'\n']
                     else:
                         # @todo - some formatter option default (between each ungrouped result)
@@ -259,6 +255,7 @@ class perform_site_check():
                 stripped_text_from_html = b''.join(regex_matched_output)
                 text_content_before_ignored_filter = stripped_text_from_html
 
+
         # Re #133 - if we should strip whitespaces from triggering the change detected comparison
         if self.datastore.data['settings']['application'].get('ignore_whitespace', False):
             fetched_md5 = hashlib.md5(stripped_text_from_html.translate(None, b'\r\n\t ')).hexdigest()
@@ -268,30 +265,29 @@ class perform_site_check():
         ############ Blocking rules, after checksum #################
         blocked = False
 
-        trigger_text = watch.get('trigger_text', [])
+        if len(watch['trigger_text']):
-        if len(trigger_text):
             # Assume blocked
             blocked = True
             # Filter and trigger works the same, so reuse it
             # It should return the line numbers that match
             result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),
-                                                  wordlist=trigger_text,
+                                                  wordlist=watch['trigger_text'],
                                                   mode="line numbers")
             # Unblock if the trigger was found
             if result:
                 blocked = False
 
-        text_should_not_be_present = watch.get('text_should_not_be_present', [])
+
-        if len(text_should_not_be_present):
+        if len(watch['text_should_not_be_present']):
             # If anything matched, then we should block a change from happening
             result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),
-                                                  wordlist=text_should_not_be_present,
+                                                  wordlist=watch['text_should_not_be_present'],
                                                   mode="line numbers")
             if result:
                 blocked = True
 
         # The main thing that all this at the moment comes down to :)
-        if watch.get('previous_md5') != fetched_md5:
+        if watch['previous_md5'] != fetched_md5:
             changed_detected = True
 
         # Looks like something changed, but did it match all the rules?
@@ -300,7 +296,7 @@ class perform_site_check():
 
         # Extract title as title
         if is_html:
-            if self.datastore.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
+            if self.datastore.data['settings']['application']['extract_title_as_title'] or watch['extract_title_as_title']:
                 if not watch['title'] or not len(watch['title']):
                     update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
 

changedetectionio/forms.py

@@ -1,10 +1,11 @@
-import os
 import re
 
 from wtforms import (
     BooleanField,
+    Field,
     Form,
     IntegerField,
+    PasswordField,
     RadioField,
     SelectField,
     StringField,
@@ -12,17 +13,15 @@ from wtforms import (
     TextAreaField,
     fields,
     validators,
-    widgets
+    widgets,
 )
-from wtforms.fields import FieldList
 from wtforms.validators import ValidationError
 
-# default
-# each select <option data-enabled="enabled-0-0"
-from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
-
 from changedetectionio import content_fetcher
 from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
     valid_notification_formats,
 )
 
@@ -193,7 +192,7 @@ class ValidateAppRiseServers(object):
                 message = field.gettext('\'%s\' is not a valid AppRise URL.' % (server_url))
                 raise ValidationError(message)
 
-class ValidateJinja2Template(object):
+class ValidateTokensList(object):
     """
     Validates that a {token} is from a valid set
     """
@@ -202,27 +201,14 @@ class ValidateJinja2Template(object):
 
     def __call__(self, form, field):
         from changedetectionio import notification
-
+        regex = re.compile('{.*?}')
-        from jinja2 import Environment, BaseLoader, TemplateSyntaxError
+        for p in re.findall(regex, field.data):
-        from jinja2.meta import find_undeclared_variables
+            if not p.strip('{}') in notification.valid_tokens:
-
+                message = field.gettext('Token \'%s\' is not a valid token.')
-
+                raise ValidationError(message % (p))
-        try:
+            
-            jinja2_env = Environment(loader=BaseLoader)
-            jinja2_env.globals.update(notification.valid_tokens)
-            rendered = jinja2_env.from_string(field.data).render()
-        except TemplateSyntaxError as e:
-            raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
-
-        ast = jinja2_env.parse(field.data)
-        undefined = ", ".join(find_undeclared_variables(ast))
-        if undefined:
-            raise ValidationError(
-                f"The following tokens used in the notification are not valid: {undefined}"
-            )
-
 class validateURL(object):
-
+    
     """
        Flask wtform validators wont work with basic auth
     """
@@ -238,7 +224,6 @@ class validateURL(object):
             message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
             raise ValidationError(message)
 
-
 class ValidateListRegex(object):
     """
     Validates that anything that looks like a regex passes as a regex
@@ -318,25 +303,7 @@ class ValidateCSSJSONXPATHInput(object):
 
                 # Re #265 - maybe in the future fetch the page and offer a
                 # warning/notice that its possible the rule doesnt yet match anything?
-                if not self.allow_json:
-                    raise ValidationError("jq not permitted in this field!")
 
-            if 'jq:' in line:
-                try:
-                    import jq
-                except ModuleNotFoundError:
-                    # `jq` requires full compilation in windows and so isn't generally available
-                    raise ValidationError("jq not support not found")
-
-                input = line.replace('jq:', '')
-
-                try:
-                    jq.compile(input)
-                except (ValueError) as e:
-                    message = field.gettext('\'%s\' is not a valid jq expression. (%s)')
-                    raise ValidationError(message % (input, str(e)))
-                except:
-                    raise ValidationError("A system-error occurred when validating your jq expression")
 
 class quickWatchForm(Form):
     url = fields.URLField('URL', validators=[validateURL()])
@@ -347,25 +314,14 @@ class quickWatchForm(Form):
 
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
-    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
+
-    notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
+    notification_urls = StringListField('Notification URL list', validators=[validators.Optional(), ValidateNotificationBodyAndTitleWhenURLisSet(), ValidateAppRiseServers()])
-    notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
+    notification_title = StringField('Notification title', default=default_notification_title, validators=[validators.Optional(), ValidateTokensList()])
-    notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
+    notification_body = TextAreaField('Notification body', default=default_notification_body, validators=[validators.Optional(), ValidateTokensList()])
-    fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    notification_format = SelectField('Notification format', choices=valid_notification_formats.keys(), default=default_notification_format)
+    fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
-    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1,
+    webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1, message="Should contain one or more seconds")] )
-                                                                                                                                    message="Should contain one or more seconds")])
-
-class SingleBrowserStep(Form):
-
-    operation = SelectField('Operation', [validators.Optional()], choices=browser_step_ui_config.keys())
-
-    # maybe better to set some <script>var..
-    selector = StringField('Selector', [validators.Optional()], render_kw={"placeholder": "CSS or xPath selector"})
-    optional_value = StringField('value', [validators.Optional()], render_kw={"placeholder": "Value"})
-#   @todo move to JS? ajax fetch new field?
-#    remove_button = SubmitField('-', render_kw={"type": "button", "class": "pure-button pure-button-primary", 'title': 'Remove'})
-#    add_button = SubmitField('+', render_kw={"type": "button", "class": "pure-button pure-button-primary", 'title': 'Add new step after'})
 
 class watchForm(commonSettingsForm):
 
@@ -374,7 +330,7 @@ class watchForm(commonSettingsForm):
 
     time_between_check = FormField(TimeBetweenCheckForm)
 
-    include_filters = StringListField('CSS/JSONPath/JQ/XPath Filters', [ValidateCSSJSONXPATHInput()], default='')
+    css_filter = StringField('CSS/JSON/XPATH Filter', [ValidateCSSJSONXPATHInput()], default='')
 
     subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
 
@@ -389,9 +345,8 @@ class watchForm(commonSettingsForm):
     ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
     check_unique_lines = BooleanField('Only trigger when new lines appear', default=False)
     trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
-    if os.getenv("PLAYWRIGHT_DRIVER_URL"):
-        browser_steps = FieldList(FormField(SingleBrowserStep), min_entries=10)
     text_should_not_be_present = StringListField('Block change-detection if text matches', [validators.Optional(), ValidateListRegex()])
+
     webdriver_js_execute_code = TextAreaField('Execute JavaScript before change detection', render_kw={"rows": "5"}, validators=[validators.Optional()])
 
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
@@ -400,9 +355,6 @@ class watchForm(commonSettingsForm):
     filter_failure_notification_send = BooleanField(
         'Send a notification when the filter can no longer be found on the page', default=False)
 
-    notification_muted = BooleanField('Notifications Muted / Off', default=False)
-    notification_screenshot = BooleanField('Attach screenshot to notification (where possible)', default=False)
-
     def validate(self, **kwargs):
         if not super().validate():
             return False
@@ -414,15 +366,6 @@ class watchForm(commonSettingsForm):
             self.body.errors.append('Body must be empty when Request Method is set to GET')
             result = False
 
-        # Attempt to validate jinja2 templates in the URL
-        from jinja2 import Environment
-        # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
-        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
-        try:
-            ready_url = str(jinja2_env.from_string(self.url.data).render())
-        except Exception as e:
-            self.url.errors.append('Invalid template syntax')
-            result = False
         return result
 
 
@@ -462,8 +405,3 @@ class globalSettingsForm(Form):
     requests = FormField(globalSettingsRequestForm)
     application = FormField(globalSettingsApplicationForm)
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-
-
-class extractDataForm(Form):
-    extract_regex = StringField('RegEx to extract', validators=[validators.Length(min=1, message="Needs a RegEx")])
-    extract_submit_button = SubmitField('Extract as CSV', render_kw={"class": "pure-button pure-button-primary"})

changedetectionio/html_tools.py

@@ -1,40 +1,32 @@
+import json
+from typing import List
 
 from bs4 import BeautifulSoup
+from jsonpath_ng.ext import parse
+import re
 from inscriptis import get_text
 from inscriptis.model.config import ParserConfig
-from jsonpath_ng.ext import parse
-from typing import List
-import json
-import re
 
-# HTML added to be sure each result matching a filter (.example) gets converted to a new line by Inscriptis
+class FilterNotFoundInResponse(ValueError):
-TEXT_FILTER_LIST_LINE_SUFFIX = "<br/>"
+    def __init__(self, msg):
-
+        ValueError.__init__(self, msg)
-# 'price' , 'lowPrice', 'highPrice' are usually under here
-# all of those may or may not appear on different websites
-LD_JSON_PRODUCT_OFFER_SELECTOR = "json:$..offers"
 
 class JSONNotFound(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
-        
+
+
 # Given a CSS Rule, and a blob of HTML, return the blob of HTML that matches
-def include_filters(include_filters, html_content, append_pretty_line_formatting=False):
+def css_filter(css_filter, html_content):
     soup = BeautifulSoup(html_content, "html.parser")
     html_block = ""
-    r = soup.select(include_filters, separator="")
+    r = soup.select(css_filter, separator="")
+    if len(html_content) > 0 and len(r) == 0:
+        raise FilterNotFoundInResponse(css_filter)
+    for item in r:
+        html_block += str(item)
 
-    for element in r:
+    return html_block + "\n"
-        # When there's more than 1 match, then add the suffix to separate each line
-        # And where the matched result doesn't include something that will cause Inscriptis to add a newline
-        # (This way each 'match' reliably has a new-line in the diff)
-        # Divs are converted to 4 whitespaces by inscriptis
-        if append_pretty_line_formatting and len(html_block) and not element.name in (['br', 'hr', 'div', 'p']):
-            html_block += TEXT_FILTER_LIST_LINE_SUFFIX
-
-        html_block += str(element)
-
-    return html_block
 
 def subtractive_css_selector(css_selector, html_content):
     soup = BeautifulSoup(html_content, "html.parser")
@@ -50,29 +42,25 @@ def element_removal(selectors: List[str], html_content):
 
 
 # Return str Utf-8 of matched rules
-def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False):
+def xpath_filter(xpath_filter, html_content):
     from lxml import etree, html
 
     tree = html.fromstring(bytes(html_content, encoding='utf-8'))
     html_block = ""
 
     r = tree.xpath(xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'})
+    if len(html_content) > 0 and len(r) == 0:
+        raise FilterNotFoundInResponse(xpath_filter)
+
     #@note: //title/text() wont work where <title>CDATA..
 
     for element in r:
-        # When there's more than 1 match, then add the suffix to separate each line
-        # And where the matched result doesn't include something that will cause Inscriptis to add a newline
-        # (This way each 'match' reliably has a new-line in the diff)
-        # Divs are converted to 4 whitespaces by inscriptis
-        if append_pretty_line_formatting and len(html_block) and (not hasattr( element, 'tag' ) or not element.tag in (['br', 'hr', 'div', 'p'])):
-            html_block += TEXT_FILTER_LIST_LINE_SUFFIX
-
         if type(element) == etree._ElementStringResult:
-            html_block += str(element)
+            html_block += str(element) + "<br/>"
         elif type(element) == etree._ElementUnicodeResult:
-            html_block += str(element)
+            html_block += str(element) + "<br/>"
         else:
-            html_block += etree.tostring(element, pretty_print=True).decode('utf-8')
+            html_block += etree.tostring(element, pretty_print=True).decode('utf-8') + "<br/>"
 
     return html_block
 
@@ -91,35 +79,19 @@ def extract_element(find='title', html_content=''):
     return element_text
 
 #
-def _parse_json(json_data, json_filter):
+def _parse_json(json_data, jsonpath_filter):
-    if 'json:' in json_filter:
+    s=[]
-        jsonpath_expression = parse(json_filter.replace('json:', ''))
+    jsonpath_expression = parse(jsonpath_filter.replace('json:', ''))
-        match = jsonpath_expression.find(json_data)
+    match = jsonpath_expression.find(json_data)
-        return _get_stripped_text_from_json_match(match)
 
-    if 'jq:' in json_filter:
-
-        try:
-            import jq
-        except ModuleNotFoundError:
-            # `jq` requires full compilation in windows and so isn't generally available
-            raise Exception("jq not support not found")
-
-        jq_expression = jq.compile(json_filter.replace('jq:', ''))
-        match = jq_expression.input(json_data).all()
-
-        return _get_stripped_text_from_json_match(match)
-
-def _get_stripped_text_from_json_match(match):
-    s = []
     # More than one result, we will return it as a JSON list.
     if len(match) > 1:
         for i in match:
-            s.append(i.value if hasattr(i, 'value') else i)
+            s.append(i.value)
 
     # Single value, use just the value, as it could be later used in a token in notifications.
     if len(match) == 1:
-        s = match[0].value if hasattr(match[0], 'value') else match[0]
+        s = match[0].value
 
     # Re #257 - Better handling where it does not exist, in the case the original 's' value was False..
     if not match:
@@ -131,26 +103,19 @@ def _get_stripped_text_from_json_match(match):
 
     return stripped_text_from_html
 
-# content - json
+def extract_json_as_string(content, jsonpath_filter):
-# json_filter - ie json:$..price
+
-# ensure_is_ldjson_info_type - str "product", optional, "@type == product" (I dont know how to do that as a json selector)
-def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None):
     stripped_text_from_html = False
 
     # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded <script type=ldjson>
     try:
-        stripped_text_from_html = _parse_json(json.loads(content), json_filter)
+        stripped_text_from_html = _parse_json(json.loads(content), jsonpath_filter)
     except json.JSONDecodeError:
 
-        # Foreach <script json></script> blob.. just return the first that matches json_filter
+        # Foreach <script json></script> blob.. just return the first that matches jsonpath_filter
         s = []
         soup = BeautifulSoup(content, 'html.parser')
-
+        bs_result = soup.findAll('script')
-        if ensure_is_ldjson_info_type:
-            bs_result = soup.findAll('script', {"type": "application/ld+json"})
-        else:
-            bs_result = soup.findAll('script')
-
 
         if not bs_result:
             raise JSONNotFound("No parsable JSON found in this document")
@@ -166,15 +131,8 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
                 # Just skip it
                 continue
             else:
-                stripped_text_from_html = _parse_json(json_data, json_filter)
+                stripped_text_from_html = _parse_json(json_data, jsonpath_filter)
-                if ensure_is_ldjson_info_type:
+                if stripped_text_from_html:
-                    # Could sometimes be list, string or something else random
-                    if isinstance(json_data, dict):
-                        # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
-                        # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
-                        if json_data.get('@type', False) and json_data.get('@type','').lower() == ensure_is_ldjson_info_type.lower() and stripped_text_from_html:
-                            break
-                elif stripped_text_from_html:
                     break
 
     if not stripped_text_from_html:
@@ -261,18 +219,6 @@ def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
 
     return text_content
 
-
-# Does LD+JSON exist with a @type=='product' and a .price set anywhere?
-def has_ldjson_product_info(content):
-    try:
-        pricing_data = extract_json_as_string(content=content, json_filter=LD_JSON_PRODUCT_OFFER_SELECTOR, ensure_is_ldjson_info_type="product")
-    except JSONNotFound as e:
-        # Totally fine
-        return False
-    x=bool(pricing_data)
-    return x
-
-
 def workarounds_for_obfuscations(content):
     """
     Some sites are using sneaky tactics to make prices and other information un-renderable by Inscriptis

changedetectionio/importer.py

@@ -103,12 +103,12 @@ class import_distill_io_json(Importer):
                     pass
                 except IndexError:
                     pass
-                extras['include_filters'] = []
+
                 try:
+                    extras['css_filter'] = d_config['selections'][0]['frames'][0]['includes'][0]['expr']
                     if d_config['selections'][0]['frames'][0]['includes'][0]['type'] == 'xpath':
-                        extras['include_filters'].append('xpath:' + d_config['selections'][0]['frames'][0]['includes'][0]['expr'])
+                        extras['css_filter'] = 'xpath:' + extras['css_filter']
-                    else:
+
-                        extras['include_filters'].append(d_config['selections'][0]['frames'][0]['includes'][0]['expr'])
                 except KeyError:
                     pass
                 except IndexError:

changedetectionio/model/App.py

@@ -13,6 +13,10 @@ class model(dict):
             'watching': {},
             'settings': {
                 'headers': {
+                    'User-Agent': getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT", 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'),
+                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+                    'Accept-Encoding': 'gzip, deflate',  # No support for brolti in python requests yet.
+                    'Accept-Language': 'en-GB,en-US;q=0.9,en;'
                 },
                 'requests': {
                     'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")),  # Default 45 seconds
@@ -27,7 +31,6 @@ class model(dict):
                     'base_url' : None,
                     'extract_title_as_title': False,
                     'empty_pages_are_a_change': False,
-                    'css_dark_mode': False,
                     'fetch_backend': getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
                     'filter_failure_notification_threshold_attempts': _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT,
                     'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum

changedetectionio/model/Watch.py

@@ -1,14 +1,14 @@
-from distutils.util import strtobool
-import logging
 import os
-import time
+import uuid as uuid_builder
-import uuid
+from distutils.util import strtobool
 
 minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
 mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
 
 from changedetectionio.notification import (
-    default_notification_format_for_watch
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
 )
 
 
@@ -16,46 +16,42 @@ class model(dict):
     __newest_history_key = None
     __history_n=0
     __base_config = {
-            #'history': {},  # Dict of timestamp and output stripped filename (removed)
+            'url': None,
-            #'newest_history_key': 0, (removed, taken from history.txt index)
+            'tag': None,
+            'last_checked': 0,
+            'paused': False,
+            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
+            #'newest_history_key': 0,
+            'title': None,
+            'previous_md5': False,
+            'uuid': str(uuid_builder.uuid4()),
+            'headers': {},  # Extra headers to send
             'body': None,
-            'check_unique_lines': False, # On change-detected, compare against all history if its something new
+            'method': 'GET',
-            'check_count': 0,
+            #'history': {},  # Dict of timestamp and output stripped filename
-            'consecutive_filter_failures': 0, # Every time the CSS/xPath filter cannot be located, reset when all is fine.
+            'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
+            # Custom notification content
+            'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
+            'notification_title': default_notification_title,
+            'notification_body': default_notification_body,
+            'notification_format': default_notification_format,
+            'notification_muted': False,
+            'css_filter': '',
+            'last_error': False,
             'extract_text': [],  # Extract text by regex after filters
-            'extract_title_as_title': False,
+            'subtractive_selectors': [],
+            'trigger_text': [],  # List of text or regex to wait for until a change is detected
+            'text_should_not_be_present': [], # Text that should not present
             'fetch_backend': None,
             'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
-            'has_ldjson_price_data': None,
+            'consecutive_filter_failures': 0, # Every time the CSS/xPath filter cannot be located, reset when all is fine.
-            'track_ldjson_price_data': None,
+            'extract_title_as_title': False,
-            'headers': {},  # Extra headers to send
+            'check_unique_lines': False, # On change-detected, compare against all history if its something new
-            'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
-            'include_filters': [],
-            'last_checked': 0,
-            'last_error': False,
-            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
-            'method': 'GET',
-             # Custom notification content
-            'notification_body': None,
-            'notification_format': default_notification_format_for_watch,
-            'notification_muted': False,
-            'notification_title': None,
-            'notification_screenshot': False, # Include the latest screenshot if available and supported by the apprise URL
-            'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
-            'paused': False,
-            'previous_md5': False,
             'proxy': None, # Preferred proxy connection
-            'subtractive_selectors': [],
-            'tag': None,
-            'text_should_not_be_present': [], # Text that should not present
             # Re #110, so then if this is set to None, we know to use the default value instead
             # Requires setting to None on submit if it's the same as the default
             # Should be all None by default, so we use the system default in this case.
             'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
-            'title': None,
-            'trigger_text': [],  # List of text or regex to wait for until a change is detected
-            'url': None,
-            'uuid': str(uuid.uuid4()),
             'webdriver_delay': None,
             'webdriver_js_execute_code': None, # Run before change-detection
         }
@@ -66,7 +62,7 @@ class model(dict):
         self.update(self.__base_config)
         self.__datastore_path = kw['datastore_path']
 
-        self['uuid'] = str(uuid.uuid4())
+        self['uuid'] = str(uuid_builder.uuid4())
 
         del kw['datastore_path']
 
@@ -88,30 +84,10 @@ class model(dict):
         return False
 
     def ensure_data_dir_exists(self):
-        if not os.path.isdir(self.watch_data_dir):
+        target_path = os.path.join(self.__datastore_path, self['uuid'])
-            print ("> Creating data dir {}".format(self.watch_data_dir))
+        if not os.path.isdir(target_path):
-            os.mkdir(self.watch_data_dir)
+            print ("> Creating data dir {}".format(target_path))
-
+            os.mkdir(target_path)
-    @property
-    def link(self):
-        url = self.get('url', '')
-        ready_url = url
-        if '{%' in url or '{{' in url:
-            from jinja2 import Environment
-            # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
-            jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
-            try:
-                ready_url = str(jinja2_env.from_string(url).render())
-            except Exception as e:
-                from flask import (
-                    flash, Markup, url_for
-                )
-                message = Markup('<a href="{}#general">The URL {} is invalid and cannot be used, click to edit</a>'.format(
-                    url_for('edit_page', uuid=self.get('uuid')), self.get('url', '')))
-                flash(message, 'error')
-                return ''
-
-        return ready_url
 
     @property
     def label(self):
@@ -135,40 +111,16 @@ class model(dict):
 
     @property
     def history(self):
-        """History index is just a text file as a list
-            {watch-uuid}/history.txt
-
-            contains a list like
-
-            {epoch-time},{filename}\n
-
-            We read in this list as the history information
-
-        """
         tmp_history = {}
+        import logging
+        import time
 
         # Read the history file as a dict
-        fname = os.path.join(self.watch_data_dir, "history.txt")
+        fname = os.path.join(self.__datastore_path, self.get('uuid'), "history.txt")
         if os.path.isfile(fname):
             logging.debug("Reading history index " + str(time.time()))
             with open(fname, "r") as f:
-                for i in f.readlines():
+                tmp_history = dict(i.strip().split(',', 2) for i in f.readlines())
-                    if ',' in i:
-                        k, v = i.strip().split(',', 2)
-
-                        # The index history could contain a relative path, so we need to make the fullpath
-                        # so that python can read it
-                        if not '/' in v and not '\'' in v:
-                            v = os.path.join(self.watch_data_dir, v)
-                        else:
-                            # It's possible that they moved the datadir on older versions
-                            # So the snapshot exists but is in a different path
-                            snapshot_fname = v.split('/')[-1]
-                            proposed_new_path = os.path.join(self.watch_data_dir, snapshot_fname)
-                            if not os.path.exists(v) and os.path.exists(proposed_new_path):
-                                v = proposed_new_path
-
-                        tmp_history[k] = v
 
         if len(tmp_history):
             self.__newest_history_key = list(tmp_history.keys())[-1]
@@ -179,7 +131,7 @@ class model(dict):
 
     @property
     def has_history(self):
-        fname = os.path.join(self.watch_data_dir, "history.txt")
+        fname = os.path.join(self.__datastore_path, self.get('uuid'), "history.txt")
         return os.path.isfile(fname)
 
     # Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0.
@@ -198,33 +150,31 @@ class model(dict):
     # Save some text file to the appropriate path and bump the history
     # result_obj from fetch_site_status.run()
     def save_history_text(self, contents, timestamp):
+        import uuid
+        import logging
+
+        output_path = "{}/{}".format(self.__datastore_path, self['uuid'])
 
         self.ensure_data_dir_exists()
 
-        # Small hack so that we sleep just enough to allow 1 second  between history snapshots
+        snapshot_fname = "{}/{}.stripped.txt".format(output_path, uuid.uuid4())
-        # this is because history.txt indexes/keys snapshots by epoch seconds and we dont want dupe keys
+        logging.debug("Saving history text {}".format(snapshot_fname))
-        if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
-            time.sleep(timestamp - self.__newest_history_key)
 
-        snapshot_fname = "{}.txt".format(str(uuid.uuid4()))
+        with open(snapshot_fname, 'wb') as f:
-
-        # in /diff/ and /preview/ we are going to assume for now that it's UTF-8 when reading
-        # most sites are utf-8 and some are even broken utf-8
-        with open(os.path.join(self.watch_data_dir, snapshot_fname), 'wb') as f:
             f.write(contents)
             f.close()
 
         # Append to index
         # @todo check last char was \n
-        index_fname = os.path.join(self.watch_data_dir, "history.txt")
+        index_fname = "{}/history.txt".format(output_path)
         with open(index_fname, 'a') as f:
             f.write("{},{}\n".format(timestamp, snapshot_fname))
             f.close()
 
         self.__newest_history_key = timestamp
-        self.__history_n += 1
+        self.__history_n+=1
 
-        # @todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status
+        #@todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status
         return snapshot_fname
 
     @property
@@ -257,26 +207,14 @@ class model(dict):
         return not local_lines.issubset(existing_history)
 
     def get_screenshot(self):
-        fname = os.path.join(self.watch_data_dir, "last-screenshot.png")
+        fname = os.path.join(self.__datastore_path, self['uuid'], "last-screenshot.png")
         if os.path.isfile(fname):
             return fname
 
-        # False is not an option for AppRise, must be type None
+        return False
-        return None
-
-    def get_screenshot_as_jpeg(self):
-
-        # Created by save_screenshot()
-        fname = os.path.join(self.watch_data_dir, "last-screenshot.jpg")
-        if os.path.isfile(fname):
-            return fname
-
-        # False is not an option for AppRise, must be type None
-        return None
-
 
     def __get_file_ctime(self, filename):
-        fname = os.path.join(self.watch_data_dir, filename)
+        fname = os.path.join(self.__datastore_path, self['uuid'], filename)
         if os.path.isfile(fname):
             return int(os.path.getmtime(fname))
         return False
@@ -301,14 +239,9 @@ class model(dict):
     def snapshot_error_screenshot_ctime(self):
         return self.__get_file_ctime('last-error-screenshot.png')
 
-    @property
-    def watch_data_dir(self):
-        # The base dir of the watch data
-        return os.path.join(self.__datastore_path, self['uuid'])
-    
     def get_error_text(self):
         """Return the text saved from a previous request that resulted in a non-200 error"""
-        fname = os.path.join(self.watch_data_dir, "last-error.txt")
+        fname = os.path.join(self.__datastore_path, self['uuid'], "last-error.txt")
         if os.path.isfile(fname):
             with open(fname, 'r') as f:
                 return f.read()
@@ -316,51 +249,7 @@ class model(dict):
 
     def get_error_snapshot(self):
         """Return path to the screenshot that resulted in a non-200 error"""
-        fname = os.path.join(self.watch_data_dir, "last-error-screenshot.png")
+        fname = os.path.join(self.__datastore_path, self['uuid'], "last-error-screenshot.png")
         if os.path.isfile(fname):
             return fname
         return False
-
-    def extract_regex_from_all_history(self, regex):
-        import csv
-        import re
-        import datetime
-        csv_output_filename = False
-        csv_writer = False
-        f = None
-
-        # self.history will be keyed with the full path
-        for k, fname in self.history.items():
-            if os.path.isfile(fname):
-                with open(fname, "r") as f:
-                    contents = f.read()
-                    res = re.findall(regex, contents, re.MULTILINE)
-                    if res:
-                        if not csv_writer:
-                            # A file on the disk can be transferred much faster via flask than a string reply
-                            csv_output_filename = 'report.csv'
-                            f = open(os.path.join(self.watch_data_dir, csv_output_filename), 'w')
-                            # @todo some headers in the future
-                            #fieldnames = ['Epoch seconds', 'Date']
-                            csv_writer = csv.writer(f,
-                                                    delimiter=',',
-                                                    quotechar='"',
-                                                    quoting=csv.QUOTE_MINIMAL,
-                                                    #fieldnames=fieldnames
-                                                    )
-                            csv_writer.writerow(['Epoch seconds', 'Date'])
-                            # csv_writer.writeheader()
-
-                        date_str = datetime.datetime.fromtimestamp(int(k)).strftime('%Y-%m-%d %H:%M:%S')
-                        for r in res:
-                            row = [k, date_str]
-                            if isinstance(r, str):
-                                row.append(r)
-                            else:
-                                row+=r
-                            csv_writer.writerow(row)
-
-        if f:
-            f.close()
-
-        return csv_output_filename

changedetectionio/notification.py

@@ -1,7 +1,5 @@
 import apprise
-from jinja2 import Environment, BaseLoader
 from apprise import NotifyFormat
-import json
 
 valid_tokens = {
     'base_url': '',
@@ -16,80 +14,35 @@ valid_tokens = {
     'current_snapshot': ''
 }
 
-default_notification_format_for_watch = 'System default'
-default_notification_format = 'Text'
-default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
-default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
-
 valid_notification_formats = {
     'Text': NotifyFormat.TEXT,
     'Markdown': NotifyFormat.MARKDOWN,
     'HTML': NotifyFormat.HTML,
-    # Used only for editing a watch (not for global)
-    default_notification_format_for_watch: default_notification_format_for_watch
 }
 
-# include the decorator
+default_notification_format = 'Text'
-from apprise.decorators import notify
+default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n'
-
+default_notification_title = 'ChangeDetection.io Notification - {watch_url}'
-@notify(on="delete")
-@notify(on="deletes")
-@notify(on="get")
-@notify(on="gets")
-@notify(on="post")
-@notify(on="posts")
-@notify(on="put")
-@notify(on="puts")
-def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
-    import requests
-    url = kwargs['meta'].get('url')
-
-    if url.startswith('post'):
-        r = requests.post
-    elif url.startswith('get'):
-        r = requests.get
-    elif url.startswith('put'):
-        r = requests.put
-    elif url.startswith('delete'):
-        r = requests.delete
-
-    url = url.replace('post://', 'http://')
-    url = url.replace('posts://', 'https://')
-    url = url.replace('put://', 'http://')
-    url = url.replace('puts://', 'https://')
-    url = url.replace('get://', 'http://')
-    url = url.replace('gets://', 'https://')
-    url = url.replace('put://', 'http://')
-    url = url.replace('puts://', 'https://')
-    url = url.replace('delete://', 'http://')
-    url = url.replace('deletes://', 'https://')
-
-    # Try to auto-guess if it's JSON
-    headers = {}
-    try:
-        json.loads(body)
-        headers = {'Content-Type': 'application/json; charset=utf-8'}
-    except ValueError as e:
-        pass
-
-
-    r(url, headers=headers, data=body)
-
 
 def process_notification(n_object, datastore):
 
-    # Insert variables into the notification content
-    notification_parameters = create_notification_parameters(n_object, datastore)
-
     # Get the notification body from datastore
-    jinja2_env = Environment(loader=BaseLoader)
+    n_body = n_object.get('notification_body', default_notification_body)
-    n_body = jinja2_env.from_string(n_object.get('notification_body', default_notification_body)).render(**notification_parameters)
+    n_title = n_object.get('notification_title', default_notification_title)
-    n_title = jinja2_env.from_string(n_object.get('notification_title', default_notification_title)).render(**notification_parameters)
     n_format = valid_notification_formats.get(
         n_object['notification_format'],
         valid_notification_formats[default_notification_format],
     )
-    
+
+    # Insert variables into the notification content
+    notification_parameters = create_notification_parameters(n_object, datastore)
+
+    for n_k in notification_parameters:
+        token = '{' + n_k + '}'
+        val = notification_parameters[n_k]
+        n_title = n_title.replace(token, val)
+        n_body = n_body.replace(token, val)
+
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
     # raise it as an exception
@@ -97,7 +50,6 @@ def process_notification(n_object, datastore):
     sent_objs=[]
     from .apprise_asset import asset
     for url in n_object['notification_urls']:
-        url = jinja2_env.from_string(url).render(**notification_parameters)
         apobj = apprise.Apprise(debug=True, asset=asset)
         url = url.strip()
         if len(url):
@@ -111,12 +63,7 @@ def process_notification(n_object, datastore):
 
                 # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
                 k = '?' if not '?' in url else '&'
-                if not 'avatar_url' in url \
+                if not 'avatar_url' in url and not url.startswith('mail'):
-                        and not url.startswith('mail') \
-                        and not url.startswith('post') \
-                        and not url.startswith('get') \
-                        and not url.startswith('delete') \
-                        and not url.startswith('put'):
                     url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
 
                 if url.startswith('tgram://'):
@@ -151,10 +98,7 @@ def process_notification(n_object, datastore):
                 apobj.notify(
                     title=n_title,
                     body=n_body,
-                    body_format=n_format,
+                    body_format=n_format)
-                    # False is not an option for AppRise, must be type None
-                    attach=n_object.get('screenshot', None)
-                )
 
                 apobj.clear()
 
@@ -194,7 +138,7 @@ def create_notification_parameters(n_object, datastore):
 
     watch_url = n_object['watch_url']
 
-    # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services
+    # Re #148 - Some people have just {base_url} in the body or title, but this may break some notification services
     #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.
     if base_url == '':
         base_url = "<base-url-env-var-not-set>"

changedetectionio/res/xpath_element_scraper.js

@@ -1,196 +0,0 @@
-// @file Scrape the page looking for elements of concern (%ELEMENTS%)
-// http://matatk.agrip.org.uk/tests/position-and-width/
-// https://stackoverflow.com/questions/26813480/when-is-element-getboundingclientrect-guaranteed-to-be-updated-accurate
-//
-// Some pages like https://www.londonstockexchange.com/stock/NCCL/ncondezi-energy-limited/analysis
-// will automatically force a scroll somewhere, so include the position offset
-// Lets hope the position doesnt change while we iterate the bbox's, but this is better than nothing
-
-var scroll_y=+document.documentElement.scrollTop || document.body.scrollTop
-
-// Include the getXpath script directly, easier than fetching
-function getxpath(e) {
-        var n = e;
-        if (n && n.id) return '//*[@id="' + n.id + '"]';
-        for (var o = []; n && Node.ELEMENT_NODE === n.nodeType;) {
-            for (var i = 0, r = !1, d = n.previousSibling; d;) d.nodeType !== Node.DOCUMENT_TYPE_NODE && d.nodeName === n.nodeName && i++, d = d.previousSibling;
-            for (d = n.nextSibling; d;) {
-                if (d.nodeName === n.nodeName) {
-                    r = !0;
-                    break
-                }
-                d = d.nextSibling
-            }
-            o.push((n.prefix ? n.prefix + ":" : "") + n.localName + (i || r ? "[" + (i + 1) + "]" : "")), n = n.parentNode
-        }
-        return o.length ? "/" + o.reverse().join("/") : ""
-    }
-
-const findUpTag = (el) => {
-    let r = el
-    chained_css = [];
-    depth = 0;
-
-    //  Strategy 1: If it's an input, with name, and there's only one, prefer that
-    if (el.name !== undefined && el.name.length) {
-        var proposed = el.tagName + "[name=" + el.name + "]";
-        var proposed_element = window.document.querySelectorAll(proposed);
-        if(proposed_element.length) {
-            if (proposed_element.length === 1) {
-                return proposed;
-            } else {
-                // Some sites change ID but name= stays the same, we can hit it if we know the index
-                // Find all the elements that match and work out the input[n]
-                var n=Array.from(proposed_element).indexOf(el);
-                // Return a Playwright selector for nthinput[name=zipcode]
-                return proposed+" >> nth="+n;
-            }
-        }
-    }
-
-    // Strategy 2: Keep going up until we hit an ID tag, imagine it's like  #list-widget div h4
-    while (r.parentNode) {
-        if (depth == 5) {
-            break;
-        }
-        if ('' !== r.id) {
-            chained_css.unshift("#" + CSS.escape(r.id));
-            final_selector = chained_css.join(' > ');
-            // Be sure theres only one, some sites have multiples of the same ID tag :-(
-            if (window.document.querySelectorAll(final_selector).length == 1) {
-                return final_selector;
-            }
-            return null;
-        } else {
-            chained_css.unshift(r.tagName.toLowerCase());
-        }
-        r = r.parentNode;
-        depth += 1;
-    }
-    return null;
-}
-
-
-// @todo - if it's SVG or IMG, go into image diff mode
-// %ELEMENTS% replaced at injection time because different interfaces use it with different settings
-var elements = window.document.querySelectorAll("%ELEMENTS%");
-var size_pos = [];
-// after page fetch, inject this JS
-// build a map of all elements and their positions (maybe that only include text?)
-var bbox;
-for (var i = 0; i < elements.length; i++) {
-    bbox = elements[i].getBoundingClientRect();
-
-    // Exclude items that are not interactable or visible
-    if(elements[i].style.opacity === "0") {
-        continue
-    }
-    if(elements[i].style.display === "none" || elements[i].style.pointerEvents === "none" ) {
-        continue
-    }
-
-    // Forget really small ones
-    if (bbox['width'] < 10 && bbox['height'] < 10) {
-        continue;
-    }
-
-    // Don't include elements that are offset from canvas
-    if (bbox['top']+scroll_y < 0 || bbox['left'] < 0) {
-        continue;
-    }
-
-    // @todo the getXpath kind of sucks, it doesnt know when there is for example just one ID sometimes
-    // it should not traverse when we know we can anchor off just an ID one level up etc..
-    // maybe, get current class or id, keep traversing up looking for only class or id until there is just one match
-
-    // 1st primitive - if it has class, try joining it all and select, if theres only one.. well thats us.
-    xpath_result = false;
-
-    try {
-        var d = findUpTag(elements[i]);
-        if (d) {
-            xpath_result = d;
-        }
-    } catch (e) {
-        console.log(e);
-    }
-
-    // You could swap it and default to getXpath and then try the smarter one
-    // default back to the less intelligent one
-    if (!xpath_result) {
-        try {
-            // I've seen on FB and eBay that this doesnt work
-            // ReferenceError: getXPath is not defined at eval (eval at evaluate (:152:29), <anonymous>:67:20) at UtilityScript.evaluate (<anonymous>:159:18) at UtilityScript.<anonymous> (<anonymous>:1:44)
-            xpath_result = getxpath(elements[i]);
-        } catch (e) {
-            console.log(e);
-            continue;
-        }
-    }
-
-    if (window.getComputedStyle(elements[i]).visibility === "hidden") {
-        continue;
-    }
-
-    // @todo Possible to ONLY list where it's clickable to save JSON xfer size
-    size_pos.push({
-        xpath: xpath_result,
-        width: Math.round(bbox['width']),
-        height: Math.round(bbox['height']),
-        left: Math.floor(bbox['left']),
-        top: Math.floor(bbox['top'])+scroll_y,
-        tagName: (elements[i].tagName) ? elements[i].tagName.toLowerCase() : '',
-        tagtype: (elements[i].tagName == 'INPUT' && elements[i].type) ? elements[i].type.toLowerCase() : '',
-        isClickable: (elements[i].onclick) || window.getComputedStyle(elements[i]).cursor == "pointer"
-    });
-
-}
-
-
-// Inject the current one set in the include_filters, which may be a CSS rule
-// used for displaying the current one in VisualSelector, where its not one we generated.
-if (include_filters.length) {
-    // Foreach filter, go and find it on the page and add it to the results so we can visualise it again
-    for (const f of include_filters) {
-        bbox = false;
-        q = false;
-
-        if (!f.length) {
-            console.log("xpath_element_scraper: Empty filter, skipping");
-            continue;
-        }
-
-        try {
-            // is it xpath?
-            if (f.startsWith('/') || f.startsWith('xpath:')) {
-                q = document.evaluate(f.replace('xpath:', ''), document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
-            } else {
-                q = document.querySelector(f);
-            }
-        } catch (e) {
-            // Maybe catch DOMException and alert?
-            console.log("xpath_element_scraper: Exception selecting element from filter "+f);
-            console.log(e);
-        }
-
-        if (q) {
-            bbox = q.getBoundingClientRect();
-            console.log("xpath_element_scraper: Got filter element, scroll from top was "+scroll_y)
-        } else {
-            console.log("xpath_element_scraper: filter element "+f+" was not found");
-        }
-
-        if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
-            size_pos.push({
-                xpath: f,
-                width: parseInt(bbox['width']),
-                height: parseInt(bbox['height']),
-                left: parseInt(bbox['left']),
-                top: parseInt(bbox['top'])+scroll_y
-            });
-        }
-    }
-}
-
-// Window.width required for proper scaling in the frontend
-return {'size_pos': size_pos, 'browser_width': window.innerWidth};

changedetectionio/run_all_tests.sh

@@ -9,8 +9,6 @@
 # exit when any command fails
 set -e
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
 find tests/test_*py -type f|while read test_name
 do
   echo "TEST RUNNING $test_name"
@@ -25,11 +23,6 @@ export BASE_URL="https://really-unique-domain.io"
 pytest tests/test_notification.py
 
 
-# Re-run with HIDE_REFERER set - could affect login
-export HIDE_REFERER=True
-pytest tests/test_access_control.py
-
-
 # Now for the selenium and playwright/browserless fetchers
 # Note - this is not UI functional tests - just checking that each one can fetch the content
 
@@ -45,9 +38,7 @@ docker kill $$-test_selenium
 
 echo "TESTING WEBDRIVER FETCH > PLAYWRIGHT/BROWSERLESS..."
 # Not all platforms support playwright (not ARM/rPI), so it's not packaged in requirements.txt
-PLAYWRIGHT_VERSION=$(grep -i -E "RUN pip install.+" "$SCRIPT_DIR/../Dockerfile" | grep --only-matching -i -E "playwright[=><~+]+[0-9\.]+")
+pip3 install playwright~=1.24
-echo "using $PLAYWRIGHT_VERSION"
-pip3 install "$PLAYWRIGHT_VERSION"
 docker run -d --name $$-test_browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
 # takes a while to spin up
 sleep 5
@@ -57,48 +48,4 @@ pytest tests/test_errorhandling.py
 pytest tests/visualselector/test_fetch_data.py
 
 unset PLAYWRIGHT_DRIVER_URL
-docker kill $$-test_browserless
+docker kill $$-test_browserless
-
-# Test proxy list handling, starting two squids on different ports
-# Each squid adds a different header to the response, which is the main thing we test for.
-docker run -d --name $$-squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf -p 3128:3128 ubuntu/squid:4.13-21.10_edge
-docker run -d --name $$-squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf -p 3129:3128 ubuntu/squid:4.13-21.10_edge
-
-
-# So, basic HTTP as env var test
-export HTTP_PROXY=http://localhost:3128
-export HTTPS_PROXY=http://localhost:3128
-pytest tests/proxy_list/test_proxy.py
-docker logs $$-squid-one 2>/dev/null|grep one.changedetection.io
-if [ $? -ne 0 ]
-then
-  echo "Did not see a request to one.changedetection.io in the squid logs (while checking env vars HTTP_PROXY/HTTPS_PROXY)"
-fi
-unset HTTP_PROXY
-unset HTTPS_PROXY
-
-
-# 2nd test actually choose the preferred proxy from proxies.json
-cp tests/proxy_list/proxies.json-example ./test-datastore/proxies.json
-# Makes a watch use a preferred proxy
-pytest tests/proxy_list/test_multiple_proxy.py
-
-# Should be a request in the default "first" squid
-docker logs $$-squid-one 2>/dev/null|grep chosen.changedetection.io
-if [ $? -ne 0 ]
-then
-  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy)"
-fi
-
-# And one in the 'second' squid (user selects this as preferred)
-docker logs $$-squid-two 2>/dev/null|grep chosen.changedetection.io
-if [ $? -ne 0 ]
-then
-  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy)"
-fi
-
-# @todo - test system override proxy selection and watch defaults, setup a 3rd squid?
-docker kill $$-squid-one
-docker kill $$-squid-two
-
-

changedetectionio/static/favicons/browserconfig.xml

@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<browserconfig>
-    <msapplication>
-        <tile>
-            <square150x150logo src="favicons/mstile-150x150.png"/>
-            <TileColor>#da532c</TileColor>
-        </tile>
-    </msapplication>
-</browserconfig>

changedetectionio/static/favicons/safari-pinned-tab.svg

@@ -1,35 +0,0 @@
-<?xml version="1.0" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN"
- "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
-<svg version="1.0" xmlns="http://www.w3.org/2000/svg"
- width="256.000000pt" height="256.000000pt" viewBox="0 0 256.000000 256.000000"
- preserveAspectRatio="xMidYMid meet">
-<metadata>
-Created by potrace 1.14, written by Peter Selinger 2001-2017
-</metadata>
-<g transform="translate(0.000000,256.000000) scale(0.100000,-0.100000)"
-fill="#000000" stroke="none">
-<path d="M0 1280 l0 -1280 1280 0 1280 0 0 1280 0 1280 -1280 0 -1280 0 0
--1280z m1555 936 c387 -112 675 -426 741 -810 24 -138 15 -352 -20 -470 -106
--353 -360 -606 -713 -712 -75 -22 -113 -27 -253 -31 -144 -5 -176 -2 -252 16
--316 75 -564 271 -707 557 -67 136 -92 237 -98 401 -7 164 5 253 47 378 106
-315 349 556 665 659 114 37 180 45 350 41 125 -2 165 -7 240 -29z"/>
-<path d="M1091 2165 c-364 -82 -629 -328 -738 -682 -24 -80 -27 -103 -27 -258
--1 -146 2 -182 21 -251 74 -271 259 -497 508 -621 477 -238 1061 -35 1294 450
-61 126 83 220 88 379 7 194 -15 307 -93 461 -126 251 -340 428 -614 507 -99
-29 -343 37 -439 15z m829 -473 c55 -54 100 -106 100 -116 0 -21 -184 -213
--212 -222 -24 -7 -48 12 -48 38 0 11 26 47 58 80 l57 60 -151 -3 c-145 -4
--152 -5 -190 -31 -22 -15 -78 -73 -124 -128 l-85 -99 -32 31 -32 31 30 38 c17
-22 70 79 117 128 66 67 97 92 127 100 22 6 106 11 188 11 81 0 147 3 147 8 0
-4 -25 31 -55 61 -55 55 -65 77 -43 99 25 25 50 10 148 -86z m-1002 -101 c46
--24 141 -121 312 -321 203 -236 290 -330 322 -346 22 -11 60 -14 169 -12 l141
-3 -51 58 c-28 32 -51 64 -51 71 0 18 21 36 43 36 24 0 217 -193 217 -217 0
--19 -185 -210 -212 -219 -24 -7 -48 12 -48 38 0 10 23 43 50 72 l50 53 -52 7
-c-29 3 -93 6 -142 6 -104 0 -152 12 -200 52 -19 15 -135 144 -258 286 -274
-316 -305 347 -354 361 -22 6 -94 11 -161 11 -67 0 -128 3 -137 6 -22 9 -21 61
-2 67 9 3 86 5 170 6 133 1 158 -2 190 -18z m227 -468 c23 -34 17 -43 -103
--172 -119 -128 -131 -133 -343 -129 l-154 3 0 35 c0 34 1 35 50 42 28 3 96 7
-153 7 64 1 115 6 136 15 20 8 71 56 127 120 52 58 99 106 105 106 7 0 20 -12
-29 -27z"/>
-</g>
-</svg>

changedetectionio/static/favicons/site.webmanifest

@@ -1,19 +0,0 @@
-{
-    "name": "",
-    "short_name": "",
-    "icons": [
-        {
-            "src": "android-chrome-192x192.png",
-            "sizes": "192x192",
-            "type": "image/png"
-        },
-        {
-            "src": "android-chrome-256x256.png",
-            "sizes": "256x256",
-            "type": "image/png"
-        }
-    ],
-    "theme_color": "#ffffff",
-    "background_color": "#ffffff",
-    "display": "standalone"
-}

changedetectionio/static/images/bell-off.svg

@@ -1,4 +1,42 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg width="15" height="16.363636" viewBox="0 0 15 16.363636" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg">
+<svg
-  <path d="m 14.318182,11.762045 v 1.1925 H 5.4102273 L 11.849318,7.1140909 C 12.234545,9.1561364 12.54,11.181818 14.318182,11.762045 Z m -6.7984093,4.601591 c 1.0759091,0 2.0256823,-0.955909 2.0256823,-2.045454 H 5.4545455 c 0,1.089545 0.9879545,2.045454 2.0652272,2.045454 z M 15,2.8622727 0.9177273,15.636136 0,14.627045 l 1.8443182,-1.6725 h -1.1625 v -1.1925 C 4.0070455,10.677273 2.1784091,4.5388636 5.3611364,2.6897727 5.8009091,2.4347727 6.0709091,1.9609091 6.0702273,1.4488636 v -0.00205 C 6.0702273,0.64772727 6.7104545,0 7.5,0 8.2895455,0 8.9297727,0.64772727 8.9297727,1.4468182 v 0.00205 C 8.9290909,1.9602319 9.199773,2.4354591 9.638864,2.6897773 10.364318,3.111141 10.827273,3.7568228 11.1525,4.5129591 L 14.085682,1.8531818 Z M 6.8181818,1.3636364 C 6.8181818,1.74 7.1236364,2.0454545 7.5,2.0454545 7.8763636,2.0454545 8.1818182,1.74 8.1818182,1.3636364 8.1818182,0.98795455 7.8763636,0.68181818 7.5,0.68181818 c -0.3763636,0 -0.6818182,0.30613637 -0.6818182,0.68181822 z" id="path2" style="fill:#f8321b;stroke-width:0.681818;fill-opacity:1"/>
+   width="15"
+   height="16.363636"
+   viewBox="0 0 15 16.363636"
+   version="1.1"
+   id="svg4"
+   sodipodi:docname="bell-off.svg"
+   inkscape:version="1.1.1 (1:1.1+202109281949+c3084ef5ed)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview5"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0"
+     inkscape:zoom="28.416667"
+     inkscape:cx="-0.59824046"
+     inkscape:cy="12"
+     inkscape:window-width="1554"
+     inkscape:window-height="896"
+     inkscape:window-x="2095"
+     inkscape:window-y="107"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg4" />
+  <defs
+     id="defs8" />
+  <path
+     d="m 14.318182,11.762045 v 1.1925 H 5.4102273 L 11.849318,7.1140909 C 12.234545,9.1561364 12.54,11.181818 14.318182,11.762045 Z m -6.7984093,4.601591 c 1.0759091,0 2.0256823,-0.955909 2.0256823,-2.045454 H 5.4545455 c 0,1.089545 0.9879545,2.045454 2.0652272,2.045454 z M 15,2.8622727 0.9177273,15.636136 0,14.627045 l 1.8443182,-1.6725 h -1.1625 v -1.1925 C 4.0070455,10.677273 2.1784091,4.5388636 5.3611364,2.6897727 5.8009091,2.4347727 6.0709091,1.9609091 6.0702273,1.4488636 v -0.00205 C 6.0702273,0.64772727 6.7104545,0 7.5,0 8.2895455,0 8.9297727,0.64772727 8.9297727,1.4468182 v 0.00205 C 8.9290909,1.9602319 9.199773,2.4354591 9.638864,2.6897773 10.364318,3.111141 10.827273,3.7568228 11.1525,4.5129591 L 14.085682,1.8531818 Z M 6.8181818,1.3636364 C 6.8181818,1.74 7.1236364,2.0454545 7.5,2.0454545 7.8763636,2.0454545 8.1818182,1.74 8.1818182,1.3636364 8.1818182,0.98795455 7.8763636,0.68181818 7.5,0.68181818 c -0.3763636,0 -0.6818182,0.30613637 -0.6818182,0.68181822 z"
+     id="path2"
+     style="fill:#f8321b;stroke-width:0.681818;fill-opacity:1" />
 </svg>

changedetectionio/static/images/notice.svg

@@ -1,51 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!-- Created with Inkscape (http://www.inkscape.org/) -->
-
-<svg
-   width="20.108334mm"
-   height="21.43125mm"
-   viewBox="0 0 20.108334 21.43125"
-   version="1.1"
-   id="svg5"
-   xmlns:xlink="http://www.w3.org/1999/xlink"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg">
-  <defs
-     id="defs2" />
-  <g
-     id="layer1"
-     transform="translate(-141.05873,-76.816635)">
-    <image
-       width="20.108334"
-       height="21.43125"
-       preserveAspectRatio="none"
-       style="image-rendering:optimizeQuality"
-       xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEwAAABRCAYAAAB430BuAAAABHNCSVQICAgIfAhkiAAABLxJREFU
-eJztnN2Z2jgUhl8Z7petIGwF0WMXsFBBoIKwFWS2gmQryKSCJRXsTAUDBTDRVBCmgkAB9tkLexh+
-bIONLGwP7xU2RjafpaOjoyNBCxHNQAJEfG5sl+3ZLrAWeAyST5/sF91mFH3bRbZbsAq4ClaQq2B7
-iKYnmg9Z318F20ICRnj8pMOd6E3HscNVsATxmQD/oeghPCnDLO26q2AkYin+TQ7XREyyrn3zgu2J
-BSEjZTBZ179pwQ7EEv7KaoovvFnBUsV6ZHrsd+0WTHhKPV1SLGivYEsA1KEtEs2grFitRjQ65VxP
-fH5JgEjAKsvXupKwFfYxaYJeSeHcWqVSCuwD7/HQQD8lRHLWDStBWG3slbAElkTc5/lTZdkIJhpN
-h6/UUZDyzAgZK8PKVoEKErE8HlD0bBVcI2ZqwdBWYbFgAT+g1UZwrBbcvRyIpofHJ1Sh1rQCZt1k
-lN5msQAm8CoYoFF8KVHOsFtQ5aayExBUhpnopJl6J/3/FREGWCrxmaH40/4z1oyQ320Yf5dDozXC
-P4QMCRkCY4S5w/tbMTtd4L2Ngo6wJmSQ4hfdScAU+OjgGazgOXEl8oJyof3Z6Spx0iTzgnLKsMoK
-w9SRuoR3rHniVVMXwRpDXQR7d+kHOJV6CFZB0khVOBGsTcE6VzWsNVGQizfJptU+N4LlD3AbVfsu
-XsOahhvB8nrB08IrtcGNYNIct+EYl2+S6mr0D8kLUMrV6BfFRTzOGs4Ey8p1aNrUnssaliaMO/vV
-sfNi3AmW5j54DgUTO/dyJ1hab9iwHhLcNskP23ZMND0kewFBXek6vZvHg/hMiUPSN00z+OBasFig
-y8wSRfnZ0adSBz+sUVwFK4jbJhnPP06To1ETczpcCnavHhltHd82LU0AXDbJMGXBU8PSBAA8Jxk0
-wnNaqlGSJuAyg+dsXIV38iZqXU3iWsmodhetSNlDQgJGriZxbWVSe1hS/gQ+S/C6j4QEfES21vxU
-icXsoC4vC5mqJvbybyXgduucG/YWaYmmj+IdHvpoxFdt8ltRP5h3iZjRqfBh60C4t1rNY7rxAU95
-aYnhEp+/u8pgxGfeRCfyJIR5SkLfFOHYXMMzu63PEDF9WQnSo8MUmhduyUWYEzGyvnRmU3683ugG
-GAG/2bqJU4RnFDNCpsfWb5chswUnwb5Xg+hxiyo9w7MGJoSVpmYulam+A8scS+5nPYtf+s9mpZw7
-J1nayDnCVuu4Ck+E6DqIBYDHHR1+is/n8kVUhfBExMBFMzm4taafkXcWL9BSfBG/nNN8sutYcE3S
-d7XI3o6lSpIe/xcAIX/svzDxMVu22BAyLNKL2q9hwrdLiZWwXbP6B99GDLaGSpoOD6JPn4yxK1i8
-B0StY1zKsCJiQNxzQ0HRbAm2BsZN2TBDGVaE5USzIVjsNix2VrzWHmUwB6J5fD32uyKCzQ7OxG5D
-vzZuQ0E2osXjRlBMjvWe5WtYPE4b2BynXQJlMEToTUegmEiwM1mzQ1nBvqvH5ov1wlZHcA+AZHdc
-xQW7vNuQS9kBtzKs1IIRMM7b0q/YvGTzto4qbFutdV5FnLtLk2x3JVWUfXKTbIu9Opc2J6Osj19S
-HLfJKO64r6rg/wFBX3+2ZapW8wAAAABJRU5ErkJggg==
-"
-       id="image832"
-       x="141.05873"
-       y="76.816635" />
-  </g>
-</svg>

changedetectionio/static/images/play.svg

@@ -1,122 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   version="1.1"
-   id="Capa_1"
-   x="0px"
-   y="0px"
-   viewBox="0 0 15 14.998326"
-   xml:space="preserve"
-   width="15"
-   height="14.998326"
-   sodipodi:docname="play.svg"
-   inkscape:version="1.1.1 (1:1.1+202109281949+c3084ef5ed)"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:dc="http://purl.org/dc/elements/1.1/"><sodipodi:namedview
-   id="namedview21"
-   pagecolor="#ffffff"
-   bordercolor="#666666"
-   borderopacity="1.0"
-   inkscape:pageshadow="2"
-   inkscape:pageopacity="0.0"
-   inkscape:pagecheckerboard="0"
-   showgrid="false"
-   inkscape:zoom="45.47174"
-   inkscape:cx="7.4991632"
-   inkscape:cy="7.4991632"
-   inkscape:window-width="1554"
-   inkscape:window-height="896"
-   inkscape:window-x="3048"
-   inkscape:window-y="227"
-   inkscape:window-maximized="0"
-   inkscape:current-layer="Capa_1" /><metadata
-   id="metadata39"><rdf:RDF><cc:Work
-       rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
-         rdf:resource="http://purl.org/dc/dcmitype/StillImage" /></cc:Work></rdf:RDF></metadata><defs
-   id="defs37" />
-<path
-   id="path2"
-   style="fill:#1b98f8;fill-opacity:1;stroke-width:0.0292893"
-   d="M 7.4980469,0 C 4.5496028,-0.04093755 1.7047721,1.8547661 0.58789062,4.5800781 -0.57819305,7.2574082 0.02636631,10.583252 2.0703125,12.671875 4.0368718,14.788335 7.2754393,15.560096 9.9882812,14.572266 12.800219,13.617028 14.874915,10.855516 14.986328,7.8847656 15.172991,4.9968456 13.497714,2.109448 10.910156,0.8203125 9.858961,0.28011352 8.6796569,-0.00179908 7.4980469,0 Z"
-   sodipodi:nodetypes="ccccccc" />
-<g
-   id="g4"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g6"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g8"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g10"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g12"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g14"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g16"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g18"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g20"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g22"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g24"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g26"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g28"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g30"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g32"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<path
-   sodipodi:type="star"
-   style="fill:#ffffff;fill-opacity:1;stroke-width:37.7953;paint-order:stroke fill markers"
-   id="path1203"
-   inkscape:flatsided="false"
-   sodipodi:sides="3"
-   sodipodi:cx="7.2964563"
-   sodipodi:cy="7.3240671"
-   sodipodi:r1="3.805218"
-   sodipodi:r2="1.9026089"
-   sodipodi:arg1="-0.0017436774"
-   sodipodi:arg2="1.0454539"
-   inkscape:rounded="0"
-   inkscape:randomized="0"
-   d="M 11.101669,7.317432 8.2506324,8.9701135 5.3995964,10.622795 5.3938504,7.3273846 5.3881041,4.0319742 8.2448863,5.6747033 Z"
-   inkscape:transform-center-x="-0.94843001"
-   inkscape:transform-center-y="0.0033175346" /></svg>

changedetectionio/static/images/price-tag-icon.svg

@@ -1,2 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg width="83.39" height="89.648" enable-background="new 0 0 122.406 122.881" version="1.1" viewBox="0 0 83.39 89.648" xml:space="preserve" xmlns="http://www.w3.org/2000/svg"><g transform="translate(5e-4 -33.234)"><path d="m44.239 42.946-39.111 39.896 34.908 34.91 39.09-39.876-1.149-34.931zm-0.91791 42.273c0.979-0.979 1.507-1.99 1.577-3.027 0.077-1.043-0.248-2.424-0.967-4.135-0.725-1.717-1.348-3.346-1.87-4.885s-0.814-3.014-0.897-4.432c-0.07-1.42 0.134-2.768 0.624-4.045 0.477-1.279 1.348-2.545 2.607-3.804 2.099-2.099 4.535-3.123 7.314-3.065 2.773 0.063 5.457 1.158 8.04 3.294l2.881 3.034c1.946 2.607 2.799 5.33 2.557 8.166-0.235 2.83-1.532 5.426-3.893 7.785l-6.296-6.297c1.291-1.291 2.035-2.531 2.238-3.727 0.191-1.197-0.165-2.252-1.081-3.168-0.821-0.82-1.717-1.195-2.69-1.139-0.967 0.064-1.908 0.547-2.817 1.457-0.922 0.922-1.393 1.914-1.412 2.977s0.306 2.416 0.973 4.064c0.661 1.652 1.24 3.25 1.736 4.801 0.496 1.553 0.782 3.035 0.858 4.445 0.076 1.426-0.127 2.787-0.591 4.104-0.477 1.316-1.336 2.596-2.588 3.848-2.125 2.125-4.522 3.186-7.212 3.18s-5.311-1.063-7.855-3.16l-3.747 3.746-2.964-2.965 3.766-3.764c-2.423-2.996-3.568-5.998-3.447-9.02 0.127-3.014 1.476-5.813 4.045-8.383l6.278 6.277c-1.412 1.412-2.175 2.799-2.277 4.16-0.108 1.367 0.414 2.627 1.571 3.783 0.839 0.84 1.755 1.26 2.741 1.242 0.985-0.017 1.92-0.47 2.798-1.347zm21.127-46.435h17.457c-0.0269 2.2368 0.69936 16.025 0.69936 16.025l0.785 23.858c0.019 0.609-0.221 1.164-0.619 1.564l5e-3 4e-3 -41.236 42.022c-0.82213 0.8378-2.175 0.83-3.004 0l-37.913-37.91c-0.83-0.83-0.83-2.176 0-3.006l41.236-42.021c0.39287-0.42671 1.502-0.53568 1.502-0.53568zm18.011 11.59c-59.392-29.687-29.696-14.843 0 0z"/></g></svg>

changedetectionio/static/images/spread.svg

@@ -1,5 +1,46 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg width="18" height="19.92" viewBox="0 0 18 19.92" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg">
+<svg
-  <path d="M -3,-2 H 21 V 22 H -3 Z" fill="none" id="path2"/>
+   width="18"
-  <path d="m 15,14.08 c -0.76,0 -1.44,0.3 -1.96,0.77 L 5.91,10.7 C 5.96,10.47 6,10.24 6,10 6,9.76 5.96,9.53 5.91,9.3 L 12.96,5.19 C 13.5,5.69 14.21,6 15,6 16.66,6 18,4.66 18,3 18,1.34 16.66,0 15,0 c -1.66,0 -3,1.34 -3,3 0,0.24 0.04,0.47 0.09,0.7 L 5.04,7.81 C 4.5,7.31 3.79,7 3,7 1.34,7 0,8.34 0,10 c 0,1.66 1.34,3 3,3 0.79,0 1.5,-0.31 2.04,-0.81 l 7.12,4.16 c -0.05,0.21 -0.08,0.43 -0.08,0.65 0,1.61 1.31,2.92 2.92,2.92 1.61,0 2.92,-1.31 2.92,-2.92 0,-1.61 -1.31,-2.92 -2.92,-2.92 z" id="path4" style="fill:#0078e7;fill-opacity:1"/>
+   height="19.92"
+   viewBox="0 0 18 19.92"
+   version="1.1"
+   id="svg6"
+   sodipodi:docname="spread.svg"
+   inkscape:version="1.1.1 (1:1.1+202109281949+c3084ef5ed)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs10" />
+  <sodipodi:namedview
+     id="namedview8"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0"
+     inkscape:zoom="28.416667"
+     inkscape:cx="9.0087975"
+     inkscape:cy="9.9941348"
+     inkscape:window-width="1920"
+     inkscape:window-height="1056"
+     inkscape:window-x="1920"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg6" />
+  <path
+     d="M -3,-2 H 21 V 22 H -3 Z"
+     fill="none"
+     id="path2" />
+  <path
+     d="m 15,14.08 c -0.76,0 -1.44,0.3 -1.96,0.77 L 5.91,10.7 C 5.96,10.47 6,10.24 6,10 6,9.76 5.96,9.53 5.91,9.3 L 12.96,5.19 C 13.5,5.69 14.21,6 15,6 16.66,6 18,4.66 18,3 18,1.34 16.66,0 15,0 c -1.66,0 -3,1.34 -3,3 0,0.24 0.04,0.47 0.09,0.7 L 5.04,7.81 C 4.5,7.31 3.79,7 3,7 1.34,7 0,8.34 0,10 c 0,1.66 1.34,3 3,3 0.79,0 1.5,-0.31 2.04,-0.81 l 7.12,4.16 c -0.05,0.21 -0.08,0.43 -0.08,0.65 0,1.61 1.31,2.92 2.92,2.92 1.61,0 2.92,-1.31 2.92,-2.92 0,-1.61 -1.31,-2.92 -2.92,-2.92 z"
+     id="path4"
+     style="fill:#0078e7;fill-opacity:1" />
 </svg>

changedetectionio/static/js/browser-steps.js

@@ -1,454 +0,0 @@
-$(document).ready(function () {
-
-    // duplicate
-    var csrftoken = $('input[name=csrf_token]').val();
-    $.ajaxSetup({
-        beforeSend: function (xhr, settings) {
-            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
-                xhr.setRequestHeader("X-CSRFToken", csrftoken)
-            }
-        }
-    })
-    var browsersteps_session_id;
-    var browserless_seconds_remaining = 0;
-    var apply_buttons_disabled = false;
-    var include_text_elements = $("#include_text_elements");
-    var xpath_data = false;
-    var current_selected_i;
-    var state_clicked = false;
-    var c;
-
-    // redline highlight context
-    var ctx;
-    var last_click_xy = {'x': -1, 'y': -1}
-
-    $(window).resize(function () {
-        set_scale();
-    });
-    // Should always be disabled
-    $('#browser_steps >li:first-child select').val('Goto site').attr('disabled', 'disabled');
-
-    $('#browsersteps-click-start').click(function () {
-        $("#browsersteps-click-start").fadeOut();
-        $("#browsersteps-selector-wrapper .spinner").fadeIn();
-        start();
-    });
-
-    $('a#browsersteps-tab').click(function () {
-        reset();
-    });
-
-    window.addEventListener('hashchange', function () {
-        if (window.location.hash == '#browser-steps') {
-            reset();
-        }
-    });
-
-    function reset() {
-        xpath_data = false;
-        $('#browsersteps-img').removeAttr('src');
-        $("#browsersteps-click-start").show();
-        $("#browsersteps-selector-wrapper .spinner").hide();
-        browserless_seconds_remaining = 0;
-        browsersteps_session_id = false;
-        apply_buttons_disabled = false;
-        ctx.clearRect(0, 0, c.width, c.height);
-        set_first_gotosite_disabled();
-    }
-
-    function set_first_gotosite_disabled() {
-        $('#browser_steps >li:first-child select').val('Goto site').attr('disabled', 'disabled');
-        $('#browser_steps >li:first-child').css('opacity', '0.5');
-    }
-
-    // Show seconds remaining until playwright/browserless needs to restart the session
-    // (See comment at the top of changedetectionio/blueprint/browser_steps/__init__.py )
-    setInterval(() => {
-        if (browserless_seconds_remaining >= 1) {
-            document.getElementById('browserless-seconds-remaining').innerText = browserless_seconds_remaining + " seconds remaining in session";
-            browserless_seconds_remaining -= 1;
-        }
-    }, "1000")
-
-
-    function set_scale() {
-
-        // some things to check if the scaling doesnt work
-        // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
-        selector_image = $("img#browsersteps-img")[0];
-        selector_image_rect = selector_image.getBoundingClientRect();
-
-        // make the canvas and input steps the same size as the image
-        $('#browsersteps-selector-canvas').attr('height', selector_image_rect.height).attr('width', selector_image_rect.width);
-        //$('#browsersteps-selector-wrapper').attr('width', selector_image_rect.width);
-        $('#browser-steps-ui').attr('width', selector_image_rect.width);
-
-        x_scale = selector_image_rect.width / xpath_data['browser_width'];
-        y_scale = selector_image_rect.height / selector_image.naturalHeight;
-        ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
-        ctx.fillStyle = 'rgba(255,0,0, 0.1)';
-        ctx.lineWidth = 3;
-        console.log("scaling set  x: " + x_scale + " by y:" + y_scale);
-    }
-
-    // bootstrap it, this will trigger everything else
-    $('#browsersteps-img').bind('load', function () {
-        $('body').addClass('full-width');
-        console.log("Loaded background...");
-
-        document.getElementById("browsersteps-selector-canvas");
-        c = document.getElementById("browsersteps-selector-canvas");
-        // redline highlight context
-        ctx = c.getContext("2d");
-        // @todo is click better?
-        $('#browsersteps-selector-canvas').off("mousemove mousedown click");
-        // Undo disable_browsersteps_ui
-        $("#browser-steps-ui").css('opacity', '1.0');
-
-        // init
-        set_scale();
-
-        // @todo click ? some better library?
-        $('#browsersteps-selector-canvas').bind('click', function (e) {
-            // https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent
-            e.preventDefault()
-        });
-
-        $('#browsersteps-selector-canvas').bind('mousedown', function (e) {
-            // https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent
-            e.preventDefault()
-            console.log(e);
-            console.log("current xpath in index is " + current_selected_i);
-            last_click_xy = {'x': parseInt((1 / x_scale) * e.offsetX), 'y': parseInt((1 / y_scale) * e.offsetY)}
-            process_selected(current_selected_i);
-            current_selected_i = false;
-
-            // if process selected returned false, then best we can do is offer a x,y click :(
-            if (!found_something) {
-                var first_available = $("ul#browser_steps li.empty").first();
-                $('select', first_available).val('Click X,Y').change();
-                $('input[type=text]', first_available).first().val(last_click_xy['x'] + ',' + last_click_xy['y']);
-                draw_circle_on_canvas(e.offsetX, e.offsetY);
-            }
-        });
-
-        $('#browsersteps-selector-canvas').bind('mousemove', function (e) {
-            if (!xpath_data) {
-                return;
-            }
-
-            // checkbox if find elements is enabled
-            ctx.clearRect(0, 0, c.width, c.height);
-            ctx.fillStyle = 'rgba(255,0,0, 0.1)';
-            ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
-
-            // Add in offset
-            if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) {
-                var targetOffset = $(e.target).offset();
-                e.offsetX = e.pageX - targetOffset.left;
-                e.offsetY = e.pageY - targetOffset.top;
-            }
-            current_selected_i = false;
-            // Reverse order - the most specific one should be deeper/"laster"
-            // Basically, find the most 'deepest'
-            //$('#browsersteps-selector-canvas').css('cursor', 'pointer');
-            for (var i = xpath_data['size_pos'].length; i !== 0; i--) {
-                // draw all of them? let them choose somehow?
-                var sel = xpath_data['size_pos'][i - 1];
-                // If we are in a bounding-box
-                if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
-                    &&
-                    e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
-
-                ) {
-                    // Only highlight these interesting types
-                    if (1) {
-                        ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-                        ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-                        current_selected_i = i - 1;
-                        break;
-
-                        // find the smallest one at this x,y
-                        // does it mean sort the xpath list by size (w*h) i think so!
-                    } else {
-
-                        if (include_text_elements[0].checked === true) {
-                            // blue one with background instead?
-                            ctx.fillStyle = 'rgba(0,0,255, 0.1)';
-                            ctx.strokeStyle = 'rgba(0,0,200, 0.7)';
-                            $('#browsersteps-selector-canvas').css('cursor', 'grab');
-                            ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-                            ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-                            current_selected_i = i - 1;
-                            break;
-                        }
-                    }
-                }
-            }
-
-        }.debounce(10));
-    });
-
-//    $("#browser-steps-fieldlist").bind('mouseover', function(e) {
-//        console.log(e.xpath_data_index);
-    // });
-
-
-    // callback for clicking on an xpath on the canvas
-    function process_selected(xpath_data_index) {
-        found_something = false;
-        var first_available = $("ul#browser_steps li.empty").first();
-
-
-        if (xpath_data_index !== false) {
-            // Nothing focused, so fill in a new one
-            // if inpt type button or <button>
-            // from the top, find the next not used one and use it
-            var x = xpath_data['size_pos'][xpath_data_index];
-            console.log(x);
-            if (x && first_available.length) {
-                // @todo will it let you click shit that has a layer ontop? probably not.
-                if (x['tagtype'] === 'text' || x['tagtype'] === 'email' || x['tagName'] === 'textarea' || x['tagtype'] === 'password' || x['tagtype'] === 'search') {
-                    $('select', first_available).val('Enter text in field').change();
-                    $('input[type=text]', first_available).first().val(x['xpath']);
-                    $('input[placeholder="Value"]', first_available).addClass('ok').click().focus();
-                    found_something = true;
-                } else {
-                    if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
-                        $('select', first_available).val('Click element').change();
-                        $('input[type=text]', first_available).first().val(x['xpath']);
-                        found_something = true;
-                    }
-                }
-
-                first_available.xpath_data_index = xpath_data_index;
-
-                if (!found_something) {
-                    if (include_text_elements[0].checked === true) {
-                        // Suggest that we use as filter?
-                        // @todo filters should always be in the last steps, nothing non-filter after it
-                        found_something = true;
-                        ctx.strokeStyle = 'rgba(0,0,255, 0.9)';
-                        ctx.fillStyle = 'rgba(0,0,255, 0.1)';
-                        $('select', first_available).val('Extract text and use as filter').change();
-                        $('input[type=text]', first_available).first().val(x['xpath']);
-                        include_text_elements[0].checked = false;
-                    }
-                }
-            }
-        }
-    }
-
-    function draw_circle_on_canvas(x, y) {
-        ctx.beginPath();
-        ctx.arc(x, y, 8, 0, 2 * Math.PI, false);
-        ctx.fillStyle = 'rgba(255,0,0, 0.6)';
-        ctx.fill();
-    }
-
-    function start() {
-        console.log("Starting browser-steps UI");
-        browsersteps_session_id = Date.now();
-        // @todo This setting of the first one should be done at the datalayer but wtforms doesnt wanna play nice
-        $('#browser_steps >li:first-child').removeClass('empty');
-        set_first_gotosite_disabled();
-        $('#browser-steps-ui .loader .spinner').show();
-        $('.clear,.remove', $('#browser_steps >li:first-child')).hide();
-        $.ajax({
-            type: "GET",
-            url: browser_steps_sync_url + "&browsersteps_session_id=" + browsersteps_session_id,
-            statusCode: {
-                400: function () {
-                    // More than likely the CSRF token was lost when the server restarted
-                    alert("There was a problem processing the request, please reload the page.");
-                }
-            }
-        }).done(function (data) {
-            xpath_data = data.xpath_data;
-            $("#loading-status-text").fadeIn();
-            // This should trigger 'Goto site'
-            console.log("Got startup response, requesting Goto-Site (first) step fake click");
-            $('#browser_steps >li:first-child .apply').click();
-            browserless_seconds_remaining = data.browser_time_remaining;
-            set_first_gotosite_disabled();
-        }).fail(function (data) {
-            console.log(data);
-            alert('There was an error communicating with the server.');
-        });
-
-    }
-
-    function disable_browsersteps_ui() {
-        set_first_gotosite_disabled();
-        $("#browser-steps-ui").css('opacity', '0.3');
-        $('#browsersteps-selector-canvas').off("mousemove mousedown click");
-    }
-
-
-    ////////////////////////// STEPS UI ////////////////////
-    $('ul#browser_steps [type="text"]').keydown(function (e) {
-        if (e.keyCode === 13) {
-            // hitting [enter] in a browser-step input should trigger the 'Apply'
-            e.preventDefault();
-            $(".apply", $(this).closest('li')).click();
-            return false;
-        }
-    });
-
-    // Look up which step was selected, and enable or disable the related extra fields
-    // So that people using it dont' get confused
-    $('ul#browser_steps select').on("change", function () {
-        var config = browser_steps_config[$(this).val()].split(' ');
-        var elem_selector = $('tr:nth-child(2) input', $(this).closest('tbody'));
-        var elem_value = $('tr:nth-child(3) input', $(this).closest('tbody'));
-
-        if (config[0] == 0) {
-            $(elem_selector).fadeOut();
-        } else {
-            $(elem_selector).fadeIn();
-        }
-        if (config[1] == 0) {
-            $(elem_value).fadeOut();
-        } else {
-            $(elem_value).fadeIn();
-        }
-
-        if ($(this).val() === 'Click X,Y' && last_click_xy['x'] > 0 && $(elem_value).val().length === 0) {
-            // @todo handle scale
-            $(elem_value).val(last_click_xy['x'] + ',' + last_click_xy['y']);
-        }
-    }).change();
-
-    function set_greyed_state() {
-        $('ul#browser_steps select').not('option:selected[value="Choose one"]').closest('li').removeClass('empty');
-        $('ul#browser_steps select option:selected[value="Choose one"]').closest('li').addClass('empty');
-    }
-
-    // Add the extra buttons to the steps
-    $('ul#browser_steps li').each(function (i) {
-            var s = '<div class="control">' + '<a data-step-index=' + i + ' class="pure-button button-secondary button-green button-xsmall apply" >Apply</a>&nbsp;';
-            if (i > 0) {
-                // The first step never gets these (Goto-site)
-                s += '<a data-step-index=' + i + ' class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;' +
-                    '<a data-step-index=' + i + ' class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>';
-            }
-            s += '</div>';
-            $(this).append(s)
-        }
-    );
-
-    $('ul#browser_steps li .control .clear').click(function (element) {
-        $("select", $(this).closest('li')).val("Choose one").change();
-        $(":text", $(this).closest('li')).val('');
-    });
-
-
-    $('ul#browser_steps li .control .remove').click(function (element) {
-        // so you wanna remove the 2nd (3rd spot 0,1,2,...)
-        var p = $("#browser_steps li").index($(this).closest('li'));
-
-        var elem_to_remove = $("#browser_steps li")[p];
-        $('.clear', elem_to_remove).click();
-        $("#browser_steps li").slice(p, 10).each(function (index) {
-            // get the next one's value from where we clicked
-            var next = $("#browser_steps li")[p + index + 1];
-            if (next) {
-                // and set THIS ones value from the next one
-                var n = $('input', next);
-                $("select", $(this)).val($('select', next).val());
-                $('input', this)[0].value = $(n)[0].value;
-                $('input', this)[1].value = $(n)[1].value;
-                // Triggers reconfiguring the field based on the system config
-                $("select", $(this)).change();
-            }
-
-        });
-
-        // Reset their hidden/empty states
-        set_greyed_state();
-    });
-
-    $('ul#browser_steps li .control .apply').click(function (event) {
-        // sequential requests @todo refactor
-        if (apply_buttons_disabled) {
-            return;
-        }
-
-        var current_data = $(event.currentTarget).closest('li');
-        $('#browser-steps-ui .loader .spinner').fadeIn();
-        apply_buttons_disabled = true;
-        $('ul#browser_steps li .control .apply').css('opacity', 0.5);
-        $("#browsersteps-img").css('opacity', 0.65);
-
-        var is_last_step = 0;
-        var step_n = $(event.currentTarget).data('step-index');
-
-        // On the last step, we should also be getting data ready for the visual selector
-        $('ul#browser_steps li select').each(function (i) {
-            if ($(this).val() !== 'Choose one') {
-                is_last_step += 1;
-            }
-        });
-
-        if (is_last_step == (step_n + 1)) {
-            is_last_step = true;
-        } else {
-            is_last_step = false;
-        }
-
-        console.log("Requesting step via POST " + $("select[id$='operation']", current_data).first().val());
-        // POST the currently clicked step form widget back and await response, redraw
-        $.ajax({
-            method: "POST",
-            url: browser_steps_sync_url + "&browsersteps_session_id=" + browsersteps_session_id,
-            data: {
-                'operation': $("select[id$='operation']", current_data).first().val(),
-                'selector': $("input[id$='selector']", current_data).first().val(),
-                'optional_value': $("input[id$='optional_value']", current_data).first().val(),
-                'step_n': step_n,
-                'is_last_step': is_last_step
-            },
-            statusCode: {
-                400: function () {
-                    // More than likely the CSRF token was lost when the server restarted
-                    alert("There was a problem processing the request, please reload the page.");
-                    $("#loading-status-text").hide();
-                    $('#browser-steps-ui .loader .spinner').fadeOut();
-                },
-                401: function (data) {
-                    // More than likely the CSRF token was lost when the server restarted
-                    alert(data.responseText);
-                    $("#loading-status-text").hide();
-                    $('#browser-steps-ui .loader .spinner').fadeOut();
-                }
-            }
-        }).done(function (data) {
-            // it should return the new state (selectors available and screenshot)
-            xpath_data = data.xpath_data;
-            $('#browsersteps-img').attr('src', data.screenshot);
-            $('#browser-steps-ui .loader .spinner').fadeOut();
-            apply_buttons_disabled = false;
-            $("#browsersteps-img").css('opacity', 1);
-            $('ul#browser_steps li .control .apply').css('opacity', 1);
-            browserless_seconds_remaining = data.browser_time_remaining;
-            $("#loading-status-text").hide();
-            set_first_gotosite_disabled();
-        }).fail(function (data) {
-            console.log(data);
-            if (data.responseText.includes("Browser session expired")) {
-                disable_browsersteps_ui();
-            }
-            apply_buttons_disabled = false;
-            $("#loading-status-text").hide();
-            $('ul#browser_steps li .control .apply').css('opacity', 1);
-            $("#browsersteps-img").css('opacity', 1);
-        });
-
-    });
-
-
-    $("ul#browser_steps select").change(function () {
-        set_greyed_state();
-    }).change();
-
-});

changedetectionio/static/js/diff-overview.js

@@ -13,8 +13,6 @@ $(document).ready(function () {
         } else if (hash_name === '#error-screenshot') {
             $("img#error-screenshot-img").attr('src', error_screenshot_url);
             $("#settings").hide();
-        } else if (hash_name === '#extract') {
-            $("#settings").hide();
         }
 
 

changedetectionio/static/js/diff-render.js

@@ -1,110 +0,0 @@
-var a = document.getElementById("a");
-var b = document.getElementById("b");
-var result = document.getElementById("result");
-
-function changed() {
-  // https://github.com/kpdecker/jsdiff/issues/389
-  // I would love to use `{ignoreWhitespace: true}` here but it breaks the formatting
-  options = {
-    ignoreWhitespace: document.getElementById("ignoreWhitespace").checked,
-  };
-
-  var diff = Diff[window.diffType](a.textContent, b.textContent, options);
-  var fragment = document.createDocumentFragment();
-  for (var i = 0; i < diff.length; i++) {
-    if (diff[i].added && diff[i + 1] && diff[i + 1].removed) {
-      var swap = diff[i];
-      diff[i] = diff[i + 1];
-      diff[i + 1] = swap;
-    }
-
-    var node;
-    if (diff[i].removed) {
-      node = document.createElement("del");
-      node.classList.add("change");
-      const wrapper = node.appendChild(document.createElement("span"));
-      wrapper.appendChild(document.createTextNode(diff[i].value));
-    } else if (diff[i].added) {
-      node = document.createElement("ins");
-      node.classList.add("change");
-      const wrapper = node.appendChild(document.createElement("span"));
-      wrapper.appendChild(document.createTextNode(diff[i].value));
-    } else {
-      node = document.createTextNode(diff[i].value);
-    }
-    fragment.appendChild(node);
-  }
-
-  result.textContent = "";
-  result.appendChild(fragment);
-
-  // Jump at start
-  inputs.current = 0;
-  next_diff();
-}
-
-window.onload = function () {
-  /* Convert what is options from UTC time.time() to local browser time */
-  var diffList = document.getElementById("diff-version");
-  if (typeof diffList != "undefined" && diffList != null) {
-    for (var option of diffList.options) {
-      var dateObject = new Date(option.value * 1000);
-      option.label = dateObject.toLocaleString();
-    }
-  }
-
-  /* Set current version date as local time in the browser also */
-  var current_v = document.getElementById("current-v-date");
-  var dateObject = new Date(newest_version_timestamp * 1000);
-  current_v.innerHTML = dateObject.toLocaleString();
-  onDiffTypeChange(
-    document.querySelector('#settings [name="diff_type"]:checked'),
-  );
-  changed();
-};
-
-a.onpaste = a.onchange = b.onpaste = b.onchange = changed;
-
-if ("oninput" in a) {
-  a.oninput = b.oninput = changed;
-} else {
-  a.onkeyup = b.onkeyup = changed;
-}
-
-function onDiffTypeChange(radio) {
-  window.diffType = radio.value;
-  // Not necessary
-  //	document.title = "Diff " + radio.value.slice(4);
-}
-
-var radio = document.getElementsByName("diff_type");
-for (var i = 0; i < radio.length; i++) {
-  radio[i].onchange = function (e) {
-    onDiffTypeChange(e.target);
-    changed();
-  };
-}
-
-document.getElementById("ignoreWhitespace").onchange = function (e) {
-  changed();
-};
-
-var inputs = document.getElementsByClassName("change");
-inputs.current = 0;
-
-function next_diff() {
-  var element = inputs[inputs.current];
-  var headerOffset = 80;
-  var elementPosition = element.getBoundingClientRect().top;
-  var offsetPosition = elementPosition - headerOffset + window.scrollY;
-
-  window.scrollTo({
-    top: offsetPosition,
-    behavior: "smooth",
-  });
-
-  inputs.current++;
-  if (inputs.current >= inputs.length) {
-    inputs.current = 0;
-  }
-}

changedetectionio/static/js/stepper.js

@@ -1,34 +0,0 @@
-$(document).ready(function(){
-   checkUserVal();
-   $('#fetch_backend input').on('change', checkUserVal);
-});
-
-var checkUserVal = function(){
-    if($('#fetch_backend input:checked').val()=='html_requests') {
-      $('#request-override').show();
-      $('#webdriver-stepper').hide();
-    } else {
-      $('#request-override').hide();
-      $('#webdriver-stepper').show();
-    }
-};
-
-$('a.row-options').on('click', function(){
-    var row=$(this.closest('tr'));
-    switch($(this).data("action")) {
-      case 'remove':
-        $(row).remove();
-      break;
-      case 'add':
-        var new_row=$(row).clone(true).insertAfter($(row));
-        $('input', new_new).val("");
-      break;
-      case 'add':
-        var new_row=$(row).clone(true).insertAfter($(row));
-        $('input', new_new).val("");
-      break;
-      case 'resend-step':
-
-      break;
-    }
-});

changedetectionio/static/js/tabs.js

@@ -3,8 +3,7 @@
 window.addEventListener('hashchange', function () {
     var tabs = document.getElementsByClassName('active');
     while (tabs[0]) {
-        tabs[0].classList.remove('active');
+        tabs[0].classList.remove('active')
-        document.body.classList.remove('full-width');
     }
     set_active_tab();
 }, false);
@@ -21,7 +20,6 @@ if (!has_errors.length) {
 }
 
 function set_active_tab() {
-    document.body.classList.remove('full-width');
     var tab = document.querySelectorAll("a[href='" + location.hash + "']");
     if (tab.length) {
         tab[0].parentElement.className = "active";
@@ -47,3 +45,4 @@ function focus_error_tab() {
 
 
 
+

changedetectionio/static/js/toggle-theme.js

@@ -1,24 +0,0 @@
-/**
- * @file
- * Toggles theme between light and dark mode.
- */
-$(document).ready(function () {
-  const button = document.getElementsByClassName("toggle-theme")[0];
-
-  button.onclick = () => {
-    const htmlElement = document.getElementsByTagName("html");
-    const isDarkMode = htmlElement[0].dataset.darkmode === "true";
-    htmlElement[0].dataset.darkmode = !isDarkMode;
-    if (isDarkMode) {
-      button.classList.remove("dark");
-      setCookieValue(false);
-    } else {
-      button.classList.add("dark");
-      setCookieValue(true);
-    }
-  };
-
-  const setCookieValue = (value) => {
-    document.cookie = `css_dark_mode=${value};max-age=31536000;path=/`
-  }
-});

changedetectionio/static/js/visual-selector.js

@@ -1,10 +1,10 @@
 // Horrible proof of concept code :)
 // yes - this is really a hack, if you are a front-ender and want to help, please get in touch!
 
-$(document).ready(function () {
+$(document).ready(function() {
 
     var current_selected_i;
-    var state_clicked = false;
+    var state_clicked=false;
 
     var c;
 
@@ -13,9 +13,9 @@ $(document).ready(function () {
     // redline highlight context
     var ctx;
 
-    var current_default_xpath = [];
+    var current_default_xpath;
-    var x_scale = 1;
+    var x_scale=1;
-    var y_scale = 1;
+    var y_scale=1;
     var selector_image;
     var selector_image_rect;
     var selector_data;
@@ -27,216 +27,204 @@ $(document).ready(function () {
         bootstrap_visualselector();
     });
 
-    $(document).on('keydown', function (event) {
+    $(document).on('keydown', function(event) {
         if ($("img#selector-background").is(":visible")) {
             if (event.key == "Escape") {
-                state_clicked = false;
+                state_clicked=false;
                 ctx.clearRect(0, 0, c.width, c.height);
             }
         }
     });
 
     // For when the page loads
-    if (!window.location.hash || window.location.hash != '#visualselector') {
+    if(!window.location.hash || window.location.hash != '#visualselector') {
-        $("img#selector-background").attr('src', '');
+        $("img#selector-background").attr('src','');
         return;
     }
 
     // Handle clearing button/link
-    $('#clear-selector').on('click', function (event) {
+    $('#clear-selector').on('click', function(event) {
-        if (!state_clicked) {
+        if(!state_clicked) {
             alert('Oops, Nothing selected!');
         }
-        state_clicked = false;
+        state_clicked=false;
         ctx.clearRect(0, 0, c.width, c.height);
         xctx.clearRect(0, 0, c.width, c.height);
-        $("#include_filters").val('');
+        $("#css_filter").val('');
     });
 
 
     bootstrap_visualselector();
 
 
+
     function bootstrap_visualselector() {
-        if (1) {
+        if ( 1 ) {
             // bootstrap it, this will trigger everything else
             $("img#selector-background").bind('load', function () {
                 console.log("Loaded background...");
-                c = document.getElementById("selector-canvas");
+               c = document.getElementById("selector-canvas");
                 // greyed out fill context
-                xctx = c.getContext("2d");
+               xctx = c.getContext("2d");
                 // redline highlight context
-                ctx = c.getContext("2d");
+               ctx = c.getContext("2d");
-                if ($("#include_filters").val().trim().length) {
+               current_default_xpath =$("#css_filter").val();
-                    current_default_xpath = $("#include_filters").val().split(/\r?\n/g);
+               fetch_data();
-                } else {
+               $('#selector-canvas').off("mousemove mousedown");
-                    current_default_xpath = [];
+               // screenshot_url defined in the edit.html template
-                }
-                fetch_data();
-                $('#selector-canvas').off("mousemove mousedown");
-                // screenshot_url defined in the edit.html template
             }).attr("src", screenshot_url);
         }
-        // Tell visualSelector that the image should update
-        var s = $("img#selector-background").attr('src')+"?"+ new Date().getTime();
-        $("img#selector-background").attr('src',s)
     }
 
     function fetch_data() {
-        // Image is ready
+      // Image is ready
-        $('.fetching-update-notice').html("Fetching element data..");
+      $('.fetching-update-notice').html("Fetching element data..");
 
-        $.ajax({
+      $.ajax({
-            url: watch_visual_selector_data_url,
+        url: watch_visual_selector_data_url,
-            context: document.body
+        context: document.body
-        }).done(function (data) {
+      }).done(function (data) {
-            $('.fetching-update-notice').html("Rendering..");
+        $('.fetching-update-notice').html("Rendering..");
-            selector_data = data;
+        selector_data = data;
-            console.log("Reported browser width from backend: " + data['browser_width']);
+        console.log("Reported browser width from backend: "+data['browser_width']);
-            state_clicked = false;
+        state_clicked=false;
-            set_scale();
+        set_scale();
-            reflow_selector();
+        reflow_selector();
-            $('.fetching-update-notice').fadeOut();
+        $('.fetching-update-notice').fadeOut();
-        });
+      });
     };
 
 
+
     function set_scale() {
 
-        // some things to check if the scaling doesnt work
+      // some things to check if the scaling doesnt work
-        // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
+      // - that the widths/sizes really are about the actual screen size cat elements.json |grep -o width......|sort|uniq
-        $("#selector-wrapper").show();
+      selector_image = $("img#selector-background")[0];
-        selector_image = $("img#selector-background")[0];
+      selector_image_rect = selector_image.getBoundingClientRect();
-        selector_image_rect = selector_image.getBoundingClientRect();
 
-        // make the canvas the same size as the image
+      // make the canvas the same size as the image
-        $('#selector-canvas').attr('height', selector_image_rect.height);
+      $('#selector-canvas').attr('height', selector_image_rect.height);
-        $('#selector-canvas').attr('width', selector_image_rect.width);
+      $('#selector-canvas').attr('width', selector_image_rect.width);
-        $('#selector-wrapper').attr('width', selector_image_rect.width);
+      $('#selector-wrapper').attr('width', selector_image_rect.width);
-        x_scale = selector_image_rect.width / selector_data['browser_width'];
+      x_scale = selector_image_rect.width / selector_data['browser_width'];
-        y_scale = selector_image_rect.height / selector_image.naturalHeight;
+      y_scale = selector_image_rect.height / selector_image.naturalHeight;
-        ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
+      ctx.strokeStyle = 'rgba(255,0,0, 0.9)';
-        ctx.fillStyle = 'rgba(255,0,0, 0.1)';
+      ctx.fillStyle = 'rgba(255,0,0, 0.1)';
-        ctx.lineWidth = 3;
+      ctx.lineWidth = 3;
-        console.log("scaling set  x: " + x_scale + " by y:" + y_scale);
+      console.log("scaling set  x: "+x_scale+" by y:"+y_scale);
-        $("#selector-current-xpath").css('max-width', selector_image_rect.width);
+      $("#selector-current-xpath").css('max-width', selector_image_rect.width);
     }
 
     function reflow_selector() {
-        $(window).resize(function () {
+        $(window).resize(function() {
             set_scale();
             highlight_current_selected_i();
         });
-        var selector_currnt_xpath_text = $("#selector-current-xpath span");
+      var selector_currnt_xpath_text=$("#selector-current-xpath span");
 
-        set_scale();
+      set_scale();
 
-        console.log(selector_data['size_pos'].length + " selectors found");
+      console.log(selector_data['size_pos'].length + " selectors found");
 
-        // highlight the default one if we can find it in the xPath list
+      // highlight the default one if we can find it in the xPath list
-        // or the xpath matches the default one
+      // or the xpath matches the default one
-        found = false;
+      found = false;
-        if (current_default_xpath.length) {
+      if(current_default_xpath.length) {
-            // Find the first one that matches
+          for (var i = selector_data['size_pos'].length; i!==0; i--) {
-            // @todo In the future paint all that match
+            var sel = selector_data['size_pos'][i-1];
-            for (const c of current_default_xpath) {
+            if(selector_data['size_pos'][i - 1].xpath == current_default_xpath) {
-                for (var i = selector_data['size_pos'].length; i !== 0; i--) {
+            console.log("highlighting "+current_default_xpath);
-                    if (selector_data['size_pos'][i - 1].xpath === c) {
+              current_selected_i = i-1;
-                        console.log("highlighting " + c);
+              highlight_current_selected_i();
-                        current_selected_i = i - 1;
+              found = true;
-                        highlight_current_selected_i();
+              break;
-                        found = true;
-                        break;
-                    }
-                }
-                if (found) {
-                    break;
-                }
-            }
-            if (!found) {
-                alert("Unfortunately your existing CSS/xPath Filter was no longer found!");
             }
+          }
+        if(!found) {
+          alert("Unfortunately your existing CSS/xPath Filter was no longer found!");
+        }
+      }
+
+
+      $('#selector-canvas').bind('mousemove', function (e) {
+        if(state_clicked) {
+          return;
+        }
+        ctx.clearRect(0, 0, c.width, c.height);
+        current_selected_i=null;
+
+        // Add in offset
+        if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) {
+          var targetOffset = $(e.target).offset();
+          e.offsetX = e.pageX - targetOffset.left;
+          e.offsetY = e.pageY - targetOffset.top;
         }
 
+        // Reverse order - the most specific one should be deeper/"laster"
+        // Basically, find the most 'deepest'
+        var found=0;
+        ctx.fillStyle = 'rgba(205,0,0,0.35)';
+        for (var i = selector_data['size_pos'].length; i!==0; i--) {
+          // draw all of them? let them choose somehow?
+          var sel = selector_data['size_pos'][i-1];
+          // If we are in a bounding-box
+          if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
+              &&
+              e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
 
-        $('#selector-canvas').bind('mousemove', function (e) {
+          ) {
-            if (state_clicked) {
-                return;
-            }
-            ctx.clearRect(0, 0, c.width, c.height);
-            current_selected_i = null;
 
-            // Add in offset
+            // FOUND ONE
-            if ((typeof e.offsetX === "undefined" || typeof e.offsetY === "undefined") || (e.offsetX === 0 && e.offsetY === 0)) {
-                var targetOffset = $(e.target).offset();
-                e.offsetX = e.pageX - targetOffset.left;
-                e.offsetY = e.pageY - targetOffset.top;
-            }
-
-            // Reverse order - the most specific one should be deeper/"laster"
-            // Basically, find the most 'deepest'
-            var found = 0;
-            ctx.fillStyle = 'rgba(205,0,0,0.35)';
-            for (var i = selector_data['size_pos'].length; i !== 0; i--) {
-                // draw all of them? let them choose somehow?
-                var sel = selector_data['size_pos'][i - 1];
-                // If we are in a bounding-box
-                if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
-                    &&
-                    e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
-
-                ) {
-
-                    // FOUND ONE
-                    set_current_selected_text(sel.xpath);
-                    ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-                    ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-
-                    // no need to keep digging
-                    // @todo or, O to go out/up, I to go in
-                    // or double click to go up/out the selector?
-                    current_selected_i = i - 1;
-                    found += 1;
-                    break;
-                }
-            }
-
-        }.debounce(5));
-
-        function set_current_selected_text(s) {
-            selector_currnt_xpath_text[0].innerHTML = s;
-        }
-
-        function highlight_current_selected_i() {
-            if (state_clicked) {
-                state_clicked = false;
-                xctx.clearRect(0, 0, c.width, c.height);
-                return;
-            }
-
-            var sel = selector_data['size_pos'][current_selected_i];
-            if (sel[0] == '/') {
-                // @todo - not sure just checking / is right
-                $("#include_filters").val('xpath:' + sel.xpath);
-            } else {
-                $("#include_filters").val(sel.xpath);
-            }
-            xctx.fillStyle = 'rgba(205,205,205,0.95)';
-            xctx.strokeStyle = 'rgba(225,0,0,0.9)';
-            xctx.lineWidth = 3;
-            xctx.fillRect(0, 0, c.width, c.height);
-            // Clear out what only should be seen (make a clear/clean spot)
-            xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-            xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
-            state_clicked = true;
             set_current_selected_text(sel.xpath);
+            ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+            ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
 
+            // no need to keep digging
+            // @todo or, O to go out/up, I to go in
+            // or double click to go up/out the selector?
+            current_selected_i=i-1;
+            found+=1;
+            break;
+          }
         }
 
+      }.debounce(5));
 
-        $('#selector-canvas').bind('mousedown', function (e) {
+      function set_current_selected_text(s) {
-            highlight_current_selected_i();
+        selector_currnt_xpath_text[0].innerHTML=s;
-        });
+      }
+
+      function highlight_current_selected_i() {
+        if(state_clicked) {
+          state_clicked=false;
+          xctx.clearRect(0,0,c.width, c.height);
+          return;
+        }
+
+        var sel = selector_data['size_pos'][current_selected_i];
+        if (sel[0] == '/') {
+        // @todo - not sure just checking / is right
+            $("#css_filter").val('xpath:'+sel.xpath);
+        } else {
+            $("#css_filter").val(sel.xpath);
+        }
+        xctx.fillStyle = 'rgba(205,205,205,0.95)';
+        xctx.strokeStyle = 'rgba(225,0,0,0.9)';
+        xctx.lineWidth = 3;
+        xctx.fillRect(0,0,c.width, c.height);
+        // Clear out what only should be seen (make a clear/clean spot)
+        xctx.clearRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+        xctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+        state_clicked=true;
+        set_current_selected_text(sel.xpath);
+
+      }
+
+
+      $('#selector-canvas').bind('mousedown', function (e) {
+        highlight_current_selected_i();
+      });
     }
 
-});
+});

changedetectionio/static/js/watch-settings.js

@@ -30,11 +30,4 @@ $(document).ready(function() {
     });
     toggle();
 
-    $('#notification-setting-reset-to-default').click(function (e) {
-        $('#notification_title').val('');
-        $('#notification_body').val('');
-        $('#notification_format').val('System default');
-        $('#notification_urls').val('');
-        e.preventDefault();
-    });
 });

changedetectionio/static/styles/diff.css

@@ -1,148 +1,10 @@
-/**
- * CSS custom properties (aka variables).
- */
-:root {
-  --color-white: #fff;
-  --color-grey-50: #111;
-  --color-grey-100: #262626;
-  --color-grey-200: #333;
-  --color-grey-300: #444;
-  --color-grey-325: #555;
-  --color-grey-350: #565d64;
-  --color-grey-400: #666;
-  --color-grey-500: #777;
-  --color-grey-600: #999;
-  --color-grey-700: #cbcbcb;
-  --color-grey-750: #ddd;
-  --color-grey-800: #e0e0e0;
-  --color-grey-850: #eee;
-  --color-grey-900: #f2f2f2;
-  --color-black: #000;
-  --color-dark-red: #a00;
-  --color-light-red: #dd0000;
-  --color-background-page: var(--color-grey-100);
-  --color-background-gradient-first: #5ad8f7;
-  --color-background-gradient-second: #2f50af;
-  --color-background-gradient-third: #9150bf;
-  --color-background: var(--color-white);
-  --color-text: var(--color-grey-200);
-  --color-link: #1b98f8;
-  --color-menu-accent: #ed5900;
-  --color-background-code: var(--color-grey-850);
-  --color-error: var(--color-dark-red);
-  --color-error-input: #ffebeb;
-  --color-error-list: var(--color-light-red);
-  --color-table-background: var(--color-background);
-  --color-table-stripe: var(--color-grey-900);
-  --color-text-tab: var(--color-white);
-  --color-background-tab: rgba(255, 255, 255, 0.2);
-  --color-background-tab-hover: rgba(255, 255, 255, 0.5);
-  --color-text-tab-active: #222;
-  --color-api-key: #0078e7;
-  --color-background-button-primary: #0078e7;
-  --color-background-button-green: #42dd53;
-  --color-background-button-red: #dd4242;
-  --color-background-button-success: rgb(28, 184, 65);
-  --color-background-button-error: rgb(202, 60, 60);
-  --color-text-button-error: var(--color-white);
-  --color-background-button-warning: rgb(202, 60, 60);
-  --color-text-button-warning: var(--color-white);
-  --color-background-button-secondary: rgb(66, 184, 221);
-  --color-background-button-cancel: rgb(200, 200, 200);
-  --color-text-button: var(--color-white);
-  --color-background-button-tag: rgb(99, 99, 99);
-  --color-background-snapshot-age: #dfdfdf;
-  --color-error-text-snapshot-age: var(--color-white);
-  --color-error-background-snapshot-age: #ff0000;
-  --color-background-button-tag-active: #9c9c9c;
-  --color-text-messages: var(--color-white);
-  --color-background-messages-message: rgba(255, 255, 255, .2);
-  --color-background-messages-error: rgba(255, 1, 1, .5);
-  --color-background-messages-notice: rgba(255, 255, 255, .5);
-  --color-border-notification: #ccc;
-  --color-background-checkbox-operations: rgba(0, 0, 0, 0.05);
-  --color-warning: #ff3300;
-  --color-border-warning: var(--color-warning);
-  --color-text-legend: var(--color-white);
-  --color-link-new-version: #e07171;
-  --color-last-checked: #bbb;
-  --color-text-footer: #444;
-  --color-border-watch-table-cell: #eee;
-  --color-text-watch-tag-list: #e70069;
-  --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
-  --color-background-new-watch-input: var(--color-white);
-  --color-text-new-watch-input: var(--color-text);
-  --color-border-input: var(--color-grey-500);
-  --color-shadow-input: var(--color-grey-400);
-  --color-background-input: var(--color-white);
-  --color-text-input: var(--color-text);
-  --color-text-input-description: var(--color-grey-500);
-  --color-text-input-placeholder: var(--color-grey-600);
-  --color-background-table-thead: var(--color-grey-800);
-  --color-border-table-cell: var(--color-grey-700);
-  --color-text-menu-heading: var(--color-grey-350);
-  --color-text-menu-link: var(--color-grey-500);
-  --color-background-menu-link-hover: var(--color-grey-850);
-  --color-text-menu-link-hover: var(--color-grey-300);
-  --color-shadow-jump: var(--color-grey-500);
-  --color-icon-github: var(--color-black);
-  --color-icon-github-hover: var(--color-grey-300);
-  --color-watch-table-error: var(--color-dark-red);
-  --color-watch-table-row-text: var(--color-grey-100); }
-
-html[data-darkmode="true"] {
-  --color-link: #59bdfb;
-  --color-text: var(--color-white);
-  --color-background-gradient-first: #3f90a5;
-  --color-background-gradient-second: #1e316c;
-  --color-background-gradient-third: #4d2c64;
-  --color-background-new-watch-input: var(--color-grey-100);
-  --color-text-new-watch-input: var(--color-text);
-  --color-background-table-thead: var(--color-grey-200);
-  --color-table-background: var(--color-grey-300);
-  --color-table-stripe: var(--color-grey-325);
-  --color-background: var(--color-grey-300);
-  --color-text-menu-heading: var(--color-grey-850);
-  --color-text-menu-link: var(--color-grey-800);
-  --color-border-table-cell: var(--color-grey-400);
-  --color-text-tab-active: var(--color-text);
-  --color-border-input: var(--color-grey-400);
-  --color-shadow-input: var(--color-grey-50);
-  --color-background-input: var(--color-grey-350);
-  --color-text-input-description: var(--color-grey-600);
-  --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: #fa3e92;
-  --color-background-code: var(--color-grey-200);
-  --color-background-tab: rgba(0, 0, 0, 0.2);
-  --color-background-tab-hover: rgba(0, 0, 0, 0.5);
-  --color-background-snapshot-age: var(--color-grey-200);
-  --color-shadow-jump: var(--color-grey-200);
-  --color-icon-github: var(--color-white);
-  --color-icon-github-hover: var(--color-grey-700);
-  --color-watch-table-error: var(--color-light-red);
-  --color-watch-table-row-text: var(--color-grey-800); }
-  html[data-darkmode="true"] .icon-spread {
-    filter: hue-rotate(-10deg) brightness(1.5); }
-  html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
-  html[data-darkmode="true"] .watch-table .current-diff-url::after {
-    filter: invert(0.5) hue-rotate(10deg) brightness(2); }
-  html[data-darkmode="true"] .watch-table .watch-controls .state-off img {
-    opacity: 0.3; }
-  html[data-darkmode="true"] .watch-table .watch-controls .state-on img {
-    opacity: 1.0; }
-  html[data-darkmode="true"] .watch-table .unviewed {
-    color: #fff; }
-    html[data-darkmode="true"] .watch-table .unviewed.error {
-      color: var(--color-watch-table-error); }
-
 #diff-ui {
-  background: var(--color-background);
+  background: #fff;
   padding: 2em;
   margin-left: 1em;
   margin-right: 1em;
-  border-radius: 5px; }
+  border-radius: 5px;
-  #diff-ui #text {
+  font-size: 11px; }
-    font-size: 11px; }
   #diff-ui table {
     table-layout: fixed;
     width: 100%; }
@@ -183,10 +45,6 @@ ins {
     margin-left: 1em;
     display: inline-block;
     font-weight: normal; }
-  #settings del {
-    padding: 0.5em; }
-  #settings ins {
-    padding: 0.5em; }
 
 .source {
   position: absolute;

changedetectionio/static/styles/diff.scss

@@ -0,0 +1,96 @@
+#diff-ui {
+
+    background: #fff;
+    padding: 2em;
+    margin-left: 1em;
+    margin-right: 1em;
+    border-radius: 5px;
+    font-size: 11px;
+
+    table {
+        table-layout: fixed;
+        width: 100%;
+    }
+    td {
+        padding: 3px 4px;
+        border: 1px solid transparent;
+        vertical-align: top;
+        font: 1em monospace;
+        text-align: left;
+    }
+    pre {
+            white-space: pre-wrap;
+    }
+}
+h1 {
+	display: inline;
+	font-size: 100%;
+}
+del {
+	text-decoration: none;
+	color: #b30000;
+	background: #fadad7;
+}
+
+ins {
+	background: #eaf2c2;
+	color: #406619;
+	text-decoration: none;
+}
+
+#result {
+	white-space: pre-wrap;
+}
+
+#settings {
+    background: rgba(0,0,0,.05);
+    padding: 1em;
+    border-radius: 10px;
+    margin-bottom: 1em;
+    color: #fff;
+    font-size: 80%;
+    label {
+	    margin-left: 1em;
+	    display: inline-block;
+	    font-weight: normal;
+    }
+}
+
+.source {
+	position: absolute;
+	right: 1%;
+	top: .2em;
+}
+
+@-moz-document url-prefix() {
+	body {
+		height: 99%; /* Hide scroll bar in Firefox */
+	}
+}
+
+td#diff-col div {
+    text-align: justify;
+    white-space: pre-wrap;
+}
+
+.ignored {
+    background-color: #ccc;
+   /*  border: #0d91fa 1px solid; */
+    opacity: 0.7;
+}
+
+.triggered {
+    background-color: #1b98f8;
+}
+
+/* ignored and triggered? make it obvious error */
+.ignored.triggered {
+  background-color: #ff0000;
+}
+
+.tab-pane-inner#screenshot {
+  text-align: center;
+  img {
+    max-width: 99%;
+  }
+}

changedetectionio/static/styles/package.json

@@ -4,8 +4,7 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "watch": "node-sass -w scss -o .",
+    "build": "node-sass styles.scss -o .;node-sass diff.scss -o ."
-    "build": "node-sass scss -o ."
   },
   "author": "",
   "license": "ISC",

changedetectionio/static/styles/scss/diff.scss

@@ -1,121 +0,0 @@
-@import "parts/_variables.scss";
-
-#diff-ui {
-
-  background: var(--color-background);
-  padding: 2em;
-  margin-left: 1em;
-  margin-right: 1em;
-  border-radius: 5px;
-
-  // The first tab 'text' diff
-  #text {
-    font-size: 11px;
-  }
-
-  table {
-    table-layout: fixed;
-    width: 100%;
-  }
-
-  td {
-    padding: 3px 4px;
-    border: 1px solid transparent;
-    vertical-align: top;
-    font: 1em monospace;
-    text-align: left;
-  }
-
-  pre {
-    white-space: pre-wrap;
-  }
-}
-
-h1 {
-  display: inline;
-  font-size: 100%;
-}
-
-del {
-  text-decoration: none;
-  color: #b30000;
-  background: #fadad7;
-}
-
-ins {
-  background: #eaf2c2;
-  color: #406619;
-  text-decoration: none;
-}
-
-#result {
-  white-space: pre-wrap;
-
-  .change {
-    span {}
-  }
-}
-
-#settings {
-  background: rgba(0, 0, 0, .05);
-  padding: 1em;
-  border-radius: 10px;
-  margin-bottom: 1em;
-  color: #fff;
-  font-size: 80%;
-
-  label {
-    margin-left: 1em;
-    display: inline-block;
-    font-weight: normal;
-  }
-
-  del {
-    padding: 0.5em;
-  }
-
-  ins {
-    padding: 0.5em;
-  }
-}
-
-.source {
-  position: absolute;
-  right: 1%;
-  top: .2em;
-}
-
-@-moz-document url-prefix() {
-  body {
-    height: 99%;
-    /* Hide scroll bar in Firefox */
-  }
-}
-
-td#diff-col div {
-  text-align: justify;
-  white-space: pre-wrap;
-}
-
-.ignored {
-  background-color: #ccc;
-  /*  border: #0d91fa 1px solid; */
-  opacity: 0.7;
-}
-
-.triggered {
-  background-color: #1b98f8;
-}
-
-/* ignored and triggered? make it obvious error */
-.ignored.triggered {
-  background-color: #ff0000;
-}
-
-.tab-pane-inner#screenshot {
-  text-align: center;
-
-  img {
-    max-width: 99%;
-  }
-}

changedetectionio/static/styles/scss/parts/_browser-steps.scss

@@ -1,95 +0,0 @@
-
-#browser_steps {
-  /* convert rows to horizontal cells */
-  th {
-    display: none;
-  }
-
-  li {
-    &:not(:first-child) {
-      &:hover {
-        opacity: 1.0;
-      }
-    }
-    list-style: decimal;
-    padding: 5px;
-    .control {
-      padding-left: 5px;
-      padding-right: 5px;
-      a {
-        font-size: 70%;
-      }
-    }
-    &.empty {
-      padding: 0px;
-      opacity: 0.35;
-      .control {
-        display: none;
-      }
-    }
-    &:hover {
-      background: #eee;
-    }
-    > label {
-      display: none;
-    }
-  }
-}
-
-#browser-steps-fieldlist {
-  height: 100%;
-  overflow-y: scroll;
-}
-
-#browser-steps .flex-wrapper {
-  display: flex;
-  flex-flow: row;
-  height: 600px; /*@todo make this dynamic */
-}
-
-/*  this is duplicate :( */
-#browsersteps-selector-wrapper {
-  height: 100%;
-  width: 100%;
-  overflow-y: scroll;
-  position: relative;
-  //width: 100%;
-  > img {
-    position: absolute;
-    max-width: 100%;
-  }
-
-  > canvas {
-    position: relative;
-    max-width: 100%;
-
-    &:hover {
-      cursor: pointer;
-    }
-  }
-
-  .loader {
-    position: absolute;
-    left: 50%;
-    top: 50%;
-    transform: translate(-50%, -50%);
-    margin-left: -40px;
-    z-index: 100;
-    max-width: 350px;
-    text-align: center;
-  }
-
-  /* nice tall skinny one */
-  .spinner, .spinner:after {
-    width: 80px;
-    height: 80px;
-    font-size: 3px;
-  }
-
-  #browsersteps-click-start {
-    &:hover {
-      cursor: pointer;
-    }
-    color: var(--color-grey-400);
-  }
-}

changedetectionio/static/styles/scss/parts/_spinners.scss

@@ -1,44 +0,0 @@
-
-/* spinner */
-.spinner,
-.spinner:after {
-  border-radius: 50%;
-  width: 10px;
-  height: 10px;
-}
-.spinner {
-  margin: 0px auto;
-  font-size: 3px;
-  vertical-align: middle;
-  display: inline-block;
-  text-indent: -9999em;
-  border-top: 1.1em solid rgba(38,104,237, 0.2);
-  border-right: 1.1em solid rgba(38,104,237, 0.2);
-  border-bottom: 1.1em solid rgba(38,104,237, 0.2);
-  border-left: 1.1em solid #2668ed;
-  -webkit-transform: translateZ(0);
-  -ms-transform: translateZ(0);
-  transform: translateZ(0);
-  -webkit-animation: load8 1.1s infinite linear;
-  animation: load8 1.1s infinite linear;
-}
-@-webkit-keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg);
-  }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg);
-  }
-}
-@keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg);
-  }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg);
-  }
-}

changedetectionio/static/styles/scss/parts/_variables.scss

@@ -1,175 +0,0 @@
-/**
- * CSS custom properties (aka variables).
- */
-
-:root {
-  --color-white: #fff;
-  --color-grey-50: #111;
-  --color-grey-100: #262626;
-  --color-grey-200: #333;
-  --color-grey-300: #444;
-  --color-grey-325: #555;
-  --color-grey-350: #565d64;
-  --color-grey-400: #666;
-  --color-grey-500: #777;
-  --color-grey-600: #999;
-  --color-grey-700: #cbcbcb;
-  --color-grey-750: #ddd;
-  --color-grey-800: #e0e0e0;
-  --color-grey-850: #eee;
-  --color-grey-900: #f2f2f2;
-  --color-black: #000;
-  --color-dark-red: #a00;
-  --color-light-red: #dd0000;
-
-  --color-background-page: var(--color-grey-100);
-  --color-background-gradient-first: #5ad8f7;
-  --color-background-gradient-second: #2f50af;
-  --color-background-gradient-third: #9150bf;
-  --color-background: var(--color-white);
-  --color-text: var(--color-grey-200);
-  --color-link: #1b98f8;
-  --color-menu-accent: #ed5900;
-  --color-background-code: var(--color-grey-850);
-  --color-error: var(--color-dark-red);
-  --color-error-input: #ffebeb;
-  --color-error-list: var(--color-light-red);
-  --color-table-background: var(--color-background);
-  --color-table-stripe: var(--color-grey-900);
-  --color-text-tab: var(--color-white);
-  --color-background-tab: rgba(255, 255, 255, 0.2);
-  --color-background-tab-hover: rgba(255, 255, 255, 0.5);
-  --color-text-tab-active: #222;
-  --color-api-key: #0078e7;
-
-  --color-background-button-primary: #0078e7;
-  --color-background-button-green: #42dd53;
-  --color-background-button-red: #dd4242;
-  --color-background-button-success: rgb(28, 184, 65);
-  --color-background-button-error: rgb(202, 60, 60);
-  --color-text-button-error: var(--color-white);
-  --color-background-button-warning: rgb(202, 60, 60);
-  --color-text-button-warning: var(--color-white);
-  --color-background-button-secondary: rgb(66, 184, 221);
-  --color-background-button-cancel: rgb(200, 200, 200);
-  --color-text-button: var(--color-white);
-  --color-background-button-tag: rgb(99, 99, 99);
-  --color-background-snapshot-age: #dfdfdf;
-  --color-error-text-snapshot-age: var(--color-white);
-  --color-error-background-snapshot-age: #ff0000;
-  --color-background-button-tag-active: #9c9c9c;
-
-  --color-text-messages: var(--color-white);
-  --color-background-messages-message: rgba(255, 255, 255, .2);
-  --color-background-messages-error: rgba(255, 1, 1, .5);
-  --color-background-messages-notice: rgba(255, 255, 255, .5);
-  --color-border-notification: #ccc;
-
-  --color-background-checkbox-operations: rgba(0, 0, 0, 0.05);
-  --color-warning: #ff3300;
-  --color-border-warning: var(--color-warning);
-  --color-text-legend: var(--color-white);
-
-  --color-link-new-version: #e07171;
-  --color-last-checked: #bbb;
-  --color-text-footer: #444;
-  --color-border-watch-table-cell: #eee;
-
-  --color-text-watch-tag-list: #e70069;
-  --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
-  --color-background-new-watch-input: var(--color-white);
-  --color-text-new-watch-input: var(--color-text);
-
-  --color-border-input: var(--color-grey-500);
-  --color-shadow-input: var(--color-grey-400);
-  --color-background-input: var(--color-white);
-  --color-text-input: var(--color-text);
-  --color-text-input-description: var(--color-grey-500);
-  --color-text-input-placeholder: var(--color-grey-600);
-
-  --color-background-table-thead: var(--color-grey-800);
-  --color-border-table-cell: var(--color-grey-700);
-
-  --color-text-menu-heading: var(--color-grey-350);
-  --color-text-menu-link: var(--color-grey-500);
-  --color-background-menu-link-hover: var(--color-grey-850);
-  --color-text-menu-link-hover: var(--color-grey-300);
-
-  --color-shadow-jump: var(--color-grey-500);
-  --color-icon-github: var(--color-black);
-  --color-icon-github-hover: var(--color-grey-300);
-
-  --color-watch-table-error: var(--color-dark-red);
-  --color-watch-table-row-text: var(--color-grey-100);
-}
-
-html[data-darkmode="true"] {
-  --color-link: #59bdfb;
-  --color-text: var(--color-white);
-
-  --color-background-gradient-first: #3f90a5;
-  --color-background-gradient-second: #1e316c;
-  --color-background-gradient-third: #4d2c64;
-
-  --color-background-new-watch-input: var(--color-grey-100);
-  --color-text-new-watch-input: var(--color-text);
-  --color-background-table-thead: var(--color-grey-200);
-  --color-table-background: var(--color-grey-300);
-  --color-table-stripe: var(--color-grey-325);
-  --color-background: var(--color-grey-300);
-  --color-text-menu-heading: var(--color-grey-850);
-  --color-text-menu-link: var(--color-grey-800);
-  --color-border-table-cell: var(--color-grey-400);
-  --color-text-tab-active: var(--color-text);
-
-  --color-border-input: var(--color-grey-400);
-  --color-shadow-input: var(--color-grey-50);
-  --color-background-input: var(--color-grey-350);
-  --color-text-input-description: var(--color-grey-600);
-  --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: #fa3e92;
-  --color-background-code: var(--color-grey-200);
-
-  --color-background-tab: rgba(0, 0, 0, 0.2);
-  --color-background-tab-hover: rgba(0, 0, 0, 0.5);
-
-  --color-background-snapshot-age: var(--color-grey-200);
-  --color-shadow-jump: var(--color-grey-200);
-  --color-icon-github: var(--color-white);
-  --color-icon-github-hover: var(--color-grey-700);
-  --color-watch-table-error: var(--color-light-red);
-  --color-watch-table-row-text: var(--color-grey-800);
-
-
-  .icon-spread {
-    filter: hue-rotate(-10deg) brightness(1.5);
-  }
-
-  .watch-table {
-
-    .title-col a[target="_blank"]::after,
-    .current-diff-url::after {
-      filter: invert(.5) hue-rotate(10deg) brightness(2);
-    }
-
-    .watch-controls {
-      .state-off {
-        img {
-          opacity: 0.3;
-        }
-      }
-      .state-on {
-        img {
-          opacity: 1.0;
-        }
-      }
-    }
-
-    .unviewed {
-      color: #fff;
-      &.error {
-        color: var(--color-watch-table-error);
-      }
-    }
-  }
-}

changedetectionio/static/styles/styles.css

@@ -1,249 +1,9 @@
 /*
  * -- BASE STYLES --
+ * Most of these are inherited from Base, but I want to change a few.
+ * nvm use v14.18.1 && npm install && npm run build
+ * or npm run watch
  */
-/**
- * CSS custom properties (aka variables).
- */
-:root {
-  --color-white: #fff;
-  --color-grey-50: #111;
-  --color-grey-100: #262626;
-  --color-grey-200: #333;
-  --color-grey-300: #444;
-  --color-grey-325: #555;
-  --color-grey-350: #565d64;
-  --color-grey-400: #666;
-  --color-grey-500: #777;
-  --color-grey-600: #999;
-  --color-grey-700: #cbcbcb;
-  --color-grey-750: #ddd;
-  --color-grey-800: #e0e0e0;
-  --color-grey-850: #eee;
-  --color-grey-900: #f2f2f2;
-  --color-black: #000;
-  --color-dark-red: #a00;
-  --color-light-red: #dd0000;
-  --color-background-page: var(--color-grey-100);
-  --color-background-gradient-first: #5ad8f7;
-  --color-background-gradient-second: #2f50af;
-  --color-background-gradient-third: #9150bf;
-  --color-background: var(--color-white);
-  --color-text: var(--color-grey-200);
-  --color-link: #1b98f8;
-  --color-menu-accent: #ed5900;
-  --color-background-code: var(--color-grey-850);
-  --color-error: var(--color-dark-red);
-  --color-error-input: #ffebeb;
-  --color-error-list: var(--color-light-red);
-  --color-table-background: var(--color-background);
-  --color-table-stripe: var(--color-grey-900);
-  --color-text-tab: var(--color-white);
-  --color-background-tab: rgba(255, 255, 255, 0.2);
-  --color-background-tab-hover: rgba(255, 255, 255, 0.5);
-  --color-text-tab-active: #222;
-  --color-api-key: #0078e7;
-  --color-background-button-primary: #0078e7;
-  --color-background-button-green: #42dd53;
-  --color-background-button-red: #dd4242;
-  --color-background-button-success: rgb(28, 184, 65);
-  --color-background-button-error: rgb(202, 60, 60);
-  --color-text-button-error: var(--color-white);
-  --color-background-button-warning: rgb(202, 60, 60);
-  --color-text-button-warning: var(--color-white);
-  --color-background-button-secondary: rgb(66, 184, 221);
-  --color-background-button-cancel: rgb(200, 200, 200);
-  --color-text-button: var(--color-white);
-  --color-background-button-tag: rgb(99, 99, 99);
-  --color-background-snapshot-age: #dfdfdf;
-  --color-error-text-snapshot-age: var(--color-white);
-  --color-error-background-snapshot-age: #ff0000;
-  --color-background-button-tag-active: #9c9c9c;
-  --color-text-messages: var(--color-white);
-  --color-background-messages-message: rgba(255, 255, 255, .2);
-  --color-background-messages-error: rgba(255, 1, 1, .5);
-  --color-background-messages-notice: rgba(255, 255, 255, .5);
-  --color-border-notification: #ccc;
-  --color-background-checkbox-operations: rgba(0, 0, 0, 0.05);
-  --color-warning: #ff3300;
-  --color-border-warning: var(--color-warning);
-  --color-text-legend: var(--color-white);
-  --color-link-new-version: #e07171;
-  --color-last-checked: #bbb;
-  --color-text-footer: #444;
-  --color-border-watch-table-cell: #eee;
-  --color-text-watch-tag-list: #e70069;
-  --color-background-new-watch-form: rgba(0, 0, 0, 0.05);
-  --color-background-new-watch-input: var(--color-white);
-  --color-text-new-watch-input: var(--color-text);
-  --color-border-input: var(--color-grey-500);
-  --color-shadow-input: var(--color-grey-400);
-  --color-background-input: var(--color-white);
-  --color-text-input: var(--color-text);
-  --color-text-input-description: var(--color-grey-500);
-  --color-text-input-placeholder: var(--color-grey-600);
-  --color-background-table-thead: var(--color-grey-800);
-  --color-border-table-cell: var(--color-grey-700);
-  --color-text-menu-heading: var(--color-grey-350);
-  --color-text-menu-link: var(--color-grey-500);
-  --color-background-menu-link-hover: var(--color-grey-850);
-  --color-text-menu-link-hover: var(--color-grey-300);
-  --color-shadow-jump: var(--color-grey-500);
-  --color-icon-github: var(--color-black);
-  --color-icon-github-hover: var(--color-grey-300);
-  --color-watch-table-error: var(--color-dark-red);
-  --color-watch-table-row-text: var(--color-grey-100); }
-
-html[data-darkmode="true"] {
-  --color-link: #59bdfb;
-  --color-text: var(--color-white);
-  --color-background-gradient-first: #3f90a5;
-  --color-background-gradient-second: #1e316c;
-  --color-background-gradient-third: #4d2c64;
-  --color-background-new-watch-input: var(--color-grey-100);
-  --color-text-new-watch-input: var(--color-text);
-  --color-background-table-thead: var(--color-grey-200);
-  --color-table-background: var(--color-grey-300);
-  --color-table-stripe: var(--color-grey-325);
-  --color-background: var(--color-grey-300);
-  --color-text-menu-heading: var(--color-grey-850);
-  --color-text-menu-link: var(--color-grey-800);
-  --color-border-table-cell: var(--color-grey-400);
-  --color-text-tab-active: var(--color-text);
-  --color-border-input: var(--color-grey-400);
-  --color-shadow-input: var(--color-grey-50);
-  --color-background-input: var(--color-grey-350);
-  --color-text-input-description: var(--color-grey-600);
-  --color-text-input-placeholder: var(--color-grey-600);
-  --color-text-watch-tag-list: #fa3e92;
-  --color-background-code: var(--color-grey-200);
-  --color-background-tab: rgba(0, 0, 0, 0.2);
-  --color-background-tab-hover: rgba(0, 0, 0, 0.5);
-  --color-background-snapshot-age: var(--color-grey-200);
-  --color-shadow-jump: var(--color-grey-200);
-  --color-icon-github: var(--color-white);
-  --color-icon-github-hover: var(--color-grey-700);
-  --color-watch-table-error: var(--color-light-red);
-  --color-watch-table-row-text: var(--color-grey-800); }
-  html[data-darkmode="true"] .icon-spread {
-    filter: hue-rotate(-10deg) brightness(1.5); }
-  html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
-  html[data-darkmode="true"] .watch-table .current-diff-url::after {
-    filter: invert(0.5) hue-rotate(10deg) brightness(2); }
-  html[data-darkmode="true"] .watch-table .watch-controls .state-off img {
-    opacity: 0.3; }
-  html[data-darkmode="true"] .watch-table .watch-controls .state-on img {
-    opacity: 1.0; }
-  html[data-darkmode="true"] .watch-table .unviewed {
-    color: #fff; }
-    html[data-darkmode="true"] .watch-table .unviewed.error {
-      color: var(--color-watch-table-error); }
-
-/* spinner */
-.spinner,
-.spinner:after {
-  border-radius: 50%;
-  width: 10px;
-  height: 10px; }
-
-.spinner {
-  margin: 0px auto;
-  font-size: 3px;
-  vertical-align: middle;
-  display: inline-block;
-  text-indent: -9999em;
-  border-top: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-right: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-left: 1.1em solid #2668ed;
-  -webkit-transform: translateZ(0);
-  -ms-transform: translateZ(0);
-  transform: translateZ(0);
-  -webkit-animation: load8 1.1s infinite linear;
-  animation: load8 1.1s infinite linear; }
-
-@-webkit-keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg); }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg); } }
-
-@keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg); }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg); } }
-
-#browser_steps {
-  /* convert rows to horizontal cells */ }
-  #browser_steps th {
-    display: none; }
-  #browser_steps li {
-    list-style: decimal;
-    padding: 5px; }
-    #browser_steps li:not(:first-child):hover {
-      opacity: 1.0; }
-    #browser_steps li .control {
-      padding-left: 5px;
-      padding-right: 5px; }
-      #browser_steps li .control a {
-        font-size: 70%; }
-    #browser_steps li.empty {
-      padding: 0px;
-      opacity: 0.35; }
-      #browser_steps li.empty .control {
-        display: none; }
-    #browser_steps li:hover {
-      background: #eee; }
-    #browser_steps li > label {
-      display: none; }
-
-#browser-steps-fieldlist {
-  height: 100%;
-  overflow-y: scroll; }
-
-#browser-steps .flex-wrapper {
-  display: flex;
-  flex-flow: row;
-  height: 600px;
-  /*@todo make this dynamic */ }
-
-/*  this is duplicate :( */
-#browsersteps-selector-wrapper {
-  height: 100%;
-  width: 100%;
-  overflow-y: scroll;
-  position: relative;
-  /* nice tall skinny one */ }
-  #browsersteps-selector-wrapper > img {
-    position: absolute;
-    max-width: 100%; }
-  #browsersteps-selector-wrapper > canvas {
-    position: relative;
-    max-width: 100%; }
-    #browsersteps-selector-wrapper > canvas:hover {
-      cursor: pointer; }
-  #browsersteps-selector-wrapper .loader {
-    position: absolute;
-    left: 50%;
-    top: 50%;
-    transform: translate(-50%, -50%);
-    margin-left: -40px;
-    z-index: 100;
-    max-width: 350px;
-    text-align: center; }
-  #browsersteps-selector-wrapper .spinner, #browsersteps-selector-wrapper .spinner:after {
-    width: 80px;
-    height: 80px;
-    font-size: 3px; }
-  #browsersteps-selector-wrapper #browsersteps-click-start {
-    color: var(--color-grey-400); }
-    #browsersteps-selector-wrapper #browsersteps-click-start:hover {
-      cursor: pointer; }
-
 .arrow {
   border: solid #1b98f8;
   border-width: 0 2px 2px 0;
@@ -263,70 +23,28 @@ html[data-darkmode="true"] {
     -webkit-transform: rotate(45deg); }
 
 body {
-  color: var(--color-text);
+  color: #333;
-  background: var(--color-background-page); }
+  background: #262626; }
-
-.visually-hidden {
-  clip: rect(0 0 0 0);
-  clip-path: inset(50%);
-  height: 1px;
-  overflow: hidden;
-  position: absolute;
-  white-space: nowrap;
-  width: 1px; }
 
 .pure-table-even {
-  background: var(--color-background); }
+  background: #fff; }
 
 /* Some styles from https://css-tricks.com/ */
 a {
   text-decoration: none;
-  color: var(--color-link); }
+  color: #1b98f8; }
 
 a.github-link {
-  color: var(--color-icon-github);
+  color: #fff; }
-  margin: 0 1rem 0 0.5rem; }
-  a.github-link svg {
-    fill: currentColor; }
-  a.github-link:hover {
-    color: var(--color-icon-github-hover); }
-
-button.toggle-theme {
-  width: 4rem;
-  background: transparent;
-  border: none;
-  cursor: pointer;
-  color: var(--color-icon-github); }
-  button.toggle-theme:hover {
-    color: var(--color-icon-github-hover); }
-  button.toggle-theme svg {
-    fill: currentColor; }
-  button.toggle-theme .icon-light {
-    display: block; }
-  button.toggle-theme .icon-dark {
-    display: none; }
-  button.toggle-theme.dark .icon-light {
-    display: none; }
-  button.toggle-theme.dark .icon-dark {
-    display: block; }
 
 .pure-menu-horizontal {
-  background: var(--color-background);
+  background: #fff;
   padding: 5px;
   display: flex;
   justify-content: space-between;
-  border-bottom: 2px solid var(--color-menu-accent);
+  border-bottom: 2px solid #ed5900;
   align-items: center; }
 
-.pure-menu-heading {
-  color: var(--color-text-menu-heading); }
-
-.pure-menu-link {
-  color: var(--color-text-menu-link); }
-  .pure-menu-link:hover {
-    background-color: var(--color-background-menu-link-hover);
-    color: var(--color-text-menu-link-hover); }
-
 section.content {
   padding-top: 5em;
   padding-bottom: 1em;
@@ -336,24 +54,21 @@ section.content {
   justify-content: center; }
 
 code {
-  background: var(--color-background-code);
+  background: #eee; }
-  color: var(--color-text); }
 
 /* table related */
 .watch-table {
   width: 100%;
   font-size: 80%; }
-  .watch-table tr {
+  .watch-table tr.unviewed {
-    color: var(--color-watch-table-row-text); }
+    font-weight: bold; }
-    .watch-table tr.unviewed {
+  .watch-table .error {
-      font-weight: bold; }
+    color: #a00; }
-    .watch-table tr.error {
-      color: var(--color-watch-table-error); }
   .watch-table td {
     white-space: nowrap; }
-    .watch-table td.title-col {
+  .watch-table td.title-col {
-      word-break: break-all;
+    word-break: break-all;
-      white-space: normal; }
+    white-space: normal; }
   .watch-table th {
     white-space: nowrap; }
     .watch-table th a {
@@ -362,13 +77,12 @@ code {
         font-weight: bolder; }
       .watch-table th a.inactive .arrow {
         display: none; }
-  .watch-table .title-col a[target="_blank"]::after,
+  .watch-table .title-col a[target="_blank"]::after, .watch-table .current-diff-url::after {
-  .watch-table .current-diff-url::after {
     content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
     margin: 0 3px 0 5px; }
 
 .watch-tag-list {
-  color: var(--color-text-watch-tag-list);
+  color: #e70069;
   white-space: nowrap; }
 
 .box {
@@ -391,10 +105,9 @@ code {
 
 body:after {
   content: "";
-  background: linear-gradient(130deg, var(--color-background-gradient-first), var(--color-background-gradient-second) 41.07%, var(--color-background-gradient-third) 84.05%); }
+  background: linear-gradient(130deg, #5ad8f7, #2f50af 41.07%, #9150bf 84.05%); }
 
-body:after,
+body:after, body:before {
-body:before {
   display: block;
   height: 650px;
   position: absolute;
@@ -410,8 +123,7 @@ body::before {
   content: "";
   background-size: cover; }
 
-body:after,
+body:after, body:before {
-body:before {
   -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
   clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%); }
 
@@ -420,103 +132,87 @@ body:before {
 
 .fetch-error {
   padding-top: 1em;
-  font-size: 80%;
+  font-size: 60%;
   max-width: 400px;
   display: block; }
 
-.pure-button-primary,
-a.pure-button-primary,
-.pure-button-selected,
-a.pure-button-selected {
-  background-color: var(--color-background-button-primary); }
-
 .button-secondary {
-  color: var(--color-text-button);
+  color: white;
   border-radius: 4px;
   text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2); }
 
 .button-success {
-  background: var(--color-background-button-success); }
+  background: #1cb841;
+  /* this is a green */ }
 
 .button-tag {
-  background: var(--color-background-button-tag);
+  background: #636363;
-  color: var(--color-text-button);
+  color: #fff;
   font-size: 65%;
   border-bottom-left-radius: initial;
   border-bottom-right-radius: initial; }
   .button-tag.active {
-    background: var(--color-background-button-tag-active);
+    background: #9c9c9c;
     font-weight: bold; }
 
 .button-error {
-  background: var(--color-background-button-error);
+  background: #ca3c3c;
-  color: var(--color-text-button-error); }
+  /* this is a maroon */ }
 
 .button-warning {
-  background: var(--color-background-button-warning);
+  background: #df7514;
-  color: var(--color-text-button-warning); }
+  /* this is an orange */ }
 
 .button-secondary {
-  background: var(--color-background-button-secondary); }
+  background: #42b8dd;
+  /* this is a light blue */ }
 
 .button-cancel {
-  background: var(--color-background-button-cancel); }
+  background: #c8c8c8;
-
+  /* this is a green */ }
-#save_button {
-  margin-right: 1rem; }
 
 .messages li {
   list-style: none;
   padding: 1em;
   border-radius: 10px;
-  color: var(--color-text-messages);
+  color: #fff;
   font-weight: bold; }
   .messages li.message {
-    background: var(--color-background-messages-message); }
+    background: rgba(255, 255, 255, 0.2); }
   .messages li.error {
-    background: var(--color-background-messages-error); }
+    background: rgba(255, 1, 1, 0.5); }
   .messages li.notice {
-    background: var(--color-background-messages-notice); }
+    background: rgba(255, 255, 255, 0.5); }
 
 .messages.with-share-link > *:hover {
   cursor: pointer; }
 
-.notifications-wrapper {
-  padding: 0.5rem 0 1rem 0; }
-
-label:hover {
-  cursor: pointer; }
-
 #notification-customisation {
-  border: 1px solid var(--color-border-notification);
+  border: 1px solid #ccc;
   padding: 0.5rem;
   border-radius: 5px; }
 
 #notification-error-log {
-  border: 1px solid var(--color-border-notification);
+  border: 1px solid #ccc;
   padding: 1rem;
   border-radius: 5px;
   overflow-wrap: break-word; }
 
-#token-table.pure-table td,
+#token-table.pure-table td, #token-table.pure-table th {
-#token-table.pure-table th {
   font-size: 80%; }
 
 #new-watch-form {
-  background: var(--color-background-new-watch-form);
+  background: rgba(0, 0, 0, 0.05);
   padding: 1em;
   border-radius: 10px;
   margin-bottom: 1em; }
   #new-watch-form input {
     display: inline-block;
     margin-bottom: 5px; }
-  #new-watch-form input:not(.pure-button) {
-    background-color: var(--color-background-new-watch-input);
-    color: var(--color-text-new-watch-input); }
   #new-watch-form .label {
     display: none; }
   #new-watch-form legend {
-    color: var(--color-text-legend);
+    color: #fff;
     font-weight: bold; }
   #new-watch-form #watch-add-wrapper-zone > div {
     display: inline-block; }
@@ -531,14 +227,14 @@ label:hover {
   position: fixed;
   left: 0px;
   top: 120px;
-  background: var(--color-background);
+  background: #fff;
   padding: 10px;
   border-top-right-radius: 5px;
   border-bottom-right-radius: 5px;
-  box-shadow: 1px 1px 4px var(--color-shadow-jump); }
+  box-shadow: 5px 0 5px -2px #888; }
   #diff-jump a {
-    color: var(--color-link);
+    color: #1b98f8;
-    cursor: pointer;
+    cursor: grabbing;
     -moz-user-select: none;
     -webkit-user-select: none;
     -ms-user-select: none;
@@ -547,8 +243,8 @@ label:hover {
 
 footer {
   padding: 10px;
-  background: var(--color-background);
+  background: #fff;
-  color: var(--color-text-footer);
+  color: #444;
   text-align: center; }
 
 #feed-icon {
@@ -556,18 +252,18 @@ footer {
 
 #top-right-menu {
   /*
-      position: absolute;
+    position: absolute;
-      right: 0px;
+    right: 0px;
-      background: linear-gradient(to right, #fff0, #fff 10%);
+    background: linear-gradient(to right, #fff0, #fff 10%);
-      padding-left: 20px;
+    padding-left: 20px;
-      padding-right: 10px;
+    padding-right: 10px;
-      */ }
+    */ }
 
 .sticky-tab {
   position: absolute;
   top: 60px;
   font-size: 65%;
-  background: var(--color-background);
+  background: #fff;
   padding: 10px; }
   .sticky-tab#left-sticky {
     left: 0px; }
@@ -579,10 +275,9 @@ footer {
     font-weight: bold; }
 
 #new-version-text a {
-  color: var(--color-link-new-version); }
+  color: #e07171; }
 
 .watch-controls {
-  color: #f8321b;
   /* default */ }
   .watch-controls .state-on img {
     opacity: 0.8; }
@@ -597,7 +292,7 @@ footer {
   font-family: monospace;
   white-space: pre;
   overflow-wrap: normal;
-  overflow-x: auto; }
+  overflow-x: scroll; }
 
 .pure-form {
   /* The input fields with errors */
@@ -607,39 +302,27 @@ footer {
     .pure-form fieldset ul {
       padding-bottom: 0px;
       margin-bottom: 0px; }
-  .pure-form .pure-control-group,
+  .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
-  .pure-form .pure-group,
-  .pure-form .pure-controls {
     padding-bottom: 1em; }
-    .pure-form .pure-control-group div,
+    .pure-form .pure-control-group div, .pure-form .pure-group div, .pure-form .pure-controls div {
-    .pure-form .pure-group div,
-    .pure-form .pure-controls div {
       margin: 0px; }
-    .pure-form .pure-control-group .checkbox > *,
+    .pure-form .pure-control-group .checkbox > *, .pure-form .pure-group .checkbox > *, .pure-form .pure-controls .checkbox > * {
-    .pure-form .pure-group .checkbox > *,
-    .pure-form .pure-controls .checkbox > * {
       display: inline;
       vertical-align: middle; }
-    .pure-form .pure-control-group .checkbox > label,
+    .pure-form .pure-control-group .checkbox > label, .pure-form .pure-group .checkbox > label, .pure-form .pure-controls .checkbox > label {
-    .pure-form .pure-group .checkbox > label,
-    .pure-form .pure-controls .checkbox > label {
       padding-left: 5px; }
-    .pure-form .pure-control-group legend,
-    .pure-form .pure-group legend,
-    .pure-form .pure-controls legend {
-      color: var(--color-text-legend); }
   .pure-form .error input {
-    background-color: var(--color-error-input); }
+    background-color: #ffebeb; }
   .pure-form ul.errors {
     padding: .5em .6em;
-    border: 1px solid var(--color-error-list);
+    border: 1px solid #dd0000;
     border-radius: 4px;
     vertical-align: middle;
     -webkit-box-sizing: border-box;
     box-sizing: border-box; }
     .pure-form ul.errors li {
       margin-left: 1em;
-      color: var(--color-error-list); }
+      color: #dd0000; }
   .pure-form label {
     font-weight: bold; }
   .pure-form textarea {
@@ -673,27 +356,23 @@ footer {
   input[type='text'] {
     width: 100%; }
   /*
-  Max width before this PARTICULAR table gets nasty
+Max width before this PARTICULAR table gets nasty
-  This query will take effect for any screen smaller than 760px
+This query will take effect for any screen smaller than 760px
-  and also iPads specifically.
+and also iPads specifically.
-  */
+*/
   .watch-table {
     /* Force table to not be like tables anymore */
     /* Force table to not be like tables anymore */
     /* Hide table headers (but not display: none;, for accessibility) */ }
-    .watch-table thead,
+    .watch-table thead, .watch-table tbody, .watch-table th, .watch-table td, .watch-table tr {
-    .watch-table tbody,
-    .watch-table th,
-    .watch-table td,
-    .watch-table tr {
       display: block; }
     .watch-table .last-checked > span {
       vertical-align: middle; }
     .watch-table .last-checked::before {
-      color: var(--color-last-checked);
+      color: #555;
       content: "Last Checked "; }
     .watch-table .last-changed::before {
-      color: var(--color-last-checked);
+      color: #555;
       content: "Last Changed "; }
     .watch-table td.inline {
       display: inline-block; }
@@ -701,13 +380,12 @@ footer {
       position: absolute;
       top: -9999px;
       left: -9999px; }
-    .watch-table .pure-table td,
+    .watch-table .pure-table td, .watch-table .pure-table th {
-    .watch-table .pure-table th {
       border: none; }
     .watch-table td {
       /* Behave  like a "row" */
       border: none;
-      border-bottom: 1px solid var(--color-border-watch-table-cell);
+      border-bottom: 1px solid #eee;
       vertical-align: middle; }
       .watch-table td:before {
         /* Top/left values mimic padding */
@@ -717,66 +395,12 @@ footer {
         padding-right: 10px;
         white-space: nowrap; }
     .watch-table.pure-table-striped tr {
-      background-color: var(--color-table-background); }
+      background-color: #fff; }
     .watch-table.pure-table-striped tr:nth-child(2n-1) {
-      background-color: var(--color-table-stripe); }
+      background-color: #eee; }
     .watch-table.pure-table-striped tr:nth-child(2n-1) td {
       background-color: inherit; } }
 
-.pure-table {
-  border-color: var(--color-border-table-cell); }
-  .pure-table thead {
-    background-color: var(--color-background-table-thead);
-    color: var(--color-text); }
-  .pure-table td,
-  .pure-table th {
-    border-left-color: var(--color-border-table-cell); }
-
-.pure-table-striped tr:nth-child(2n-1) td {
-  background-color: var(--color-table-stripe); }
-
-.pure-form input[type=color],
-.pure-form input[type=date],
-.pure-form input[type=datetime-local],
-.pure-form input[type=datetime],
-.pure-form input[type=email],
-.pure-form input[type=month],
-.pure-form input[type=number],
-.pure-form input[type=password],
-.pure-form input[type=search],
-.pure-form input[type=tel],
-.pure-form input[type=text],
-.pure-form input[type=time],
-.pure-form input[type=url],
-.pure-form input[type=week],
-.pure-form select,
-.pure-form textarea {
-  border: var(--color-border-input);
-  box-shadow: inset 0 1px 3px var(--color-shadow-input);
-  background-color: var(--color-background-input);
-  color: var(--color-text-input); }
-  .pure-form input[type=color]:active,
-  .pure-form input[type=date]:active,
-  .pure-form input[type=datetime-local]:active,
-  .pure-form input[type=datetime]:active,
-  .pure-form input[type=email]:active,
-  .pure-form input[type=month]:active,
-  .pure-form input[type=number]:active,
-  .pure-form input[type=password]:active,
-  .pure-form input[type=search]:active,
-  .pure-form input[type=tel]:active,
-  .pure-form input[type=text]:active,
-  .pure-form input[type=time]:active,
-  .pure-form input[type=url]:active,
-  .pure-form input[type=week]:active,
-  .pure-form select:active,
-  .pure-form textarea:active {
-    background-color: var(--color-background-input); }
-
-input::placeholder,
-textarea::placeholder {
-  color: var(--color-text-input-placeholder); }
-
 /** Desktop vs mobile input field strategy
 - We dont use 'size' with <input> because `size` is too unreliable to override, and will often push-out
 - Rely always on width in CSS
@@ -793,29 +417,25 @@ textarea::placeholder {
   .tabs ul li {
     margin-right: 3px;
     display: inline-block;
-    color: var(--color-text-tab);
+    color: #fff;
     border-top-left-radius: 5px;
     border-top-right-radius: 5px;
-    background-color: var(--color-background-tab); }
+    background-color: rgba(255, 255, 255, 0.2); }
-    .tabs ul li:not(.active):hover {
+    .tabs ul li.active, .tabs ul li :target {
-      background-color: var(--color-background-tab-hover); }
+      background-color: #fff; }
-    .tabs ul li.active,
+      .tabs ul li.active a, .tabs ul li :target a {
-    .tabs ul li :target {
+        color: #222;
-      background-color: var(--color-background); }
-      .tabs ul li.active a,
-      .tabs ul li :target a {
-        color: var(--color-text-tab-active);
         font-weight: bold; }
     .tabs ul li a {
       display: block;
       padding: 0.8em;
-      color: var(--color-text-tab); }
+      color: #fff; }
 
 .pure-form-stacked > div:first-child {
   display: block; }
 
 .login-form .inner {
-  background: var(--color-background);
+  background: #fff;
   padding: 20px;
   border-radius: 5px; }
 
@@ -826,7 +446,7 @@ textarea::placeholder {
   .tab-pane-inner:target {
     display: block; }
 
-.beta-logo {
+#beta-logo {
   height: 50px;
   right: -3px;
   top: -3px;
@@ -835,9 +455,6 @@ textarea::placeholder {
 #selector-header {
   padding-bottom: 1em; }
 
-body.full-width .edit-form {
-  width: 95%; }
-
 .edit-form {
   min-width: 70%;
   /* so it cant overflow */
@@ -845,16 +462,13 @@ body.full-width .edit-form {
   .edit-form .box-wrap {
     position: relative; }
   .edit-form .inner {
-    background: var(--color-background);
+    background: #fff;
     padding: 20px; }
   .edit-form #actions {
     display: block;
-    background: var(--color-background); }
+    background: #fff; }
   .edit-form .pure-form-message-inline {
-    padding-left: 0;
+    padding-left: 0; }
-    color: var(--color-text-input-description); }
-    .edit-form .pure-form-message-inline code {
-      font-size: .875em; }
 
 ul {
   padding-left: 1em;
@@ -867,7 +481,7 @@ ul {
     width: 5em; }
 
 #selector-wrapper {
-  height: 100%;
+  height: 600px;
   overflow-y: scroll;
   position: relative; }
   #selector-wrapper > img {
@@ -891,40 +505,59 @@ ul {
   cursor: pointer; }
 
 #api-key-copy {
-  color: var(--color-api-key); }
+  color: #0078e7; }
 
-.button-green {
+/* spinner */
-  background-color: var(--color-background-button-green); }
+.loader,
+.loader:after {
+  border-radius: 50%;
+  width: 10px;
+  height: 10px; }
 
-.button-red {
+.loader {
-  background-color: var(--color-background-button-red); }
+  margin: 0px auto;
+  font-size: 3px;
+  vertical-align: middle;
+  display: inline-block;
+  text-indent: -9999em;
+  border-top: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-right: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-left: 1.1em solid #2668ed;
+  -webkit-transform: translateZ(0);
+  -ms-transform: translateZ(0);
+  transform: translateZ(0);
+  -webkit-animation: load8 1.1s infinite linear;
+  animation: load8 1.1s infinite linear; }
 
-.noselect {
+@-webkit-keyframes load8 {
-  -webkit-touch-callout: none;
+  0% {
-  /* iOS Safari */
+    -webkit-transform: rotate(0deg);
-  -webkit-user-select: none;
+    transform: rotate(0deg); }
-  /* Safari */
+  100% {
-  -moz-user-select: none;
+    -webkit-transform: rotate(360deg);
-  /* Old versions of Firefox */
+    transform: rotate(360deg); } }
-  -ms-user-select: none;
+
-  /* Internet Explorer/Edge */
+@keyframes load8 {
-  user-select: none;
+  0% {
-  /* Non-prefixed version, currently
+    -webkit-transform: rotate(0deg);
-    supported by Chrome, Edge, Opera and Firefox */ }
+    transform: rotate(0deg); }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg); } }
 
 .snapshot-age {
   padding: 4px;
-  margin: 0.5rem 0;
+  background-color: #dfdfdf;
-  background-color: var(--color-background-snapshot-age);
   border-radius: 3px;
   font-weight: bold;
   margin-bottom: 4px; }
   .snapshot-age.error {
-    background-color: var(--color-error-background-snapshot-age);
+    background-color: #ff0000;
-    color: var(--color-error-text-snapshot-age); }
+    color: #fff; }
 
 #checkbox-operations {
-  background: var(--color-background-checkbox-operations);
+  background: rgba(0, 0, 0, 0.05);
   padding: 1em;
   border-radius: 10px;
   margin-bottom: 1em;
@@ -932,37 +565,3 @@ ul {
 
 .checkbox-uuid > * {
   vertical-align: middle; }
-
-.inline-warning {
-  border: 1px solid var(--color-border-warning);
-  padding: 0.5rem;
-  border-radius: 5px;
-  color: var(--color-warning); }
-  .inline-warning > span {
-    display: inline-block;
-    vertical-align: middle; }
-  .inline-warning img.inline-warning-icon {
-    display: inline;
-    height: 26px;
-    vertical-align: middle; }
-
-/* automatic price following helpers */
-.tracking-ldjson-price-data {
-  background-color: var(--color-background-button-green);
-  color: #000;
-  padding: 3px;
-  border-radius: 3px;
-  white-space: nowrap; }
-
-.ldjson-price-track-offer {
-  font-weight: bold;
-  font-style: italic; }
-  .ldjson-price-track-offer a.pure-button {
-    border-radius: 3px;
-    padding: 3px;
-    background-color: var(--color-background-button-green); }
-
-.price-follow-tag-icon {
-  display: inline-block;
-  height: 0.8rem;
-  vertical-align: middle; }

changedetectionio/static/styles/styles.scss

@@ -0,0 +1,788 @@
+/*
+ * -- BASE STYLES --
+ * Most of these are inherited from Base, but I want to change a few.
+ * nvm use v14.18.1 && npm install && npm run build
+ * or npm run watch
+ */
+@import "parts/_arrows.scss";
+
+body {
+  color: #333;
+  background: #262626;
+}
+.pure-table-even {
+  background: #fff;
+}
+
+/* Some styles from https://css-tricks.com/ */
+a {
+  text-decoration: none;
+  color: #1b98f8;
+}
+
+a.github-link {
+  color: #fff;
+}
+
+.pure-menu-horizontal {
+  background: #fff;
+  padding: 5px;
+  display: flex;
+  justify-content: space-between;
+  border-bottom: 2px solid #ed5900;
+  align-items: center;
+}
+
+section.content {
+  padding-top: 5em;
+  padding-bottom: 1em;
+  flex-direction: column;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+code {
+  background: #eee;
+}
+
+/* table related */
+.watch-table {
+  width: 100%;
+  font-size: 80%;
+
+  tr.unviewed {
+    font-weight: bold;
+  }
+
+  .error {
+    color: #a00;
+  }
+
+  td {
+    white-space: nowrap;
+  }
+
+  td.title-col {
+    word-break: break-all;
+    white-space: normal;
+  }
+
+  th {
+    white-space: nowrap;
+    a {
+      font-weight: normal;
+      &.active {
+        font-weight: bolder;
+      }
+      &.inactive {
+        .arrow {
+          display: none;
+        }
+      }
+    }
+  }
+
+  .title-col a[target="_blank"]::after, .current-diff-url::after {
+    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+    margin: 0 3px 0 5px;
+  }
+}
+
+.watch-tag-list {
+  color: #e70069;
+  white-space: nowrap;
+}
+
+.box {
+  max-width: 80%;
+  flex-direction: column;
+  display: flex;
+  justify-content: center;
+}
+
+
+#post-list-buttons {
+  text-align: right;
+  padding: 0px;
+  margin: 0px;
+
+  li {
+    display: inline-block;
+  }
+
+  a {
+    border-top-left-radius: initial;
+    border-top-right-radius: initial;
+    border-bottom-left-radius: 5px;
+    border-bottom-right-radius: 5px;
+  }
+}
+
+
+body:after {
+  content: "";
+  background: linear-gradient(130deg, #5ad8f7, #2f50af 41.07%, #9150bf 84.05%);
+}
+
+body:after, body:before {
+  display: block;
+  height: 650px;
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  z-index: -1;
+}
+
+body::after {
+  opacity: 0.91;
+}
+
+body::before {
+  // background-image set in base.html so it works with reverse proxies etc
+  content: "";
+  background-size: cover
+}
+
+body:after, body:before {
+  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
+  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
+}
+
+.button-small {
+  font-size: 85%;
+}
+
+.fetch-error {
+  padding-top: 1em;
+  font-size: 60%;
+  max-width: 400px;
+  display: block;
+}
+
+
+.button-secondary {
+  color: white;
+  border-radius: 4px;
+  text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
+}
+
+.button-success {
+  background: rgb(28, 184, 65);
+  /* this is a green */
+}
+
+.button-tag {
+  background: rgb(99, 99, 99);
+  color: #fff;
+  font-size: 65%;
+  border-bottom-left-radius: initial;
+  border-bottom-right-radius: initial;
+
+  &.active {
+    background: #9c9c9c;
+    font-weight: bold;
+  }
+
+}
+
+.button-error {
+  background: rgb(202, 60, 60);
+  /* this is a maroon */
+}
+
+.button-warning {
+  background: rgb(223, 117, 20);
+  /* this is an orange */
+}
+
+.button-secondary {
+  background: rgb(66, 184, 221);
+  /* this is a light blue */
+}
+
+
+.button-cancel {
+  background: rgb(200, 200, 200);
+  /* this is a green */
+}
+
+.messages {
+    li {
+        list-style: none;
+        padding: 1em;
+        border-radius: 10px;
+        color: #fff;
+        font-weight: bold;
+        &.message {
+            background: rgba(255, 255, 255, .2);
+        }
+        &.error {
+            background: rgba(255, 1, 1, .5);
+        }
+        &.notice {
+            background: rgba(255, 255, 255, .5);
+        }
+    }
+    &.with-share-link {
+     > *:hover {
+       cursor:pointer;
+     }
+    }
+}
+
+#notification-customisation {
+    border: 1px solid #ccc;
+    padding: 0.5rem;
+    border-radius: 5px;
+}
+
+#notification-error-log {
+    border: 1px solid #ccc;
+    padding: 1rem;
+    border-radius: 5px;
+    overflow-wrap: break-word;
+}
+
+#token-table {
+    &.pure-table td, &.pure-table th {
+        font-size: 80%;
+    }
+}
+
+#new-watch-form {
+  background: rgba(0, 0, 0, .05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+  input {
+    display: inline-block;
+    margin-bottom: 5px;
+  }
+  .label {
+    display: none;
+  }
+  legend {
+    color: #fff;
+    font-weight: bold;
+  }
+
+  #watch-add-wrapper-zone {
+    > div {
+      display: inline-block;
+    }
+    @media only screen and (max-width: 760px) {
+      #url {
+        width: 100%;
+      }
+    }
+  }
+}
+
+
+
+
+#diff-col {
+  padding-left: 40px;
+}
+
+#diff-jump {
+  position: fixed;
+  left: 0px;
+  top: 120px;
+  background: #fff;
+  padding: 10px;
+  border-top-right-radius: 5px;
+  border-bottom-right-radius: 5px;
+  box-shadow: 5px 0 5px -2px #888;
+     a {
+      color: #1b98f8;
+      cursor: grabbing;
+      -moz-user-select: none;
+      -webkit-user-select: none;
+      -ms-user-select: none;
+      user-select: none;
+      -o-user-select: none;
+    }
+}
+
+footer {
+  padding: 10px;
+  background: #fff;
+  color: #444;
+  text-align: center;
+}
+
+#feed-icon {
+  vertical-align: middle;
+}
+
+#top-right-menu {
+// Just let flex overflow the x axis for now
+/*
+    position: absolute;
+    right: 0px;
+    background: linear-gradient(to right, #fff0, #fff 10%);
+    padding-left: 20px;
+    padding-right: 10px;
+    */
+}
+
+.sticky-tab {
+  position: absolute;
+  top: 60px;
+  font-size: 65%;
+  background: #fff;
+  padding: 10px;
+  &#left-sticky {
+    left: 0px;
+  }
+  &#right-sticky {
+    right: 0px;
+  }
+  &#hosted-sticky {
+    right: 0px;
+    top: 100px;
+    font-weight: bold;
+  }
+}
+
+#new-version-text a {
+  color: #e07171;
+}
+
+.watch-controls {
+  .state-on {
+    img {
+      opacity: 0.8;
+    }
+  }
+
+  /* default */
+  img {
+    opacity: 0.2;
+  }
+
+  img {
+    &:hover {
+      transition: opacity 0.3s;
+      opacity: 0.8;
+    }
+  }
+
+}
+
+.monospaced-textarea {
+    textarea {
+        width: 100%;
+        font-family: monospace;
+        white-space: pre;
+        overflow-wrap: normal;
+        overflow-x: scroll;
+    }
+}
+
+
+.pure-form {
+    fieldset {
+        padding-top: 0px;
+        ul {
+            padding-bottom: 0px;
+            margin-bottom: 0px;
+        }
+    }
+    .pure-control-group, .pure-group, .pure-controls {
+        padding-bottom: 1em;
+        div {
+            margin: 0px;
+        }
+        .checkbox {
+            > * {
+              display: inline;
+              vertical-align: middle;
+            }
+            > label {
+               padding-left: 5px;
+            }
+        }
+    }
+  /* The input fields with errors */
+  .error {
+    input {
+        background-color: #ffebeb;
+    }
+  }
+
+  /* The list of errors */
+  ul.errors {
+    padding: .5em .6em;
+    border: 1px solid #dd0000;
+    border-radius: 4px;
+    vertical-align: middle;
+    -webkit-box-sizing: border-box;
+    box-sizing: border-box;
+    li {
+        margin-left: 1em;
+        color: #dd0000;
+    }
+  }
+
+  label {
+    font-weight: bold;
+  }
+
+  textarea {
+    width: 100%;
+  }
+  .inline-radio {
+      ul {
+        margin: 0px;
+        list-style: none;
+        li {
+            > * {
+                display: inline-block;
+            }
+        }
+      }
+  }
+}
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+  .box {
+    max-width: 95%
+  }
+  .edit-form {
+    padding: 0.5em;
+    margin: 0;
+  }
+  #nav-menu {
+    overflow-x: scroll;
+  }
+}
+
+
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 800px) {
+
+  div.sticky-tab#hosted-sticky {
+    top: 60px;
+    left: 0px;
+    right: auto;
+  }
+
+  section.content {
+    padding-top: 110px;
+  }
+
+  // Make the tabs easier to hit, they will be all nice and horizontal
+  div.tabs.collapsable ul li {
+    display: block;
+    border-radius: 0px;
+    margin-right: 0px;
+  }
+
+  input[type='text'] {
+    width: 100%;
+  }
+
+/*
+Max width before this PARTICULAR table gets nasty
+This query will take effect for any screen smaller than 760px
+and also iPads specifically.
+*/
+  .watch-table {
+    /* Force table to not be like tables anymore */
+    thead, tbody, th, td, tr {
+      display: block;
+    }
+
+    .last-checked {
+      > span {
+        vertical-align: middle;
+      }
+    }
+
+    .last-checked::before {
+      color: #555;
+      content: "Last Checked ";
+    }
+
+    .last-changed::before {
+      color: #555;
+      content: "Last Changed ";
+    }
+
+    /* Force table to not be like tables anymore */
+    td.inline {
+      display: inline-block;
+    }
+
+    /* Hide table headers (but not display: none;, for accessibility) */
+    thead tr {
+      position: absolute;
+      top: -9999px;
+      left: -9999px;
+    }
+
+    .pure-table td, .pure-table th {
+      border: none;
+    }
+
+    td {
+      /* Behave  like a "row" */
+      border: none;
+      border-bottom: 1px solid #eee;
+      vertical-align: middle;
+      &:before {
+        /* Top/left values mimic padding */
+        top: 6px;
+        left: 6px;
+        width: 45%;
+        padding-right: 10px;
+        white-space: nowrap;
+      }
+    }
+
+    &.pure-table-striped {
+      tr {
+        background-color: #fff;
+      }
+
+      tr:nth-child(2n-1) {
+        background-color: #eee;
+      }
+
+      tr:nth-child(2n-1) td {
+        background-color: inherit;
+      }
+    }
+
+  }
+}
+
+
+/** Desktop vs mobile input field strategy
+- We dont use 'size' with <input> because `size` is too unreliable to override, and will often push-out
+- Rely always on width in CSS
+*/
+@media only screen and (min-width: 761px) {
+/* m-d is medium-desktop */
+    .m-d {
+        min-width: 80%;
+    }
+}
+
+
+
+.tabs {
+  ul {
+    margin: 0px;
+    padding: 0px;
+    display:block;
+    li {
+      margin-right: 3px;
+      display: inline-block;
+      color: #fff;
+      border-top-left-radius: 5px;
+      border-top-right-radius: 5px;
+      background-color: rgba(255, 255, 255, 0.2);
+
+      &.active,:target {
+        background-color: #fff;
+        a {
+          color: #222;
+          font-weight: bold;
+        }
+      }
+      a {
+        display: block;
+        padding: 0.8em;
+        color: #fff;
+      }
+    }
+  }
+}
+
+$form-edge-padding: 20px;
+.pure-form-stacked {
+  >div:first-child {
+    display: block;
+  }
+}
+
+.login-form {
+  .inner {
+    background: #fff;;
+    padding: $form-edge-padding;
+    border-radius: 5px;
+  }
+}
+
+.tab-pane-inner {
+    &:not(:target) {
+        display: none;
+    }
+    &:target {
+      display: block;
+    }
+    // doesnt need padding because theres another row of buttons/activity
+    padding: 0px;
+}
+
+#beta-logo {
+    height: 50px;
+    // looks better when it's hanging off a little
+    right: -3px;
+    top: -3px;
+    position: absolute;
+}
+
+#selector-header {
+    padding-bottom: 1em;
+}
+
+.edit-form {
+  min-width: 70%;
+  /* so it cant overflow */
+  max-width: 95%;
+  .box-wrap {
+    position: relative;
+  }
+  .inner {
+    background: #fff;;
+    padding: $form-edge-padding;
+  }
+  #actions {
+    display: block;
+    background: #fff;
+  }
+
+  .pure-form-message-inline {
+    padding-left: 0;
+  }
+}
+
+ul {
+    padding-left: 1em;
+    padding-top: 0px;
+    margin-top: 4px;
+}
+
+.time-check-widget {
+    tr {
+        display: inline;
+        input[type="number"] {
+            width: 5em;
+        }
+    }
+}
+
+#selector-wrapper {
+ height: 600px;
+ overflow-y: scroll;
+ position: relative;
+    //width: 100%;
+ > img {
+    position: absolute;
+    z-index: 4;
+    max-width: 100%;
+ }
+ >canvas {
+    position: relative;
+    z-index: 5;
+     max-width: 100%;
+     &:hover {
+     cursor: pointer;
+     }
+ }
+}
+
+#selector-current-xpath {
+  font-size: 80%;
+}
+
+#webdriver-override-options {
+        input[type="number"] {
+            width: 5em;
+        }
+}
+
+#api-key {
+  &:hover {
+    cursor: pointer;
+  }
+}
+
+#api-key-copy {
+  color: #0078e7;
+}
+
+/* spinner */
+.loader,
+.loader:after {
+  border-radius: 50%;
+  width: 10px;
+  height: 10px;
+}
+.loader {
+  margin: 0px auto;
+  font-size: 3px;
+  vertical-align: middle;
+  display: inline-block;
+  text-indent: -9999em;
+  border-top: 1.1em solid rgba(38,104,237, 0.2);
+  border-right: 1.1em solid rgba(38,104,237, 0.2);
+  border-bottom: 1.1em solid rgba(38,104,237, 0.2);
+  border-left: 1.1em solid #2668ed;
+  -webkit-transform: translateZ(0);
+  -ms-transform: translateZ(0);
+  transform: translateZ(0);
+  -webkit-animation: load8 1.1s infinite linear;
+  animation: load8 1.1s infinite linear;
+}
+@-webkit-keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg);
+  }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg);
+  }
+}
+@keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg);
+  }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg);
+  }
+}
+
+.snapshot-age {
+  padding: 4px;
+  background-color: #dfdfdf;
+  border-radius: 3px;
+  font-weight: bold;
+  margin-bottom: 4px;
+  &.error {
+    background-color: #ff0000;
+    color: #fff;
+  }
+}
+
+#checkbox-operations {
+  background: rgba(0, 0, 0, 0.05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+  display: none;
+}
+.checkbox-uuid {
+  > * {
+    vertical-align: middle;
+  }
+}

changedetectionio/store.py

@@ -27,18 +27,17 @@ class ChangeDetectionStore:
     # For when we edit, we should write to disk
     needs_write_urgent = False
 
-    __version_check = True
-
     def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
         # Should only be active for docker
         # logging.basicConfig(filename='/dev/stdout', level=logging.INFO)
-        self.__data = App.model()
+        self.needs_write = False
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
-        self.needs_write = False
         self.proxy_list = None
-        self.start_time = time.time()
         self.stop_thread = False
+
+        self.__data = App.model()
+
         # Base definition for all watchers
         # deepcopy part of #569 - not sure why its needed exactly
         self.generic_definition = deepcopy(Watch.model(datastore_path = datastore_path, default={}))
@@ -82,13 +81,10 @@ class ChangeDetectionStore:
         except (FileNotFoundError, json.decoder.JSONDecodeError):
             if include_default_watches:
                 print("Creating JSON store at", self.datastore_path)
-                self.add_watch(url='https://news.ycombinator.com/',
-                               tag='Tech news',
-                               extras={'fetch_backend': 'html_requests'})
 
-                self.add_watch(url='https://changedetection.io/CHANGELOG.txt',
+                self.add_watch(url='http://www.quotationspage.com/random.php', tag='test')
-                               tag='changedetection.io',
+                self.add_watch(url='https://news.ycombinator.com/', tag='Tech news')
-                               extras={'fetch_backend': 'html_requests'})
+                self.add_watch(url='https://changedetection.io/CHANGELOG.txt', tag='changedetection.io')
 
         self.__data['version_tag'] = version_tag
 
@@ -117,7 +113,9 @@ class ChangeDetectionStore:
             self.__data['settings']['application']['api_access_token'] = secret
 
         # Proxy list support - available as a selection in settings when text file is imported
-        proxy_list_file = "{}/proxies.json".format(self.datastore_path)
+        # CSV list
+        # "name, address", or just "name"
+        proxy_list_file = "{}/proxies.txt".format(self.datastore_path)
         if path.isfile(proxy_list_file):
             self.import_proxy_list(proxy_list_file)
 
@@ -246,19 +244,20 @@ class ChangeDetectionStore:
 
         return False
 
+    def get_val(self, uuid, val):
+        # Probably their should be dict...
+        return self.data['watching'][uuid].get(val)
+
     # Remove a watchs data but keep the entry (URL etc)
     def clear_watch_history(self, uuid):
         import pathlib
 
-        self.__data['watching'][uuid].update({
+        self.__data['watching'][uuid].update(
-                'last_checked': 0,
+            {'last_checked': 0,
-                'has_ldjson_price_data': None,
+             'last_viewed': 0,
-                'last_error': False,
+             'previous_md5': False,
-                'last_notification_error': False,
+             'last_notification_error': False,
-                'last_viewed': 0,
+             'last_error': False})
-                'previous_md5': False,
-                'track_ldjson_price_data': None,
-            })
 
         # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc
         for item in pathlib.Path(os.path.join(self.datastore_path, uuid)).rglob("*.*"):
@@ -275,7 +274,7 @@ class ChangeDetectionStore:
             extras = {}
         # should always be str
         if tag is None or not tag:
-            tag = ''
+            tag=''
 
         # Incase these are copied across, assume it's a reference and deepcopy()
         apply_extras = deepcopy(extras)
@@ -290,32 +289,17 @@ class ChangeDetectionStore:
                 res = r.json()
 
                 # List of permissible attributes we accept from the wild internet
-                for k in [
+                for k in ['url', 'tag',
-                    'body',
+                          'paused', 'title',
-                    'browser_steps',
+                          'previous_md5', 'headers',
-                    'css_filter',
+                          'body', 'method',
-                    'extract_text',
+                          'ignore_text', 'css_filter',
-                    'extract_title_as_title',
+                          'subtractive_selectors', 'trigger_text',
-                    'headers',
+                          'extract_title_as_title', 'extract_text',
-                    'ignore_text',
+                          'text_should_not_be_present',
-                    'include_filters',
+                          'webdriver_js_execute_code']:
-                    'method',
-                    'paused',
-                    'previous_md5',
-                    'subtractive_selectors',
-                    'tag',
-                    'text_should_not_be_present',
-                    'title',
-                    'trigger_text',
-                    'url',
-                    'webdriver_js_execute_code',
-                ]:
                     if res.get(k):
-                        if k != 'css_filter':
+                        apply_extras[k] = res[k]
-                            apply_extras[k] = res[k]
-                        else:
-                            # We renamed the field and made it a list
-                            apply_extras['include_filters'] = [res['css_filter']]
 
             except Exception as e:
                 logging.error("Error fetching metadata for shared watch link", url, str(e))
@@ -338,13 +322,12 @@ class ChangeDetectionStore:
                     del apply_extras[k]
 
             new_watch.update(apply_extras)
-            self.__data['watching'][new_uuid] = new_watch
+            self.__data['watching'][new_uuid]=new_watch
 
         self.__data['watching'][new_uuid].ensure_data_dir_exists()
 
         if write_to_disk_now:
             self.sync_to_json()
-
         return new_uuid
 
     def visualselector_data_is_ready(self, watch_uuid):
@@ -372,12 +355,6 @@ class ChangeDetectionStore:
             f.write(screenshot)
             f.close()
 
-        # Make a JPEG that's used in notifications (due to being a smaller size) available
-        from PIL import Image
-        im1 = Image.open(target_path)
-        im1.convert('RGB').save(target_path.replace('.png','.jpg'), quality=int(os.getenv("NOTIFICATION_SCREENSHOT_JPG_QUALITY", 75)))
-
-
     def save_error_text(self, watch_uuid, contents):
         if not self.data['watching'].get(watch_uuid):
             return
@@ -464,42 +441,20 @@ class ChangeDetectionStore:
                     unlink(item)
 
     def import_proxy_list(self, filename):
-        with open(filename) as f:
+        import csv
-            self.proxy_list = json.load(f)
+        with open(filename, newline='') as f:
-            print ("Registered proxy list", list(self.proxy_list.keys()))
+            reader = csv.reader(f, skipinitialspace=True)
+            # @todo This loop can could be improved
+            l = []
+            for row in reader:
+                if len(row):
+                    if len(row)>=2:
+                        l.append(tuple(row[:2]))
+                    else:
+                        l.append(tuple([row[0], row[0]]))
+            self.proxy_list = l if len(l) else None
 
 
-    def get_preferred_proxy_for_watch(self, uuid):
-        """
-        Returns the preferred proxy by ID key
-        :param uuid: UUID
-        :return: proxy "key" id
-        """
-
-        proxy_id = None
-        if self.proxy_list is None:
-            return None
-
-        # If its a valid one
-        watch = self.data['watching'].get(uuid)
-
-        if watch.get('proxy') and watch.get('proxy') in list(self.proxy_list.keys()):
-            return watch.get('proxy')
-
-        # not valid (including None), try the system one
-        else:
-            system_proxy_id = self.data['settings']['requests'].get('proxy')
-            # Is not None and exists
-            if self.proxy_list.get(system_proxy_id):
-                return system_proxy_id
-
-        # Fallback - Did not resolve anything, use the first available
-        if system_proxy_id is None:
-            first_default = list(self.proxy_list)[0]
-            return first_default
-
-        return None
-
     # Run all updates
     # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
     #             So therefor - each `update_n` should be very careful about checking if it needs to actually run
@@ -584,84 +539,4 @@ class ChangeDetectionStore:
                 del(watch['last_changed'])
             except:
                 continue
-        return
+        return
-
-    def update_5(self):
-        # If the watch notification body, title look the same as the global one, unset it, so the watch defaults back to using the main settings
-        # In other words - the watch notification_title and notification_body are not needed if they are the same as the default one
-        current_system_body = self.data['settings']['application']['notification_body'].translate(str.maketrans('', '', "\r\n "))
-        current_system_title = self.data['settings']['application']['notification_body'].translate(str.maketrans('', '', "\r\n "))
-        for uuid, watch in self.data['watching'].items():
-            try:
-                watch_body = watch.get('notification_body', '')
-                if watch_body and watch_body.translate(str.maketrans('', '', "\r\n ")) == current_system_body:
-                    # Looks the same as the default one, so unset it
-                    watch['notification_body'] = None
-
-                watch_title = watch.get('notification_title', '')
-                if watch_title and watch_title.translate(str.maketrans('', '', "\r\n ")) == current_system_title:
-                    # Looks the same as the default one, so unset it
-                    watch['notification_title'] = None
-            except Exception as e:
-                continue
-        return
-
-
-    # We incorrectly used common header overrides that should only apply to Requests
-    # These are now handled in content_fetcher::html_requests and shouldnt be passed to Playwright/Selenium
-    def update_7(self):
-        # These were hard-coded in early versions
-        for v in ['User-Agent', 'Accept', 'Accept-Encoding', 'Accept-Language']:
-            if self.data['settings']['headers'].get(v):
-                del self.data['settings']['headers'][v]
-
-    # Convert filters to a list of filters css_filter -> include_filters
-    def update_8(self):
-        for uuid, watch in self.data['watching'].items():
-            try:
-                existing_filter = watch.get('css_filter', '')
-                if existing_filter:
-                    watch['include_filters'] = [existing_filter]
-            except:
-                continue
-        return
-
-    # Convert old static notification tokens to jinja2 tokens
-    def update_9(self):
-        # Each watch
-        import re
-        # only { } not {{ or }}
-        r = r'(?<!{){(?!{)(\w+)(?<!})}(?!})'
-        for uuid, watch in self.data['watching'].items():
-            try:
-                n_body = watch.get('notification_body', '')
-                if n_body:
-                    watch['notification_body'] = re.sub(r, r'{{\1}}', n_body)
-
-                n_title = watch.get('notification_title')
-                if n_title:
-                    watch['notification_title'] = re.sub(r, r'{{\1}}', n_title)
-
-                n_urls = watch.get('notification_urls')
-                if n_urls:
-                    for i, url in enumerate(n_urls):
-                        watch['notification_urls'][i] = re.sub(r, r'{{\1}}', url)
-
-            except:
-                continue
-
-        # System wide
-        n_body = self.data['settings']['application'].get('notification_body')
-        if n_body:
-            self.data['settings']['application']['notification_body'] = re.sub(r, r'{{\1}}', n_body)
-
-        n_title = self.data['settings']['application'].get('notification_title')
-        if n_body:
-            self.data['settings']['application']['notification_title'] = re.sub(r, r'{{\1}}', n_title)
-
-        n_urls =  self.data['settings']['application'].get('notification_urls')
-        if n_urls:
-            for i, url in enumerate(n_urls):
-                self.data['settings']['application']['notification_urls'][i] = re.sub(r, r'{{\1}}', url)
-
-        return

changedetectionio/templates/_common_fields.jinja

@@ -1,14 +1,13 @@
 
 {% from '_helpers.jinja' import render_field %}
 
-{% macro render_common_settings_form(form, emailprefix, settings_application) %}
+{% macro render_common_settings_form(form, current_base_url, emailprefix) %}
                         <div class="pure-control-group">
                             {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
     Gitter - gitter://token/room
     Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
     AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
-    SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com",
+    SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com", class="notification-urls")
-    class="notification-urls" )
                             }}
                             <div class="pure-form-message-inline">
                               <ul>
@@ -16,35 +15,31 @@
                                 <li><code>discord://</code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
                                 <li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
                                 <li><code>tgram://</code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
-                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
                               </ul>
                             </div>
-                            <div class="notifications-wrapper">
+                            <br/>
-                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Send test notification</a>
+                            <a id="send-test-notification" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Send test notification</a>
-                            {% if emailprefix %}
+{% if emailprefix %}
-                              <a id="add-email-helper" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Add email</a>
+                            <a id="add-email-helper" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Add email</a>
-                            {% endif %}
+{% endif %}
-                              <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Notification debug logs</a>
+                            <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Notification debug logs</a>
-                            </div>
                         </div>
                         <div id="notification-customisation" class="pure-control-group">
                             <div class="pure-control-group">
-                                {{ render_field(form.notification_title, class="m-d notification-title", placeholder=settings_application['notification_title']) }}
+                                {{ render_field(form.notification_title, class="m-d notification-title") }}
                                 <span class="pure-form-message-inline">Title for all notifications</span>
                             </div>
                             <div class="pure-control-group">
-                                {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }}
+                                {{ render_field(form.notification_body , rows=5, class="notification-body") }}
                                 <span class="pure-form-message-inline">Body for all notifications</span>
                             </div>
                             <div class="pure-control-group">
-                            <!-- unsure -->
+                                {{ render_field(form.notification_format , rows=5, class="notification-format") }}
-                                {{ render_field(form.notification_format , class="notification-format") }}
                                 <span class="pure-form-message-inline">Format for all notifications</span>
                             </div>
                             <div class="pure-controls">
-                            <p class="pure-form-message-inline">
+                            <span class="pure-form-message-inline">
-                                You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL.
+                                These tokens can be used in the notification body and title to customise the notification text.
-                            </p>
 
                                 <table class="pure-table" id="token-table">
                                     <thead>
@@ -55,49 +50,52 @@
                                     </thead>
                                     <tbody>
                                     <tr>
-                                        <td><code>{{ '{{ base_url }}' }}</code></td>
+                                        <td><code>{base_url}</code></td>
                                         <td>The URL of the changedetection.io instance you are running.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ watch_url }}' }}</code></td>
+                                        <td><code>{watch_url}</code></td>
                                         <td>The URL being watched.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ watch_uuid }}' }}</code></td>
+                                        <td><code>{watch_uuid}</code></td>
                                         <td>The UUID of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ watch_title }}' }}</code></td>
+                                        <td><code>{watch_title}</code></td>
                                         <td>The title of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ watch_tag }}' }}</code></td>
+                                        <td><code>{watch_tag}</code></td>
-                                        <td>The watch label / tag</td>
+                                        <td>The tag of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ preview_url }}' }}</code></td>
+                                        <td><code>{preview_url}</code></td>
                                         <td>The URL of the preview page generated by changedetection.io.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ diff_url }}' }}</code></td>
+                                        <td><code>{diff}</code></td>
                                         <td>The diff output - differences only</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ diff_full }}' }}</code></td>
+                                        <td><code>{diff_full}</code></td>
                                         <td>The diff output - full difference output</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{ current_snapshot }}' }}</code></td>
+                                        <td><code>{diff_url}</code></td>
+                                        <td>The URL of the diff page generated by changedetection.io.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{current_snapshot}</code></td>
                                         <td>The current snapshot value, useful when combined with JSON or CSS filters
                                         </td>
                                     </tr>
                                     </tbody>
                                 </table>
-                                <div class="pure-form-message-inline">
+                                <br/>
-                                    <br>
+                                URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
-                                    URLs generated by changedetection.io (such as <code>{{ '{{ diff_url }}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
+                                Your <code>BASE_URL</code> var is currently "{{current_base_url}}"
-                                    Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
+                            </span>
-                                </div>
                             </div>
                         </div>
 {% endmacro %}

changedetectionio/templates/base.html

@@ -1,152 +1,110 @@
-<!DOCTYPE html>
+<!doctype html>
-<html lang="en" data-darkmode="{{ get_darkmode_state() }}">
+<html lang="en">
-
+<head>
-  <head>
+    <meta charset="utf-8">
-    <meta charset="utf-8"/>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+    <meta name="description" content="Self hosted website change detection.">
-    <meta name="description" content="Self hosted website change detection."/>
     <title>Change Detection{{extra_title}}</title>
-    <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag %}- {{active_tag}}{% endif %}" href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"/>
+    <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag %}- {{active_tag}}{% endif %}" href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}" />
-    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}"/>
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}">
-    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}"/>
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}">
     {% if extra_stylesheets %}
-      {% for m in extra_stylesheets %}
+        {% for m in extra_stylesheets %}
-        <link rel="stylesheet" href="{{ m }}?ver=1000"/>
+        <link rel="stylesheet" href="{{ m }}?ver=1000">
-      {% endfor %}
+        {% endfor %}
     {% endif %}
-
-    <link rel="apple-touch-icon" sizes="180x180" href="{{url_for('static_content', group='favicons', filename='apple-touch-icon.png')}}"/>
-    <link rel="icon" type="image/png" sizes="32x32" href="{{url_for('static_content', group='favicons', filename='favicon-32x32.png')}}"/>
-    <link rel="icon" type="image/png" sizes="16x16" href="{{url_for('static_content', group='favicons', filename='favicon-16x16.png')}}"/>
-    <link rel="manifest" href="{{url_for('static_content', group='favicons', filename='site.webmanifest')}}"/>
-    <link rel="mask-icon" href="{{url_for('static_content', group='favicons', filename='safari-pinned-tab.svg')}}" color="#5bbad5"/>
-    <link rel="shortcut icon" href="{{url_for('static_content', group='favicons', filename='favicon.ico')}}"/>
-    <meta name="msapplication-TileColor" content="#da532c"/>
-    <meta name="msapplication-config" content="favicons/browserconfig.xml"/>
-    <meta name="theme-color" content="#ffffff"/>
-
     <style>
-      body::before {
+    body::before {
-        background-image: url({{url_for('static_content', group='images', filename='gradient-border.png') }});
+        background-image: url({{url_for('static_content', group='images', filename='gradient-border.png')}});
-      }
+    }
     </style>
     <script type="text/javascript" src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
-  </head>
 
-  <body>
+</head>
-    <div class="header">
+
-      <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
+<body>
+
+<div class="header">
+
+    <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
         {% if has_password and not current_user.is_authenticated %}
-          <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener">
+            <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener"><strong>Change</strong>Detection.io</a>
-            <strong>Change</strong>Detection.io</a>
         {% else %}
-          <a class="pure-menu-heading" href="{{url_for('index')}}">
+            <a class="pure-menu-heading" href="{{url_for('index')}}"><strong>Change</strong>Detection.io</a>
-            <strong>Change</strong>Detection.io</a>
         {% endif %}
         {% if current_diff_url %}
-          <a class="current-diff-url" href="{{ current_diff_url }}">
+        <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</span></a>
-            <span style="max-width: 30%; overflow: hidden">{{ current_diff_url }}</span></a>
         {% else %}
-          {% if new_version_available and not(has_password and not current_user.is_authenticated) %}
+        {% if new_version_available and not (has_password and not current_user.is_authenticated) %}
-            <span id="new-version-text" class="pure-menu-heading">
+        <span id="new-version-text" class="pure-menu-heading"><a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a></span>
-              <a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a>
+        {% endif %}
-            </span>
-          {% endif %}
         {% endif %}
 
-        <ul class="pure-menu-list" id="top-right-menu">
+        <ul class="pure-menu-list"  id="top-right-menu">
-          {% if current_user.is_authenticated or not has_password %}
+        {% if current_user.is_authenticated or not has_password %}
-            {% if not
+            {% if not current_diff_url %}
-            current_diff_url %}
+            <li class="pure-menu-item">
-              <li class="pure-menu-item">
                 <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
-              </li>
+            </li>
-              <li class="pure-menu-item">
+            <li class="pure-menu-item">
                 <a href="{{ url_for('import_page')}}" class="pure-menu-link">IMPORT</a>
-              </li>
+            </li>
-              <li class="pure-menu-item">
+            <li class="pure-menu-item">
                 <a href="{{ url_for('get_backup')}}" class="pure-menu-link">BACKUP</a>
-              </li>
+            </li>
             {% else %}
-              <li class="pure-menu-item">
+            <li class="pure-menu-item">
                 <a href="{{ url_for('edit_page', uuid=uuid, next='diff') }}" class="pure-menu-link">EDIT</a>
-              </li>
-            {% endif %}
-          {% else %}
-            <li class="pure-menu-item">
-              <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
             </li>
-          {% endif %}
-          {% if current_user.is_authenticated %}
-            <li class="pure-menu-item">
-              <a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a>
-            </li>
-          {% endif %}
-          <li class="pure-menu-item">
-            {% if dark_mode %}
-            {% set darkClass = 'dark' %}
             {% endif %}
-            <button class="toggle-theme {{darkClass}}" type="button" title="Toggle Light/Dark Mode">
+        {% else %}
-              <span class="visually-hidden">Toggle light/dark mode</span>
+            <li class="pure-menu-item">
-              <span class="icon-light">
+                <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
-                {% include "svgs/light-mode-toggle-icon.svg" %}
+            </li>
-              </span>
+        {% endif %}
-              <span class="icon-dark">
+
-                {% include "svgs/dark-mode-toggle-icon.svg" %}
+        {% if current_user.is_authenticated %}
-              </span>
+            <li class="pure-menu-item"><a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a></li>
-            </button>
+        {% endif %}
-          </li>
+            <li class="pure-menu-item"><a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
-          <li class="pure-menu-item">
+                <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16"
-            <a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
+                     version="1.1"
-              {% include "svgs/github.svg" %}
+                     width="32" aria-hidden="true">
-            </a>
+                    <path fill-rule="evenodd"
-          </li>
+                          d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
+                </svg>
+            </a></li>
         </ul>
-      </div>
     </div>
-    {% if hosted_sticky %}
+</div>
-      <div class="sticky-tab" id="hosted-sticky">
+{% if hosted_sticky %}<div class="sticky-tab" id="hosted-sticky"><a href="https://lemonade.changedetection.io/start?ref={{guid}}">Let us host your instance!</a></div>{% endif %}
-        <a href="https://lemonade.changedetection.io/start?ref={{guid}}">Let us host your instance!</a>
+{% if left_sticky %}<div class="sticky-tab" id="left-sticky"><a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a></div> {% endif %}
-      </div>
+{% if right_sticky %}<div class="sticky-tab" id="right-sticky">{{ right_sticky }}</div> {% endif %}
-    {% endif %}
+<section class="content">
-    {% if left_sticky %}
+    <header>
-      <div class="sticky-tab" id="left-sticky">
-        <a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a>
-      </div>
-    {% endif %}
-    {% if right_sticky %}
-      <div class="sticky-tab" id="right-sticky">{{ right_sticky }}</div>
-    {% endif %}
-    <section class="content">
-      <header>
         {% block header %}{% endblock %}
-      </header>
+    </header>
 
-      {% with messages = get_flashed_messages(with_categories = true) %}
+    {% with messages = get_flashed_messages(with_categories=true) %}
-      {% if
+      {% if messages %}
-      messages %}
+        <ul class=messages>
-        <ul class="messages">
+        {% for category, message in messages %}
-          {% for category, message in messages %}
+          <li class="{{ category }}">{{ message }}</li>
-            <li class="{{ category }}">{{ message }}</li>
+        {% endfor %}
-          {% endfor %}
         </ul>
       {% endif %}
-      {% endwith %}
+    {% endwith %}
-      {% if session['share-link'] %}
+
+    {% if session['share-link'] %}
         <ul class="messages with-share-link">
-          <li class="message">
+          <li class="message">Share this link: <span id="share-link">{{ session['share-link'] }}</span> <img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='copy.svg')}}" /></li>
-            Share this link:
-            <span id="share-link">{{ session['share-link'] }}</span>
-            <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='copy.svg')}}"/>
-          </li>
         </ul>
-      {% endif %}
+    {% endif %}
-      {% block content %}{% endblock %}
-    </section>
-    <script
-      type="text/javascript"
-      src="{{url_for('static_content', group='js', filename='toggle-theme.js')}}"
-      defer></script>
-  </body>
 
+    {% block content %}
+
+    {% endblock %}
+</section>
+
+</body>
 </html>

changedetectionio/templates/clear_all_history.html

@@ -1,49 +1,32 @@
-{% extends 'base.html' %} {% block content %}
+{% extends 'base.html' %}
+
+{% block content %}
 <div class="edit-form">
-  <div class="box-wrap inner">
+    <div class="box-wrap inner">
-    <form
+    <form class="pure-form pure-form-stacked" action="{{url_for('clear_all_history')}}" method="POST">
-      class="pure-form pure-form-stacked"
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
-      action="{{url_for('clear_all_history')}}"
+        <fieldset>
-      method="POST"
+            <div class="pure-control-group">
-    >
+                This will remove version history (snapshots) for ALL watches, but keep your list of URLs! <br/>
-      <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
+                You may like to use the <strong>BACKUP</strong> link first.<br/>
-      <fieldset>
+            </div>
-        <div class="pure-control-group">
+            <br/>
-          This will remove version history (snapshots) for ALL watches, but keep
+            <div class="pure-control-group">
-          your list of URLs! <br />
+                <label for="confirmtext">Confirmation text</label>
-          You may like to use the <strong>BACKUP</strong> link first.<br />
+                <input type="text" id="confirmtext" required="" name="confirmtext" value="" size="10"/>
-        </div>
+                <span class="pure-form-message-inline">Type in the word <strong>clear</strong> to confirm that you understand.</span>
-        <br />
+            </div>
-        <div class="pure-control-group">
+            <br/>
-          <label for="confirmtext">Confirmation text</label>
+            <div class="pure-control-group">
-          <input
+                <button type="submit" class="pure-button pure-button-primary">Clear History!</button>
-            type="text"
+            </div>
-            id="confirmtext"
+            <br/>
-            required=""
+            <div class="pure-control-group">
-            name="confirmtext"
+                <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Cancel</a>
-            value=""
+            </div>
-            size="10"
+        </fieldset>
-          />
-          <span class="pure-form-message-inline"
-            >Type in the word <strong>clear</strong> to confirm that you
-            understand.</span
-          >
-        </div>
-        <br />
-        <div class="pure-control-group">
-          <button type="submit" class="pure-button pure-button-primary">
-            Clear History!
-          </button>
-        </div>
-        <br />
-        <div class="pure-control-group">
-          <a href="{{url_for('index')}}" class="pure-button button-cancel"
-            >Cancel</a
-          >
-        </div>
-      </fieldset>
     </form>
-  </div>
+    </div>
 </div>
 
 {% endblock %}

changedetectionio/templates/diff.html

@@ -1,5 +1,5 @@
 {% extends 'base.html' %}
-{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
+
 {% block content %}
 <script>
     const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
@@ -21,9 +21,6 @@
 
             <label for="diffChars" class="pure-checkbox">
                 <input type="radio" name="diff_type" id="diffChars" value="diffChars"/> Chars</label>
-            <!-- @todo - when mimetype is JSON, select this by default? -->
-            <label for="diffJson" class="pure-checkbox">
-                <input type="radio" name="diff_type" id="diffJson" value="diffJson" /> JSON</label>
 
             {% if versions|length >= 1 %}
             <label for="diff-version">Compare newest (<span id="current-v-date"></span>) with</label>
@@ -40,11 +37,6 @@
     </form>
     <del>Removed text</del>
     <ins>Inserted Text</ins>
-    <span>
-        <!-- https://github.com/kpdecker/jsdiff/issues/389 ? -->
-        <label for="ignoreWhitespace" class="pure-checkbox" id="label-diff-ignorewhitespace">
-            <input type="checkbox" id="ignoreWhitespace" name="ignoreWhitespace"/> Ignore Whitespace</label>
-    </span>
 </div>
 
 <div id="diff-jump">
@@ -58,7 +50,6 @@
         {% if last_error_screenshot %}<li class="tab" id="error-screenshot-tab"><a href="#error-screenshot">Error Screenshot</a></li> {% endif %}
         <li class="tab" id=""><a href="#text">Text</a></li>
         <li class="tab" id="screenshot-tab"><a href="#screenshot">Screenshot</a></li>
-        <li class="tab" id="extract-tab"><a href="#extract">Extract Data</a></li>
     </ul>
 </div>
 
@@ -109,45 +100,124 @@
            <strong>Screenshot requires Playwright/WebDriver enabled</strong>
          {% endif %}
      </div>
-    <div class="tab-pane-inner" id="extract">
-        <form id="extract-data-form" class="pure-form pure-form-stacked edit-form"
-              action="{{ url_for('diff_history_page', uuid=uuid) }}#extract"
-              method="POST">
-            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
-
-            <p>This tool will extract text data from all of the watch history.</p>
-
-            <div class="pure-control-group">
-                {{ render_field(extract_form.extract_regex) }}
-                <span class="pure-form-message-inline">
-                    A <strong>RegEx</strong> is a pattern that identifies exactly which part inside of the text that you want to extract.<br/>
-
-                    <p>
-                        For example, to extract only the numbers from text &dash;</br>
-                        <strong>Raw text</strong>: <code>Temperature <span style="color: red">5.5</span>°C in Sydney</code></br>
-                        <strong>RegEx to extract:</strong> <code>Temperature <span style="color: red">([0-9\.]+)</span></code><br/>
-                    </p>
-                    <p>
-                        <a href="https://RegExr.com/">Be sure to test your RegEx here.</a>
-                    </p>
-                    <p>
-                        Each RegEx group bracket <code>()</code> will be in its own column, the first column value is always the date.
-                    </p>
-                </span>
-            </div>
-            <div class="pure-control-group">
-                {{ render_button(extract_form.extract_submit_button) }}
-            </div>
-        </form>
-    </div>
 </div>
 
-<script>
-    const newest_version_timestamp = {{newest_version_timestamp}};
-</script>
-<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff.min.js')}}"></script>
 
-<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff-render.js')}}"></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff.js')}}"></script>
+
+<script defer="">
+
+var a = document.getElementById('a');
+var b = document.getElementById('b');
+var result = document.getElementById('result');
+
+function changed() {
+	var diff = JsDiff[window.diffType](a.textContent, b.textContent);
+	var fragment = document.createDocumentFragment();
+	for (var i=0; i < diff.length; i++) {
+
+		if (diff[i].added && diff[i + 1] && diff[i + 1].removed) {
+			var swap = diff[i];
+			diff[i] = diff[i + 1];
+			diff[i + 1] = swap;
+		}
+
+		var node;
+		if (diff[i].removed) {
+			node = document.createElement('del');
+			node.classList.add("change");
+			node.appendChild(document.createTextNode(diff[i].value));
+
+		} else if (diff[i].added) {
+			node = document.createElement('ins');
+			node.classList.add("change");
+			node.appendChild(document.createTextNode(diff[i].value));
+		} else {
+			node = document.createTextNode(diff[i].value);
+		}
+		fragment.appendChild(node);
+	}
+
+	result.textContent = '';
+	result.appendChild(fragment);
+
+	// Jump at start
+	inputs.current=0;
+    next_diff();
+}
+
+window.onload = function() {
+
+
+    /* Convert what is options from UTC time.time() to local browser time */
+    var diffList=document.getElementById("diff-version");
+    if (typeof(diffList) != 'undefined' && diffList != null) {
+        for (var option of diffList.options) {
+          var dateObject = new Date(option.value*1000);
+          option.label=dateObject.toLocaleString();
+        }
+    }
+
+    /* Set current version date as local time in the browser also */
+    var current_v = document.getElementById("current-v-date");
+    var dateObject = new Date({{ newest_version_timestamp }}*1000);
+    current_v.innerHTML=dateObject.toLocaleString();
+
+
+	onDiffTypeChange(document.querySelector('#settings [name="diff_type"]:checked'));
+	changed();
+
+};
+
+a.onpaste = a.onchange =
+b.onpaste = b.onchange = changed;
+
+if ('oninput' in a) {
+	a.oninput = b.oninput = changed;
+} else {
+	a.onkeyup = b.onkeyup = changed;
+}
+
+function onDiffTypeChange(radio) {
+	window.diffType = radio.value;
+// Not necessary 
+//	document.title = "Diff " + radio.value.slice(4);
+}
+
+var radio = document.getElementsByName('diff_type');
+for (var i = 0; i < radio.length; i++) {
+	radio[i].onchange = function(e) {
+		onDiffTypeChange(e.target);
+		changed();
+	}
+}
+
+
+var inputs = document.getElementsByClassName('change');
+inputs.current=0;
+
+
+function next_diff() {
+
+    var element = inputs[inputs.current];
+    var headerOffset = 80;
+    var elementPosition = element.getBoundingClientRect().top;
+    var offsetPosition = elementPosition - headerOffset +  window.scrollY;
+
+    window.scrollTo({
+         top: offsetPosition,
+         behavior: "smooth"
+    });
+
+    inputs.current++;
+    if(inputs.current >= inputs.length) {
+      inputs.current=0;
+    }
+}
+
+
+
+</script>
 
 
 {% endblock %}

changedetectionio/templates/edit.html

@@ -13,17 +13,11 @@
     const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
 
-    const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
-    const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
 </script>
-
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
-<script type="text/javascript" src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='visual-selector.js')}}" defer></script>
-{% if playwright_enabled %}
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>
-<script type="text/javascript" src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
-{% endif %}
 
 <div class="edit-form monospaced-textarea">
 
@@ -31,9 +25,6 @@
         <ul>
             <li class="tab" id=""><a href="#general">General</a></li>
             <li class="tab"><a href="#request">Request</a></li>
-            {% if playwright_enabled %}
-            <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
-            {% endif %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
             <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
             <li class="tab"><a href="#notifications">Notifications</a></li>
@@ -49,8 +40,7 @@
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
-                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span><br/>
+                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span>
-                        <span class="pure-form-message-inline">You can use variables in the URL, perfect for inserting the current date and other logic, <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a></span><br/>
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.title, class="m-d") }}
@@ -87,7 +77,6 @@
                         <span class="pure-form-message-inline">
                             <p>Use the <strong>Basic</strong> method (default) where your watched site doesn't need Javascript to render.</p>
                             <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
-                            Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using BrightData Proxies, find out more here.</a>
                         </span>
                     </div>
                 {% if form.proxy %}
@@ -144,72 +133,12 @@ User-Agent: wonderbra 1.0") }}
                     </div>
                 </fieldset>
             </div>
-            {% if playwright_enabled %}
-            <div class="tab-pane-inner" id="browser-steps">
-                <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}">
-                <fieldset>
-                    <div class="pure-control-group">
-                        <!--
-                        Too hard right now, better to just send the events to the fetcher for now and leave it in the final screenshot
-                        and/or report an error
-                        <a id="play-steps" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Play steps  ▶</a>
-                        -->
-
-                        <!---  Do this later -->
-                        <div class="checkbox" style="display: none;">
-                            <input type=checkbox id="include_text_elements" > <label for="include_text_elements">Turn on text finder</label>
-                        </div>
-
-                        <div id="loading-status-text" style="display: none;">Please wait, first browser step can take a little time to load..<div class="spinner"></div></div>
-                        <div class="flex-wrapper" >
-
-                            <div id="browser-steps-ui" class="noselect"  style="width: 100%; background-color: #eee; border-radius: 5px;">
-
-                                <div class="noselect"  id="browsersteps-selector-wrapper" style="width: 100%">
-                                    <span class="loader" >
-                                        <span id="browsersteps-click-start">
-                                            <h2 >Click here to Start</h2>
-                                            Please allow 10-15 seconds for the browser to connect.
-                                        </span>
-                                        <div class="spinner"  style="display: none;"></div>
-                                    </span>
-                                    <img  class="noselect" id="browsersteps-img" src="" style="max-width: 100%; width: 100%;" />
-                                    <canvas  class="noselect" id="browsersteps-selector-canvas" style="max-width: 100%; width: 100%;"></canvas>
-                                </div>
-                            </div>
-                            <div id="browser-steps-fieldlist" style="padding-left: 1em;  width: 350px; font-size: 80%;" >
-                                <span id="browserless-seconds-remaining">Loading</span> <span style="font-size: 80%;"> (<a target=_new href="https://github.com/dgtlmoon/changedetection.io/pull/478/files#diff-1a79d924d1840c485238e66772391268a89c95b781d69091384cf1ea1ac146c9R4">?</a>) </span>
-                                {{ render_field(form.browser_steps) }}
-                            </div>
-                        </div>
-                    </div>
-                </fieldset>
-            </div>
-            {% endif %}
 
             <div class="tab-pane-inner" id="notifications">
+                <strong>Note: <i>These settings override the global settings for this watch.</i></strong>
                 <fieldset>
-                    <div  class="pure-control-group inline-radio">
+                    <div class="field-group">
-                      {{ render_checkbox_field(form.notification_muted) }}
+                        {{ render_common_settings_form(form, current_base_url, emailprefix) }}
-                    </div>
-                    {% if is_html_webdriver %}
-                    <div class="pure-control-group inline-radio">
-                      {{ render_checkbox_field(form.notification_screenshot) }}
-                        <span class="pure-form-message-inline">
-                            <strong>Use with caution!</strong> This will easily fill up your email storage quota or flood other storages.
-                        </span>
-                    </div>
-                    {% endif %}
-                    <div class="field-group" id="notification-field-group">
-                        {% if has_default_notification_urls %}
-                        <div class="inline-warning">
-                            <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!"/>
-                            There are <a href="{{ url_for('settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
-                        </div>
-                        {% endif %}
-                        <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
-
-                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
                     </div>
                 </fieldset>
             </div>
@@ -233,30 +162,19 @@ User-Agent: wonderbra 1.0") }}
                         </div>
                     </fieldset>
                     <div class="pure-control-group">
-                        {% set field = render_field(form.include_filters,
+                        {% set field = render_field(form.css_filter,
-                            rows=5,
+                            placeholder=".class-name or #some-id, or other CSS selector rule.",
-                            placeholder="#example
-xpath://body/div/span[contains(@class, 'example-class')]",
                             class="m-d")
                         %}
                         {{ field }}
                         {% if '/text()' in  field %}
                           <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br/>
                         {% endif %}
-                        <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br/>
+                        <span class="pure-form-message-inline">
-
                     <ul>
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
-                        <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
+                        <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <code>"json:"</code>, use <code>json:$</code> to force re-formatting if required,  <a
-                            <ul>
+                                href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
-                                <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
-                                {% if jq_support %}
-                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
-                                {% else %}
-                                <li>jq support not installed</li>
-                                {% endif %}
-                            </ul>
-                        </li>
                         <li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
                             <ul>
                                 <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
@@ -265,7 +183,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                             </ul>
                             </li>
                     </ul>
-                    Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
+                    Please be sure that you thoroughly understand how to write CSS or JSONPath, XPath selector rules before filing an issue on GitHub! <a
                                 href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
                 </span>
                     </div>
@@ -347,32 +265,39 @@ Unavailable") }}
             </div>
 
             <div class="tab-pane-inner visual-selector-ui" id="visualselector">
-                <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}">
+                <img id="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}">
-
+                <strong>Pro-tip:</strong> This tool is only for limiting which elements will be included on a change-detection, not for interacting with browser directly.
                 <fieldset>
                     <div class="pure-control-group">
                         {% if visualselector_enabled %}
+                            {% if visualselector_data_is_ready %}
+                                <div id="selector-header">
+                                    <a id="clear-selector" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Clear selection</a>
+                                    <i class="fetching-update-notice" style="font-size: 80%;">One moment, fetching screenshot and element information..</i>
+                                </div>
+                                <div id="selector-wrapper">
+                                    <!-- request the screenshot and get the element offset info ready -->
+                                    <!-- use img src ready load to know everything is ready to map out -->
+                                    <!-- @todo: maybe something interesting like a field to select 'elements that contain text... and their parents n' -->
+                                    <img id="selector-background" />
+                                    <canvas id="selector-canvas"></canvas>
+
+                                </div>
+                                <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
+
                             <span class="pure-form-message-inline">
-                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection &dash; after the <i>Browser Steps</i> has completed.<br/><br/>
+                                <p><span style="font-weight: bold">Beta!</span> The Visual Selector is new and there may be minor bugs, please report pages that dont work, help us to improve this software!</p>
                             </span>
 
-                            <div id="selector-header">
+                            {% else %}
-                                <a id="clear-selector" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Clear selection</a>
+                                <span class="pure-form-message-inline">Screenshot and element data is not available or not yet ready.</span>
-                                <i class="fetching-update-notice" style="font-size: 80%;">One moment, fetching screenshot and element information..</i>
+                            {% endif %}
-                            </div>
-                            <div id="selector-wrapper" style="display: none">
-                                <!-- request the screenshot and get the element offset info ready -->
-                                <!-- use img src ready load to know everything is ready to map out -->
-                                <!-- @todo: maybe something interesting like a field to select 'elements that contain text... and their parents n' -->
-                                <img id="selector-background" />
-                                <canvas id="selector-canvas"></canvas>
-                            </div>
-                            <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
                         {% else %}
                             <span class="pure-form-message-inline">
                                 <p>Sorry, this functionality only works with Playwright/Chrome enabled watches.</p>
                                 <p>Enable the Playwright Chrome fetcher, or alternatively try our <a href="https://lemonade.changedetection.io/start">very affordable subscription based service</a>.</p>
                                 <p>This is because Selenium/WebDriver can not extract full page screenshots reliably.</p>
+
                             </span>
                         {% endif %}
                     </div>

changedetectionio/templates/settings.html

@@ -60,7 +60,7 @@
                         {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
                         class="m-d") }}
                         <span class="pure-form-message-inline">
-                            Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"),
+                            Base URL used for the {base_url} token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{current_base_url}}"),
                             <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
                         </span>
                     </div>
@@ -87,7 +87,7 @@
             <div class="tab-pane-inner" id="notifications">
                 <fieldset>
                     <div class="field-group">
-                        {{ render_common_settings_form(form.application.form, emailprefix, settings_application) }}
+                        {{ render_common_settings_form(form.application.form, current_base_url, emailprefix) }}
                     </div>
                 </fieldset>
             </div>
@@ -99,8 +99,6 @@
                         <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
                         <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                     </span>
-                    <br/>
-                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using BrightData Proxies, find out more here.</a>
                 </div>
                 <fieldset class="pure-group" id="webdriver-override-options">
                     <div class="pure-form-message-inline">

changedetectionio/templates/svgs/dark-mode-toggle-icon.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" shape-rendering="geometricPrecision" text-rendering="geometricPrecision" image-rendering="optimizeQuality" fill-rule="evenodd" clip-rule="evenodd" viewBox="0 0 512 262.86"><path fill-rule="nonzero" d="M316.78 16.55h-205.9c-30.5 0-58.22 12.48-78.31 32.57C12.47 69.21 0 96.93 0 127.44c0 30.5 12.47 58.22 32.57 78.31 20.09 20.1 47.81 32.57 78.31 32.57h193.25c21.54 15.43 47.9 24.54 76.26 24.54h.18c36.14 0 69.02-14.79 92.83-38.6 23.8-23.81 38.6-56.67 38.6-92.83 0-36.15-14.78-69.03-38.63-92.8C449.53 14.8 416.67 0 380.57 0h-.18c-23.02 0-44.72 6.02-63.61 16.55zm70.62 97.17.43.09c.82-3.45 2.83-6.19 6.04-8.16 3.2-1.98 6.53-2.57 10.01-1.75l.1-.43c-3.47-.82-6.2-2.83-8.17-6.03-1.98-3.22-2.57-6.55-1.75-10.01l-.43-.1c-.82 3.47-2.83 6.2-6.03 8.18-3.21 1.98-6.55 2.56-10.02 1.74l-.1.43c3.47.82 6.2 2.84 8.18 6.04 1.99 3.19 2.56 6.52 1.74 10zm36.87 16.77.53.12c1.02-4.35 3.55-7.78 7.58-10.26 4.02-2.49 8.2-3.22 12.56-2.19l.13-.53c-4.35-1.03-7.78-3.55-10.26-7.59-2.49-4.03-3.22-8.22-2.2-12.56l-.53-.12c-1.02 4.35-3.55 7.77-7.58 10.26-4.02 2.49-8.21 3.22-12.56 2.19l-.13.53c4.36 1.03 7.78 3.55 10.26 7.58 2.49 4.02 3.22 8.22 2.2 12.57zm-38.79-61.01c-15.69 7.67-26.98 23.26-28.29 41.93-1.96 27.88 19.05 52.06 46.92 54.02 13.23.93 25.64-3.32 35.22-11.02 4.75-3.82 9.66-.45 7.59 4.36-11.33 26.42-38.45 44.04-68.74 41.91-38.29-2.69-67.14-35.91-64.45-74.19C316.3 89.8 347.05 61.67 383.44 62c6.71.06 8.13 4.5 2.04 7.48zm-5.09-53.95h.18c63.75 0 115.91 52.15 115.91 115.9 0 63.75-52.23 115.91-115.91 115.91h-.18c-63.68 0-115.91-52.16-115.91-115.91s52.16-115.9 115.91-115.9z"/></svg>

changedetectionio/templates/svgs/github.svg

@@ -1,3 +0,0 @@
-<svg class="octicon octicon-mark-github v-align-middle" height="32" viewbox="0 0 16 16" version="1.1" width="32" aria-hidden="true">
-  <path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
-</svg>

changedetectionio/templates/svgs/light-mode-toggle-icon.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" shape-rendering="geometricPrecision" text-rendering="geometricPrecision" image-rendering="optimizeQuality" fill-rule="evenodd" clip-rule="evenodd" viewBox="0 0 512 256.04"><path d="M128.02 0h.18c22.03 0 42.83 5.66 61 15.6h210.38c30.89 0 59 12.65 79.38 33.04C499.35 68.99 512 97.1 512 128.02c0 30.92-12.66 59.03-33.02 79.4l-.42.38c-20.34 20.15-48.29 32.64-78.98 32.64H189.24c-18.17 9.93-38.98 15.6-61.04 15.6h-.18c-35.2 0-67.22-14.41-90.42-37.6C14.41 195.25 0 163.24 0 128.02s14.4-67.24 37.59-90.43l.91-.83C61.65 14.05 93.29 0 128.02 0zm-5.95 54.42c0-1.95.8-3.73 2.08-5 2.74-2.77 7.27-2.76 10.02-.01l.14.16a7.042 7.042 0 0 1 1.94 4.85v12.95c0 1.95-.8 3.73-2.08 5.01-2.75 2.75-7.27 2.75-10.02 0a7.084 7.084 0 0 1-2.08-5.01V54.42zm6.05 31.17c11.72 0 22.32 4.75 30 12.43 7.67 7.68 12.43 18.29 12.43 30 0 11.72-4.75 22.32-12.43 30s-18.28 12.43-30 12.43c-11.72 0-22.32-4.75-30.01-12.43-7.67-7.68-12.43-18.28-12.43-30 0-11.72 4.76-22.32 12.43-30 7.69-7.67 18.3-12.43 30.01-12.43zm-56.33-5.34a7.114 7.114 0 0 1-2.07-5.01c0-3.9 3.18-7.09 7.09-7.09 1.81 0 3.62.69 5 2.07l9.16 9.16a7.065 7.065 0 0 1 2.08 5.01c0 1.8-.7 3.62-2.08 5.01a7.057 7.057 0 0 1-5.01 2.08c-1.8 0-3.61-.7-5-2.07l-9.17-9.16zm-17.28 53.81c-1.95 0-3.73-.8-5-2.08-2.77-2.74-2.76-7.27-.01-10.01l.15-.14a7.04 7.04 0 0 1 4.86-1.94h12.94a7.082 7.082 0 0 1 7.09 7.09c0 1.95-.8 3.73-2.07 5.01a7.099 7.099 0 0 1-5.02 2.07H54.51zm25.82 50.28a7.049 7.049 0 0 1-5 2.07c-3.91 0-7.09-3.16-7.09-7.08 0-1.81.68-3.62 2.07-5.01l9.31-9.29a7.02 7.02 0 0 1 4.86-1.94 7.09 7.09 0 0 1 7.09 7.09c0 1.79-.69 3.6-2.08 4.99l-9.16 9.17zm53.82 17.29c0 1.94-.8 3.73-2.08 5-2.74 2.76-7.27 2.75-10.02 0l-.13-.15a7.033 7.033 0 0 1-1.94-4.85v-12.95c0-1.96.8-3.73 2.07-5.01 2.76-2.75 7.27-2.75 10.03 0a7.1 7.1 0 0 1 2.07 5.01v12.95zm50.28-25.83a7.055 7.055 0 0 1 2.07 5.01c0 3.89-3.18 7.09-7.08 7.09-1.81 0-3.63-.69-5.01-2.07l-9.16-9.16a7.095 7.095 0 0 1-2.07-5.02c0-3.9 3.18-7.09 7.08-7.09 1.8 0 3.61.7 5 2.08l9.17 9.16zm17.29-53.82c1.93 0 3.73.81 5 2.08 2.76 2.75 2.75 7.27 0 10.02l-.15.14a7.098 7.098 0 0 1-4.85 1.94h-12.95c-1.96 0-3.74-.8-5.01-2.08-2.76-2.75-2.76-7.27 0-10.02a7.049 7.049 0 0 1 5.01-2.08h12.95zM175.89 71.7a7.074 7.074 0 0 1 5-2.07c3.9 0 7.1 3.19 7.1 7.09 0 1.81-.69 3.62-2.07 5l-9.32 9.31a7.12 7.12 0 0 1-4.86 1.93c-3.91 0-7.09-3.18-7.09-7.09 0-1.8.7-3.61 2.08-5l9.16-9.17zm34.17-41.87c2.96 2.47 5.81 5.07 8.53 7.8 23.22 23.15 37.63 55.17 37.63 90.39s-14.42 67.23-37.6 90.42a130.2 130.2 0 0 1-8.5 7.77h189.46c26.83 0 51.24-10.91 69.02-28.5l.32-.35c17.79-17.79 28.85-42.35 28.85-69.34 0-26.99-11.06-51.55-28.85-69.35-17.77-17.8-42.33-28.84-69.34-28.84H210.06zm-82.04-14.71h.18c62.09 0 112.89 50.81 112.89 112.9 0 62.1-50.86 112.9-112.89 112.9h-.18c-62.03 0-112.9-50.8-112.9-112.9 0-62.09 50.81-112.9 112.9-112.9z"/></svg>

changedetectionio/templates/watch-overview.html

@@ -14,7 +14,7 @@
             <div id="watch-add-wrapper-zone">
                 <div>
                     {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch label / tag") }}
+                    {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch group") }}
                 </div>
                 <div>
                     {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
@@ -30,9 +30,6 @@
     <div id="checkbox-operations">
         <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="pause">Pause</button>
         <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="unpause">UnPause</button>
-        <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="mute">Mute</button>
-        <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="unmute">UnMute</button>
-        <button class="pure-button button-secondary button-xsmall" style="font-size: 70%" name="op" value="notification-default">Use default notification</button>
         <button class="pure-button button-secondary button-xsmall" style="background: #dd4242; font-size: 70%" name="op" value="delete">Delete</button>
     </div>
     <div>
@@ -79,30 +76,20 @@
                 {% if watch.uuid in queued_uuids %}queued{% endif %}">
                 <td class="inline checkbox-uuid" ><input name="uuids"  type="checkbox" value="{{ watch.uuid}} "/> <span>{{ loop.index }}</span></td>
                 <td class="inline watch-controls">
-                    {% if not watch.paused %}
+                    <a class="state-{{'on' if watch.paused }}" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks"/></a>
-                    <a class="state-off" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause"/></a>
+                    <a class="state-{{'on' if watch.notification_muted}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications"/></a>
-                    {% else %}
-                    <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause"/></a>
-                    {% endif %}
-                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute"/></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
-                    <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
+                    <a class="external" target="_blank" rel="noopener" href="{{ watch.url.replace('source:','') }}"></a>
-                    <a class="link-spread" href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="icon icon-spread" title="Create a link to share watch config with others" /></a>
+                    <a href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
 
-                    {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a chrome browser" />{% endif %}
+                    {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
 
                     {% if watch.last_error is defined and watch.last_error != False %}
                     <div class="fetch-error">{{ watch.last_error }}</div>
                     {% endif %}
                     {% if watch.last_notification_error is defined and watch.last_notification_error != False %}
-                    <div class="fetch-error notification-error"><a href="{{url_for('notification_logs')}}">{{ watch.last_notification_error }}</a></div>
+                    <div class="fetch-error notification-error">{{ watch.last_notification_error }}</div>
-                    {% endif %}
-                    {% if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data']  %}
-                    <div class="ldjson-price-track-offer">Embedded price data detected, follow only price data? <a href="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">Yes</a> <a href="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="">No</a></div>
-                    {% endif %}
-                    {% if watch['track_ldjson_price_data'] == 'accepted' %}
-                    <span class="tracking-ldjson-price-data" title="Automatically following embedded price information"><img src="{{url_for('static_content', group='images', filename='price-tag-icon.svg')}}"  class="price-follow-tag-icon"/> Price</span>
                     {% endif %}
                     {% if not active_tag %}
                     <span class="watch-tag-list">{{ watch.tag}}</span>
@@ -117,13 +104,13 @@
                 </td>
                 <td>
                     <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('form_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
-                       class="recheck pure-button pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
+                       class="recheck pure-button button-small pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
-                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button pure-button-primary">Edit</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
                     {% if watch.history_n >= 2 %}
-                    <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button pure-button-primary diff-link">Diff</a>
+                    <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary diff-link">Diff</a>
                     {% else %}
                         {% if watch.history_n == 1 or (watch.history_n ==0 and watch.error_text_ctime )%}
-                            <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button pure-button-primary">Preview</a>
+                            <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
                         {% endif %}
                     {% endif %}
                 </td>

changedetectionio/tests/conftest.py

@@ -41,7 +41,7 @@ def app(request):
 
     cleanup(datastore_path)
 
-    app_config = {'datastore_path': datastore_path, 'disable_checkver' : True}
+    app_config = {'datastore_path': datastore_path}
     cleanup(app_config['datastore_path'])
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
     app = changedetection_app(app_config, datastore)

changedetectionio/tests/fetchers/test_content.py

@@ -5,7 +5,7 @@ from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks
 import logging
 
-# Requires playwright to be installed
+
 def test_fetch_webdriver_content(client, live_server):
     live_server_setup(live_server)
 

changedetectionio/tests/proxy_list/__init__.py

@@ -1,2 +0,0 @@
-"""Tests for the app."""
-

changedetectionio/tests/proxy_list/conftest.py

@@ -1,14 +0,0 @@
-#!/usr/bin/python3
-
-from .. import conftest
-
-#def pytest_addoption(parser):
-#    parser.addoption("--url_suffix", action="store", default="identifier for request")
-
-
-#def pytest_generate_tests(metafunc):
-#    # This is called for every test. Only get/set command line arguments
-#    # if the argument is specified in the list of test "fixturenames".
-#    option_value = metafunc.config.option.url_suffix
-#    if 'url_suffix' in metafunc.fixturenames and option_value is not None:
-#        metafunc.parametrize("url_suffix", [option_value])

changedetectionio/tests/proxy_list/proxies.json-example

@@ -1,10 +0,0 @@
-{
-  "proxy-one": {
-    "label": "One",
-    "url": "http://127.0.0.1:3128"
-  },
-  "proxy-two": {
-    "label": "two",
-    "url": "http://127.0.0.1:3129"
-  }
-}

changedetectionio/tests/proxy_list/squid.conf

@@ -1,41 +0,0 @@
-acl localnet src 0.0.0.1-0.255.255.255  # RFC 1122 "this" network (LAN)
-acl localnet src 10.0.0.0/8             # RFC 1918 local private network (LAN)
-acl localnet src 100.64.0.0/10          # RFC 6598 shared address space (CGN)
-acl localnet src 169.254.0.0/16         # RFC 3927 link-local (directly plugged) machines
-acl localnet src 172.16.0.0/12          # RFC 1918 local private network (LAN)
-acl localnet src 192.168.0.0/16         # RFC 1918 local private network (LAN)
-acl localnet src fc00::/7               # RFC 4193 local private network range
-acl localnet src fe80::/10              # RFC 4291 link-local (directly plugged) machines
-acl localnet src 159.65.224.174
-acl SSL_ports port 443
-acl Safe_ports port 80          # http
-acl Safe_ports port 21          # ftp
-acl Safe_ports port 443         # https
-acl Safe_ports port 70          # gopher
-acl Safe_ports port 210         # wais
-acl Safe_ports port 1025-65535  # unregistered ports
-acl Safe_ports port 280         # http-mgmt
-acl Safe_ports port 488         # gss-http
-acl Safe_ports port 591         # filemaker
-acl Safe_ports port 777         # multiling http
-acl CONNECT method CONNECT
-
-http_access deny !Safe_ports
-http_access deny CONNECT !SSL_ports
-http_access allow localhost manager
-http_access deny manager
-http_access allow localhost
-http_access allow localnet
-http_access deny all
-http_port 3128
-coredump_dir /var/spool/squid
-refresh_pattern ^ftp:           1440    20%     10080
-refresh_pattern ^gopher:        1440    0%      1440
-refresh_pattern -i (/cgi-bin/|\?) 0     0%      0
-refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims
-refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims
-refresh_pattern \/InRelease$ 0 0% 0 refresh-ims
-refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims
-refresh_pattern .               0       20%     4320
-logfile_rotate 0
-

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -1,38 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from ..util import live_server_setup
-
-def test_preferred_proxy(client, live_server):
-    time.sleep(1)
-    live_server_setup(live_server)
-    time.sleep(1)
-    url = "http://chosen.changedetection.io"
-
-    res = client.post(
-        url_for("import_page"),
-        # Because a URL wont show in squid/proxy logs due it being SSLed
-        # Use plain HTTP or a specific domain-name here
-        data={"urls": url},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-
-    time.sleep(2)
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-                "include_filters": "",
-                "fetch_backend": "html_requests",
-                "headers": "",
-                "proxy": "proxy-two",
-                "tag": "",
-                "url": url,
-              },
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    time.sleep(2)
-    # Now the request should appear in the second-squid logs

changedetectionio/tests/proxy_list/test_proxy.py

@@ -1,19 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
-
-# just make a request, we will grep in the docker logs to see it actually got called
-def test_check_basic_change_detection_functionality(client, live_server):
-    live_server_setup(live_server)
-    res = client.post(
-        url_for("import_page"),
-        # Because a URL wont show in squid/proxy logs due it being SSLed
-        # Use plain HTTP or a specific domain-name here
-        data={"urls": "http://one.changedetection.io"},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-    time.sleep(3)

changedetectionio/tests/test_api.py

@@ -147,16 +147,6 @@ def test_api_simple(client, live_server):
     # @todo how to handle None/default global values?
     assert watch['history_n'] == 2, "Found replacement history section, which is in its own API"
 
-    # basic systeminfo check
-    res = client.get(
-        url_for("systeminfo"),
-        headers={'x-api-key': api_key},
-    )
-    info = json.loads(res.data)
-    assert info.get('watch_count') == 1
-    assert info.get('uptime') > 0.5
-
-
     # Finally delete the watch
     res = client.delete(
         url_for("watch", uuid=watch_uuid),

changedetectionio/tests/test_auth.py

@@ -19,16 +19,17 @@ def test_basic_auth(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
 
     # Check form validation
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"css_filter": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(1)
     res = client.get(
         url_for("preview_page", uuid="first"),

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -1,146 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from .util import live_server_setup, extract_UUID_from_client, extract_api_key_from_UI
-
-def set_response_with_ldjson():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>Which is across multiple lines</p>
-     </br>
-     So let's see what happens.  </br>
-     <div class="sametext">Some text thats the same</div>
-     <div class="changetext">Some text that will change</div>
-     <script type="application/ld+json">
-        {
-           "@context":"https://schema.org/",
-           "@type":"Product",
-           "@id":"https://www.some-virtual-phone-shop.com/celular-iphone-14/p",
-           "name":"Celular Iphone 14 Pro Max 256Gb E Sim A16 Bionic",
-           "brand":{
-              "@type":"Brand",
-              "name":"APPLE"
-           },
-           "image":"https://www.some-virtual-phone-shop.com/15509426/image.jpg",
-           "description":"You dont need it",
-           "mpn":"111111",
-           "sku":"22222",
-           "offers":{
-              "@type":"AggregateOffer",
-              "lowPrice":8097000,
-              "highPrice":8099900,
-              "priceCurrency":"COP",
-              "offers":[
-                 {
-                    "@type":"Offer",
-                    "price":8097000,
-                    "priceCurrency":"COP",
-                    "availability":"http://schema.org/InStock",
-                    "sku":"102375961",
-                    "itemCondition":"http://schema.org/NewCondition",
-                    "seller":{
-                       "@type":"Organization",
-                       "name":"ajax"
-                    }
-                 }
-              ],
-              "offerCount":1
-           }
-        }
-       </script>
-     </body>
-     </html>
-"""
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-def set_response_without_ldjson():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>Which is across multiple lines</p>
-     </br>
-     So let's see what happens.  </br>
-     <div class="sametext">Some text thats the same</div>
-     <div class="changetext">Some text that will change</div>     
-     </body>
-     </html>
-"""
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-# actually only really used by the distll.io importer, but could be handy too
-def test_check_ldjson_price_autodetect(client, live_server):
-    live_server_setup(live_server)
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
-    set_response_with_ldjson()
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    time.sleep(3)
-
-    # Should get a notice that it's available
-    res = client.get(url_for("index"))
-    assert b'ldjson-price-track-offer' in res.data
-
-    # Accept it
-    uuid = extract_UUID_from_client(client)
-
-    client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
-    time.sleep(2)
-
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
-    # Offer should be gone
-    res = client.get(url_for("index"))
-    assert b'Embedded price data' not in res.data
-    assert b'tracking-ldjson-price-data' in res.data
-
-    # and last snapshop (via API) should be just the price
-    api_key = extract_api_key_from_UI(client)
-    res = client.get(
-        url_for("watchsinglehistory", uuid=uuid, timestamp='latest'),
-        headers={'x-api-key': api_key},
-    )
-
-    # Should see this (dont know where the whitespace came from)
-    assert b'"highPrice": 8099900' in res.data
-    # And not this cause its not the ld-json
-    assert b"So let's see what happens" not in res.data
-
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-
-    ##########################################################################################
-    # And we shouldnt see the offer
-    set_response_without_ldjson()
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    time.sleep(3)
-    res = client.get(url_for("index"))
-    assert b'ldjson-price-track-offer' not in res.data
-    
-    ##########################################################################################
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_backend.py

@@ -3,7 +3,7 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, live_server_setup
 
 sleep_time_for_fetch_thread = 3
 
@@ -36,7 +36,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
         client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
-        wait_for_all_checks(client)
+        time.sleep(sleep_time_for_fetch_thread)
 
         # It should report nothing found (no new 'unviewed' class)
         res = client.get(url_for("index"))
@@ -69,7 +69,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
     assert b'1 watches are queued for rechecking.' in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # Now something should be ready, indicated by having a 'unviewed' class
     res = client.get(url_for("index"))
@@ -98,14 +98,14 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'which has this one new line' in res.data
     assert b'Which is across multiple lines' not in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     # Do this a few times.. ensures we dont accidently set the status
     for n in range(2):
         client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
-        wait_for_all_checks(client)
+        time.sleep(sleep_time_for_fetch_thread)
 
         # It should report nothing found (no new 'unviewed' class)
         res = client.get(url_for("index"))
@@ -125,7 +125,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
     )
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data

changedetectionio/tests/test_backup.py

@@ -1,31 +1,18 @@
 #!/usr/bin/python3
 
-from .util import set_original_response, set_modified_response, live_server_setup
+import time
 from flask import url_for
 from urllib.request import urlopen
-from zipfile import ZipFile
+from . util import set_original_response, set_modified_response, live_server_setup
-import re
-import time
 
 
 def test_backup(client, live_server):
-    live_server_setup(live_server)
 
-    set_original_response()
+    live_server_setup(live_server)
 
     # Give the endpoint time to spin up
     time.sleep(1)
 
-    # Add our URL to the import page
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": url_for('test_endpoint', _external=True)},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-    time.sleep(3)
-
     res = client.get(
         url_for("get_backup"),
         follow_redirects=True
@@ -33,19 +20,6 @@ def test_backup(client, live_server):
 
     # Should get the right zip content type
     assert res.content_type == "application/zip"
-
     # Should be PK/ZIP stream
     assert res.data.count(b'PK') >= 2
 
-    # ZipFile from buffer seems non-obvious, just save it instead
-    with open("download.zip", 'wb') as f:
-        f.write(res.data)
-
-    zip = ZipFile('download.zip')
-    l = zip.namelist()
-    uuid4hex = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.*txt', re.I)
-    newlist = list(filter(uuid4hex.match, l))  # Read Note below
-
-    # Should be two txt files in the archive (history and the snapshot)
-    assert len(newlist) == 2
-

changedetectionio/tests/test_css_selector.py

@@ -46,23 +46,22 @@ def set_modified_response():
 
 
 # Test that the CSS extraction works how we expect, important here is the right placing of new lines \n's
-def test_include_filters_output():
+def test_css_filter_output():
+    from changedetectionio import fetch_site_status
     from inscriptis import get_text
 
     # Check text with sub-parts renders correctly
     content = """<html> <body><div id="thingthing" >  Some really <b>bold</b> text  </div> </body> </html>"""
-    html_blob = include_filters(include_filters="#thingthing", html_content=content)
+    html_blob = css_filter(css_filter="#thingthing", html_content=content)
     text = get_text(html_blob)
     assert text == "  Some really bold text"
 
     content = """<html> <body>
     <p>foo bar blah</p>
-    <DIV class="parts">Block A</DiV> <div class="parts">Block B</DIV></body> 
+    <div class="parts">Block A</div> <div class="parts">Block B</div></body> 
     </html>
 """
-
+    html_blob = css_filter(css_filter=".parts", html_content=content)
-    # in xPath this would be //*[@class='parts']
-    html_blob = include_filters(include_filters=".parts", html_content=content)
     text = get_text(html_blob)
 
     # Divs are converted to 4 whitespaces by inscriptis
@@ -70,10 +69,10 @@ def test_include_filters_output():
 
 
 # Tests the whole stack works with the CSS Filter
-def test_check_markup_include_filters_restriction(client, live_server):
+def test_check_markup_css_filter_restriction(client, live_server):
     sleep_time_for_fetch_thread = 3
 
-    include_filters = "#sametext"
+    css_filter = "#sametext"
 
     set_original_response()
 
@@ -89,6 +88,9 @@ def test_check_markup_include_filters_restriction(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -96,16 +98,19 @@ def test_check_markup_include_filters_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    time.sleep(1)
+
     # Check it saved
     res = client.get(
         url_for("edit_page", uuid="first"),
     )
-    assert bytes(include_filters.encode('utf-8')) in res.data
+    assert bytes(css_filter.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
@@ -121,58 +126,3 @@ def test_check_markup_include_filters_restriction(client, live_server):
     # Because it should be looking at only that 'sametext' id
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
-
-
-# Tests the whole stack works with the CSS Filter
-def test_check_multiple_filters(client, live_server):
-    sleep_time_for_fetch_thread = 3
-
-    include_filters = "#blob-a\r\nxpath://*[contains(@id,'blob-b')]"
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write("""<html><body>
-     <div id="blob-a">Blob A</div>
-     <div id="blob-b">Blob B</div>
-     <div id="blob-c">Blob C</div>
-     </body>
-     </html>
-    """)
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    time.sleep(1)
-
-    # Goto the edit page, add our ignore text
-    # Add our URL to the import page
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"include_filters": include_filters,
-              "url": test_url,
-              "tag": "",
-              "headers": "",
-              'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    # Only the two blobs should be here
-    assert b"Blob A" in res.data # CSS was ok
-    assert b"Blob B" in res.data # xPath was ok
-    assert b"Blob C" not in res.data # Should not be included

changedetectionio/tests/test_element_removal.py

@@ -121,7 +121,7 @@ def test_element_removal_full(client, live_server):
         url_for("import_page"), data={"urls": test_url}, follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(1)
+
     # Goto the edit page, add the filter data
     # Not sure why \r needs to be added - absent of the #changetext this is not necessary
     subtractive_selectors_data = "header\r\nfooter\r\nnav\r\n#changetext"

changedetectionio/tests/test_encoding.py

@@ -38,6 +38,9 @@ def test_check_encoding_detection(client, live_server):
         follow_redirects=True
     )
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(2)
 
@@ -67,6 +70,9 @@ def test_check_encoding_detection_missing_content_type_header(client, live_serve
         follow_redirects=True
     )
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(2)
 

changedetectionio/tests/test_errorhandling.py

@@ -77,8 +77,7 @@ def test_DNS_errors(client, live_server):
     time.sleep(3)
 
     res = client.get(url_for("index"))
-    found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
+    assert b'Name or service not known' in res.data
-    assert found_name_resolution_error
     # Should always record that we tried
     assert bytes("just now".encode('utf-8')) in res.data
 

changedetectionio/tests/test_extract_csv.py

@@ -1,70 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
-
-sleep_time_for_fetch_thread = 3
-
-
-
-def test_check_extract_text_from_diff(client, live_server):
-    import time
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write("Now it's {} seconds since epoch, time flies!".format(str(time.time())))
-
-    live_server_setup(live_server)
-
-    # Add our URL to the import page
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": url_for('test_endpoint', _external=True)},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-    time.sleep(1)
-
-    # Load in 5 different numbers/changes
-    last_date=""
-    for n in range(5):
-        # Give the thread time to pick it up
-        print("Bumping snapshot and checking.. ", n)
-        last_date = str(time.time())
-        with open("test-datastore/endpoint-content.txt", "w") as f:
-            f.write("Now it's {} seconds since epoch, time flies!".format(last_date))
-
-        client.get(url_for("form_watch_checknow"), follow_redirects=True)
-        wait_for_all_checks(client)
-
-    res = client.post(
-        url_for("diff_history_page", uuid="first"),
-        data={"extract_regex": "Now it's ([0-9\.]+)",
-              "extract_submit_button": "Extract as CSV"},
-        follow_redirects=False
-    )
-
-    assert b'Nothing matches that RegEx' not in res.data
-    assert res.content_type == 'text/csv'
-
-    # Read the csv reply as stringio
-    from io import StringIO
-    import csv
-
-    f = StringIO(res.data.decode('utf-8'))
-    reader = csv.reader(f, delimiter=',')
-    output=[]
-
-    for row in reader:
-        output.append(row)
-
-    assert output[0][0] == 'Epoch seconds'
-
-    # Header line + 1 origin/first + 5 changes
-    assert(len(output) == 7)
-
-    # We expect to find the last bumped date in the changes in the last field of the spreadsheet
-    assert(output[6][2] == last_date)
-    # And nothing else, only that group () of the decimal and .
-    assert "time flies" not in output[6][2]

changedetectionio/tests/test_extract_regex.py

@@ -88,7 +88,7 @@ def test_check_filter_multiline(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": '',
+        data={"css_filter": '',
               'extract_text': '/something.+?6 billion.+?lines/si',
               "url": test_url,
               "tag": "",
@@ -116,7 +116,7 @@ def test_check_filter_multiline(client, live_server):
 
 def test_check_filter_and_regex_extract(client, live_server):
     sleep_time_for_fetch_thread = 3
-    include_filters = ".changetext"
+    css_filter = ".changetext"
 
     set_original_response()
 
@@ -143,7 +143,7 @@ def test_check_filter_and_regex_extract(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": include_filters,
+        data={"css_filter": css_filter,
               'extract_text': '\d+ online\r\n\d+ guests\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i',
               "url": test_url,
               "tag": "",

changedetectionio/tests/test_filter_exist_changes.py

@@ -73,17 +73,17 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     # Just a regular notification setting, this will be used by the special 'filter not found' notification
     notification_form_data = {"notification_urls": notification_url,
-                              "notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
+                              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
-                              "notification_body": "BASE URL: {{base_url}}\n"
+                              "notification_body": "BASE URL: {base_url}\n"
-                                                   "Watch URL: {{watch_url}}\n"
+                                                   "Watch URL: {watch_url}\n"
-                                                   "Watch UUID: {{watch_uuid}}\n"
+                                                   "Watch UUID: {watch_uuid}\n"
-                                                   "Watch title: {{watch_title}}\n"
+                                                   "Watch title: {watch_title}\n"
-                                                   "Watch tag: {{watch_tag}}\n"
+                                                   "Watch tag: {watch_tag}\n"
-                                                   "Preview: {{preview_url}}\n"
+                                                   "Preview: {preview_url}\n"
-                                                   "Diff URL: {{diff_url}}\n"
+                                                   "Diff URL: {diff_url}\n"
-                                                   "Snapshot: {{current_snapshot}}\n"
+                                                   "Snapshot: {current_snapshot}\n"
-                                                   "Diff: {{diff}}\n"
+                                                   "Diff: {diff}\n"
-                                                   "Diff Full: {{diff_full}}\n"
+                                                   "Diff Full: {diff_full}\n"
                                                    ":-)",
                               "notification_format": "Text"}
 
@@ -92,7 +92,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
         "tag": "my tag",
         "title": "my title",
         "headers": "",
-        "include_filters": '.ticket-available',
+        "css_filter": '.ticket-available',
         "fetch_backend": "html_requests"})
 
     res = client.post(

changedetectionio/tests/test_filter_failure_notification.py

@@ -56,17 +56,17 @@ def run_filter_test(client, content_filter):
 
     # Just a regular notification setting, this will be used by the special 'filter not found' notification
     notification_form_data = {"notification_urls": notification_url,
-                              "notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
+                              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
-                              "notification_body": "BASE URL: {{base_url}}\n"
+                              "notification_body": "BASE URL: {base_url}\n"
-                                                   "Watch URL: {{watch_url}}\n"
+                                                   "Watch URL: {watch_url}\n"
-                                                   "Watch UUID: {{watch_uuid}}\n"
+                                                   "Watch UUID: {watch_uuid}\n"
-                                                   "Watch title: {{watch_title}}\n"
+                                                   "Watch title: {watch_title}\n"
-                                                   "Watch tag: {{watch_tag}}\n"
+                                                   "Watch tag: {watch_tag}\n"
-                                                   "Preview: {{preview_url}}\n"
+                                                   "Preview: {preview_url}\n"
-                                                   "Diff URL: {{diff_url}}\n"
+                                                   "Diff URL: {diff_url}\n"
-                                                   "Snapshot: {{current_snapshot}}\n"
+                                                   "Snapshot: {current_snapshot}\n"
-                                                   "Diff: {{diff}}\n"
+                                                   "Diff: {diff}\n"
-                                                   "Diff Full: {{diff_full}}\n"
+                                                   "Diff Full: {diff_full}\n"
                                                    ":-)",
                               "notification_format": "Text"}
 
@@ -76,7 +76,7 @@ def run_filter_test(client, content_filter):
         "title": "my title",
         "headers": "",
         "filter_failure_notification_send": 'y',
-        "include_filters": content_filter,
+        "css_filter": content_filter,
         "fetch_backend": "html_requests"})
 
     res = client.post(
@@ -84,7 +84,6 @@ def run_filter_test(client, content_filter):
         data=notification_form_data,
         follow_redirects=True
     )
-
     assert b"Updated watch." in res.data
     time.sleep(3)
 
@@ -96,7 +95,7 @@ def run_filter_test(client, content_filter):
         time.sleep(3)
 
     # We should see something in the frontend
-    assert b'Warning, no filters were found' in res.data
+    assert b'Warning, filter' in res.data
 
     # Now it should exist and contain our "filter not found" alert
     assert os.path.isfile("test-datastore/notification.txt")
@@ -132,7 +131,7 @@ def run_filter_test(client, content_filter):
 def test_setup(live_server):
     live_server_setup(live_server)
 
-def test_check_include_filters_failure_notification(client, live_server):
+def test_check_css_filter_failure_notification(client, live_server):
     set_original_response()
     time.sleep(1)
     run_filter_test(client, '#nope-doesnt-exist')

changedetectionio/tests/test_ignore_text.py

@@ -101,6 +101,9 @@ def test_check_ignore_text_functionality(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -196,6 +199,9 @@ def test_check_global_ignore_text_functionality(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 

changedetectionio/tests/test_ignorestatuscode.py

@@ -69,6 +69,8 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server):
     assert b"1 Imported" in res.data
 
     time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     set_some_changed_response()
     time.sleep(sleep_time_for_fetch_thread)
@@ -102,6 +104,9 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -114,9 +119,11 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     )
     assert b"Updated watch." in res.data
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
-
     #  Make a change
     set_some_changed_response()
 

changedetectionio/tests/test_jinja2.py

@@ -1,33 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from .util import live_server_setup
-
-
-# If there was only a change in the whitespacing, then we shouldnt have a change detected
-def test_jinja2_in_url_query(client, live_server):
-    live_server_setup(live_server)
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
-    # Add our URL to the import page
-    test_url = url_for('test_return_query', _external=True)
-
-    # because url_for() will URL-encode the var, but we dont here
-    full_url = "{}?{}".format(test_url,
-                              "date={% now 'Europe/Berlin', '%Y' %}.{% now 'Europe/Berlin', '%m' %}.{% now 'Europe/Berlin', '%d' %}", )
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": full_url, "tag": "test"},
-        follow_redirects=True
-    )
-    assert b"Watch added" in res.data
-    time.sleep(3)
-    # It should report nothing found (no new 'unviewed' class)
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    assert b'date=2' in res.data

changedetectionio/tests/test_jsonpath_selector.py

@@ -2,15 +2,10 @@
 # coding=utf-8
 
 import time
-from flask import url_for, escape
+from flask import url_for
 from . util import live_server_setup
 import pytest
-jq_support = True
 
-try:
-    import jq
-except ModuleNotFoundError:
-    jq_support = False
 
 def test_setup(live_server):
     live_server_setup(live_server)
@@ -41,28 +36,16 @@ and it can also be repeated
     from .. import html_tools
 
     # See that we can find the second <script> one, which is not broken, and matches our filter
-    text = html_tools.extract_json_as_string(content, "json:$.offers.price")
+    text = html_tools.extract_json_as_string(content, "$.offers.price")
     assert text == "23.5"
 
-    # also check for jq
+    text = html_tools.extract_json_as_string('{"id":5}', "$.id")
-    if jq_support:
-        text = html_tools.extract_json_as_string(content, "jq:.offers.price")
-        assert text == "23.5"
-
-        text = html_tools.extract_json_as_string('{"id":5}', "jq:.id")
-        assert text == "5"
-
-    text = html_tools.extract_json_as_string('{"id":5}', "json:$.id")
     assert text == "5"
 
     # When nothing at all is found, it should throw JSONNOTFound
     # Which is caught and shown to the user in the watch-overview table
     with pytest.raises(html_tools.JSONNotFound) as e_info:
-        html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "json:$.id")
+        html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "$.id")
-
-    if jq_support:
-        with pytest.raises(html_tools.JSONNotFound) as e_info:
-            html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jq:.id")
 
 def set_original_ext_response():
     data = """
@@ -83,7 +66,6 @@ def set_original_ext_response():
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(data)
-    return None
 
 def set_modified_ext_response():
     data = """
@@ -104,7 +86,6 @@ def set_modified_ext_response():
 
     with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(data)
-    return None
 
 def set_original_response():
     test_return_data = """
@@ -132,7 +113,7 @@ def set_original_response():
     return None
 
 
-def set_json_response_with_html():
+def set_response_with_html():
     test_return_data = """
     {
       "test": [
@@ -176,7 +157,7 @@ def set_modified_response():
 def test_check_json_without_filter(client, live_server):
     # Request a JSON document from a application/json source containing HTML
     # and be sure it doesn't get chewed up by instriptis
-    set_json_response_with_html()
+    set_response_with_html()
 
     # Give the endpoint time to spin up
     time.sleep(1)
@@ -189,6 +170,9 @@ def test_check_json_without_filter(client, live_server):
         follow_redirects=True
     )
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(3)
 
@@ -197,14 +181,13 @@ def test_check_json_without_filter(client, live_server):
         follow_redirects=True
     )
 
-    # Should still see '"html": "<b>"'
     assert b'&#34;&lt;b&gt;' in res.data
     assert res.data.count(b'{\n') >= 2
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
 
-def check_json_filter(json_filter, client, live_server):
+def test_check_json_filter(client, live_server):
+    json_filter = 'json:boss.name'
+
     set_original_response()
 
     # Give the endpoint time to spin up
@@ -219,6 +202,9 @@ def check_json_filter(json_filter, client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(3)
 
@@ -226,7 +212,7 @@ def check_json_filter(json_filter, client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": json_filter,
+        data={"css_filter": json_filter,
               "url": test_url,
               "tag": "",
               "headers": "",
@@ -240,7 +226,10 @@ def check_json_filter(json_filter, client, live_server):
     res = client.get(
         url_for("edit_page", uuid="first"),
     )
-    assert bytes(escape(json_filter).encode('utf-8')) in res.data
+    assert bytes(json_filter.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -263,17 +252,10 @@ def check_json_filter(json_filter, client, live_server):
     # And #462 - check we see the proper utf-8 string there
     assert "Örnsköldsvik".encode('utf-8') in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
 
-def test_check_jsonpath_filter(client, live_server):
+def test_check_json_filter_bool_val(client, live_server):
-    check_json_filter('json:boss.name', client, live_server)
+    json_filter = "json:$['available']"
 
-def test_check_jq_filter(client, live_server):
-    if jq_support:
-        check_json_filter('jq:.boss.name', client, live_server)
-
-def check_json_filter_bool_val(json_filter, client, live_server):
     set_original_response()
 
     # Give the endpoint time to spin up
@@ -293,7 +275,7 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": json_filter,
+        data={"css_filter": json_filter,
               "url": test_url,
               "tag": "",
               "headers": "",
@@ -303,6 +285,11 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     )
     assert b"Updated watch." in res.data
 
+    time.sleep(3)
+
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(3)
     #  Make a change
@@ -317,22 +304,14 @@ def check_json_filter_bool_val(json_filter, client, live_server):
     # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
     assert b'false' in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_check_jsonpath_filter_bool_val(client, live_server):
-    check_json_filter_bool_val("json:$['available']", client, live_server)
-
-def test_check_jq_filter_bool_val(client, live_server):
-    if jq_support:
-        check_json_filter_bool_val("jq:.available", client, live_server)
-
 # Re #265 - Extended JSON selector test
 # Stuff to consider here
 # - Selector should be allowed to return empty when it doesnt match (people might wait for some condition)
 # - The 'diff' tab could show the old and new content
 # - Form should let us enter a selector that doesnt (yet) match anything
-def check_json_ext_filter(json_filter, client, live_server):
+def test_check_json_ext_filter(client, live_server):
+    json_filter = 'json:$[?(@.status==Sold)]'
+
     set_original_ext_response()
 
     # Give the endpoint time to spin up
@@ -347,6 +326,9 @@ def check_json_ext_filter(json_filter, client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
     time.sleep(3)
 
@@ -354,7 +336,7 @@ def check_json_ext_filter(json_filter, client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": json_filter,
+        data={"css_filter": json_filter,
               "url": test_url,
               "tag": "",
               "headers": "",
@@ -368,7 +350,10 @@ def check_json_ext_filter(json_filter, client, live_server):
     res = client.get(
         url_for("edit_page", uuid="first"),
     )
-    assert bytes(escape(json_filter).encode('utf-8')) in res.data
+    assert bytes(json_filter.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
     time.sleep(3)
@@ -391,12 +376,3 @@ def check_json_ext_filter(json_filter, client, live_server):
     assert b'ForSale' not in res.data
     assert b'Sold' in res.data
 
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_check_jsonpath_ext_filter(client, live_server):
-    check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server)
-
-def test_check_jq_ext_filter(client, live_server):
-    if jq_support:
-        check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)