Better type checking

.github/workflows/containers.yml

@@ -50,6 +50,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
 
       - name: Create release metadata
         run: |
@@ -98,8 +99,6 @@ jobs:
           platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
-# Looks like this was disabled
-#          provenance: false
 
       # A new tagged release is required, which builds :tag and :latest
       - name: Build and push :tag
@@ -118,8 +117,6 @@ jobs:
           platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
-# Looks like this was disabled
-#          provenance: false
 
       - name: Image digest
         run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}

.github/workflows/pypi.yml

@@ -0,0 +1,44 @@
+name: PyPi Test and Push tagged release
+
+# Triggers the workflow on push or pull request events
+on:
+  workflow_run:
+    workflows: ["ChangeDetection.io Test"]
+    tags: '*.*'
+    types: [completed]
+
+
+jobs:
+  test-build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+#      - name: Install dependencies
+#        run: |
+#          python -m pip install --upgrade pip
+#          pip install flake8 pytest
+#          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+#          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+      - name: Test that pip builds without error
+        run: |
+          pip3 --version
+          python3 -m pip install wheel
+          python3 setup.py bdist_wheel
+          python3 -m pip install dist/changedetection.io-*-none-any.whl --force
+          changedetection.io -d /tmp -p 10000 &
+          sleep 3
+          curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+          killall -9 changedetection.io
+
+      # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
+      # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
+
+      # https://github.com/docker/buildx/issues/495#issuecomment-918925854
+#if: ${{ github.event_name == 'release'}}

.github/workflows/test-container-build.yml

@@ -10,13 +10,11 @@ on:
     paths:
       - requirements.txt
       - Dockerfile
-      - .github/workflows/*
 
   pull_request:
     paths:
       - requirements.txt
       - Dockerfile
-      - .github/workflows/*
 
   # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
   # @todo: some kind of path filter for requirements.txt and Dockerfile

.github/workflows/test-only.yml

@@ -8,83 +8,32 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-
-      # Mainly just for link/flake8
-      - name: Set up Python 3.10
+      - name: Set up Python 3.9
         uses: actions/setup-python@v2
         with:
-          python-version: '3.10'
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
 
       - name: Lint with flake8
         run: |
-          pip3 install flake8
           # stop the build if there are Python syntax errors or undefined names
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
-      - name: Spin up ancillary testable services
+      - name: Unit tests
         run: |
-          
-          docker network create changedet-network
+          python3 -m unittest changedetectionio.tests.unit.test_notification_diff
 
-          # Selenium+browserless
-          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome-debug:3.141.59
-          docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
-
-      - name: Build changedetection.io container for testing
-        run: |         
-          # Build a changedetection.io container and start testing inside
-          docker build . -t test-changedetectionio
-
-      - name: Test built container with pytest
+      - name: Test with pytest
         run: |
-          
-          # Unit tests
-          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
-          
-          # All tests
-          docker run --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
+          # Each test is totally isolated and performs its own cleanup/reset
+          cd changedetectionio; ./run_all_tests.sh
 
-      - name: Test built container selenium+browserless/playwright
-        run: |
-          
-          # Selenium fetch
-          docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
-          
-          # Playwright/Browserless fetch
-          docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
-          
-          # Settings headers playwright tests - Call back in from Browserless, check headers
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          
-          # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it
-          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 
-      - name: Test with puppeteer fetcher and disk cache
-        run: |
-          docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
-          # Browserless would have had -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" added above
-
-      - name: Test proxy interaction
-        run: |
-          cd changedetectionio
-          ./run_proxy_tests.sh
-          cd ..
-
-      - name: Test changedetection.io container starts+runs basically without error
-        run: |
-          docker run -p 5556:5000 -d test-changedetectionio
-          sleep 3
-          # Should return 0 (no error) when grep finds it
-          curl -s http://localhost:5556 |grep -q checkbox-uuid
-          
-          # and IPv6
-          curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
-          
-
-#export WEBDRIVER_URL=http://localhost:4444/wd/hub
-#pytest tests/fetchers/test_content.py
-#pytest tests/test_errorhandling.py

.github/workflows/test-pip-build.yml

@@ -1,36 +0,0 @@
-name: ChangeDetection.io PIP package test
-
-# Triggers the workflow on push or pull request events
-
-# This line doesnt work, even tho it is the documented one
-on: [push, pull_request]
-
-  # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
-  # @todo: some kind of path filter for requirements.txt and Dockerfile
-jobs:
-  test-pip-build-basics:
-    runs-on: ubuntu-latest
-    steps:
-        - uses: actions/checkout@v2
-
-        - name: Set up Python 3.9
-          uses: actions/setup-python@v2
-          with:
-            python-version: 3.9
-
-
-        - name: Test that the basic pip built package runs without error
-          run: |
-            set -e
-            mkdir dist
-            pip3 install wheel
-            python3 setup.py bdist_wheel            
-            pip3 install -r requirements.txt
-            rm ./changedetection.py
-            rm -rf changedetectio
-            
-            pip3 install dist/changedetection.io*.whl
-            changedetection.io -d /tmp -p 10000 &
-            sleep 3
-            curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
-            killall -9 changedetection.io

CONTRIBUTING.md

@@ -7,3 +7,9 @@ Otherwise, it's always best to PR into the `dev` branch.
 Please be sure that all new functionality has a matching test!
 
 Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notification.py` for example
+
+```
+pip3 install -r requirements-dev
+```
+
+this is from https://github.com/dgtlmoon/changedetection.io/blob/master/requirements-dev.txt

Dockerfile

@@ -1,7 +1,7 @@
 # pip dependencies install stage
-FROM python:3.10-slim-bullseye as builder
+FROM python:3.8-slim as builder
 
-# See `cryptography` pin comment in requirements.txt
+# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -29,16 +29,22 @@ RUN pip install --target=/dependencies playwright~=1.27.1 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.10-slim-bullseye
+FROM python:3.8-slim
 
+# Actual packages needed at runtime, usually due to the notification (apprise) backend
+# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
+ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
+
+# Re #93, #73, excluding rustc (adds another 430Mb~)
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    libssl1.1 \
-    libxslt1.1 \
-    # For pdftohtml
-    poppler-utils \
-    zlib1g \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
-
+    g++ \
+    gcc \
+    libc-dev \
+    libffi-dev \
+    libjpeg-dev \
+    libssl-dev \
+    libxslt-dev \
+    zlib1g-dev
 
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1

MANIFEST.in

@@ -1,11 +1,9 @@
 recursive-include changedetectionio/api *
-recursive-include changedetectionio/blueprint *
-recursive-include changedetectionio/model *
-recursive-include changedetectionio/processors *
-recursive-include changedetectionio/res *
-recursive-include changedetectionio/static *
 recursive-include changedetectionio/templates *
+recursive-include changedetectionio/static *
+recursive-include changedetectionio/model *
 recursive-include changedetectionio/tests *
+recursive-include changedetectionio/res *
 prune changedetectionio/static/package-lock.json
 prune changedetectionio/static/styles/node_modules
 prune changedetectionio/static/styles/package-lock.json

README-pip.md

@@ -2,10 +2,10 @@
 
 Live your data-life pro-actively, track website content changes and receive notifications via Discord, Email, Slack, Telegram and 70+ more
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://changedetection.io)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start?src=pip)
 
 
-[**Don't have time? Let us host it for you! try our extremely affordable subscription use our proxies and support!**](https://changedetection.io) 
+[**Don't have time? Let us host it for you! try our extremely affordable subscription use our proxies and support!**](https://lemonade.changedetection.io/start) 
 
 
 #### Example use cases

README.md

@@ -1,17 +1,15 @@
-## Web Site Change Detection, Restock monitoring and notifications.
+## Web Site Change Detection, Monitoring and Notification.
 
-**_Detect website content changes and perform meaningful actions - trigger notifications via Discord, Email, Slack, Telegram, API calls and many more._**
-
-_Live your data-life pro-actively._ 
+_Live your data-life pro-actively, Detect website changes and perform meaningful actions, trigger notifications via Discord, Email, Slack, Telegram, API calls and many more._
 
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://changedetection.io?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start?src=github)
 
 [![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)
 
 ![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 
-[**Don't have time? Let us host it for you! try our $8.99/month subscription - use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_
+[**Don't have time? Let us host it for you! try our $6.99/month subscription - use our proxies and support!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
 
 - Chrome browser included.
 - Super fast, no registration needed setup.
@@ -22,11 +20,11 @@ _Live your data-life pro-actively._
 
 Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://changedetection.io?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://lemonade.changedetection.io/start?src=github)
 
 ### Easily see what changed, examine by word, line, or individual character.
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://changedetection.io?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://lemonade.changedetection.io/start?src=github)
 
 
 ### Perform interactive browser steps
@@ -35,7 +33,7 @@ Fill in text boxes, click buttons and more, setup your changedetection scenario.
 
 Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
 
-[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Website change detection with interactive browser steps, login, cookies etc" />](https://changedetection.io?src=github)
+[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Website change detection with interactive browser steps, login, cookies etc" />](https://lemonade.changedetection.io/start?src=github)
 
 After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
 Requires Playwright to be enabled.
@@ -45,11 +43,9 @@ Requires Playwright to be enabled.
 
 - Products and services have a change in pricing
 - _Out of stock notification_ and _Back In stock notification_
-- Monitor and track PDF file changes, know when a PDF file has text changes.
 - Governmental department updates (changes are often only on their websites)
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
-- Discogs restock alerts and monitoring
 - Realestate listing changes
 - Know when your favourite whiskey is on sale, or other special deals are announced before anyone else
 - COVID related news from government websites
@@ -64,9 +60,6 @@ Requires Playwright to be enabled.
 - You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
 - Get notified when certain keywords appear in Twitter search results
 - Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
-- Get alerts when new job positions are open on Bamboo HR and other job platforms
-- Website defacement monitoring
-- Pokémon Card Restock Tracker / Pokémon TCG Tracker
 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 
@@ -75,7 +68,6 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 - Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
 - Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
 - Switch between fast non-JS and Chrome JS based "fetchers"
-- Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)
 - Easily specify how often a site should be checked
 - Execute JS before extracting text (Good for logging in, see examples in the UI!)
 - Override Request Headers, Specify `POST` or `GET` and other methods
@@ -104,8 +96,6 @@ $ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/d
 
 `:latest` tag is our latest stable release, `:dev` tag is our bleeding edge `master` branch.
 
-Alternative docker repository over at ghcr - [ghcr.io/dgtlmoon/changedetection.io](https://ghcr.io/dgtlmoon/changedetection.io)
-
 ### Windows
 
 See the install instructions at the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Microsoft-Windows
@@ -145,7 +135,7 @@ See the wiki for more information https://github.com/dgtlmoon/changedetection.io
 ## Filters
 
 XPath, JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. 
-(We support LXML `re:test`, `re:match` and `re:replace`.)
+(We support LXML `re:test`, `re:math` and `re:replace`.)
 
 ## Notifications
 
@@ -197,29 +187,11 @@ When you enable a `json:` or `jq:` filter, you can even automatically extract an
 <html>
 ...
 <script type="application/ld+json">
-
-{
-   "@context":"http://schema.org/",
-   "@type":"Product",
-   "offers":{
-      "@type":"Offer",
-      "availability":"http://schema.org/InStock",
-      "price":"3949.99",
-      "priceCurrency":"USD",
-      "url":"https://www.newegg.com/p/3D5-000D-001T1"
-   },
-   "description":"Cobratype King Cobra Hero Desktop Gaming PC",
-   "name":"Cobratype King Cobra Hero Desktop Gaming PC",
-   "sku":"3D5-000D-001T1",
-   "itemCondition":"NewCondition"
-}
+  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","price": 23.50 }
 </script>
 ```  
 
-`json:$..price` or `jq:..price` would give `3949.99`, or you can extract the whole structure (use a JSONpath test website to validate with)
-
-The application also supports notifying you that it can follow this information automatically
-
+`json:$.price` or `jq:.price` would give `23.50`, or you can extract the whole structure
 
 ## Proxy Configuration
 
@@ -229,16 +201,13 @@ See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configura
 
 Raspberry Pi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! See the wiki for [details](https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver)
 
-## API Support
-
-Supports managing the website watch list [via our API](https://changedetection.io/docs/api_v1/index.html)
 
 ## Support us
 
 Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 
 
-Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
+Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://lemonade.changedetection.io/start) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
 
 Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)
 
@@ -256,5 +225,5 @@ I offer commercial support, this software is depended on by network security, ae
 [test-shield]: https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master
 
 [license-shield]: https://img.shields.io/github/license/dgtlmoon/changedetection.io.svg?style=for-the-badge
-[release-link]: https://github.com/dgtlmoon/changedetection.io/releases
+[release-link]: https://github.com/dgtlmoon.com/changedetection.io/releases
 [docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io

changedetection.py

@@ -7,7 +7,7 @@
 
 from changedetectionio import changedetection
 import multiprocessing
-import sys
+import signal
 import os
 
 def sigchld_handler(_signo, _stack_frame):
@@ -35,9 +35,6 @@ if __name__ == '__main__':
     try:
         while True:
             time.sleep(1)
-            if not parse_process.is_alive():
-                # Process died/crashed for some reason, exit with error set
-                sys.exit(1)
 
     except KeyboardInterrupt:
         #parse_process.terminate() not needed, because this process will issue it to the sub-process anyway

changedetectionio/api/api_schema.py

@@ -1,117 +0,0 @@
-# Responsible for building the storage dict into a set of rules ("JSON Schema") acceptable via the API
-# Probably other ways to solve this when the backend switches to some ORM
-
-def build_time_between_check_json_schema():
-    # Setup time between check schema
-    schema_properties_time_between_check = {
-        "type": "object",
-        "additionalProperties": False,
-        "properties": {}
-    }
-    for p in ['weeks', 'days', 'hours', 'minutes', 'seconds']:
-        schema_properties_time_between_check['properties'][p] = {
-            "anyOf": [
-                {
-                    "type": "integer"
-                },
-                {
-                    "type": "null"
-                }
-            ]
-        }
-
-    return schema_properties_time_between_check
-
-def build_watch_json_schema(d):
-    # Base JSON schema
-    schema = {
-        'type': 'object',
-        'properties': {},
-    }
-
-    for k, v in d.items():
-        # @todo 'integer' is not covered here because its almost always for internal usage
-
-        if isinstance(v, type(None)):
-            schema['properties'][k] = {
-                "anyOf": [
-                    {"type": "null"},
-                ]
-            }
-        elif isinstance(v, list):
-            schema['properties'][k] = {
-                "anyOf": [
-                    {"type": "array",
-                     # Always is an array of strings, like text or regex or something
-                     "items": {
-                         "type": "string",
-                         "maxLength": 5000
-                     }
-                     },
-                ]
-            }
-        elif isinstance(v, bool):
-            schema['properties'][k] = {
-                "anyOf": [
-                    {"type": "boolean"},
-                ]
-            }
-        elif isinstance(v, str):
-            schema['properties'][k] = {
-                "anyOf": [
-                    {"type": "string",
-                     "maxLength": 5000},
-                ]
-            }
-
-    # Can also be a string (or None by default above)
-    for v in ['body',
-              'notification_body',
-              'notification_format',
-              'notification_title',
-              'proxy',
-              'tag',
-              'title',
-              'webdriver_js_execute_code'
-              ]:
-        schema['properties'][v]['anyOf'].append({'type': 'string', "maxLength": 5000})
-
-    # None or Boolean
-    schema['properties']['track_ldjson_price_data']['anyOf'].append({'type': 'boolean'})
-
-    schema['properties']['method'] = {"type": "string",
-                                      "enum": ["GET", "POST", "DELETE", "PUT"]
-                                      }
-
-    schema['properties']['fetch_backend']['anyOf'].append({"type": "string",
-                                                           "enum": ["html_requests", "html_webdriver"]
-                                                           })
-
-
-
-    # All headers must be key/value type dict
-    schema['properties']['headers'] = {
-        "type": "object",
-        "patternProperties": {
-            # Should always be a string:string type value
-            ".*": {"type": "string"},
-        }
-    }
-
-    from changedetectionio.notification import valid_notification_formats
-
-    schema['properties']['notification_format'] = {'type': 'string',
-                                                   'enum': list(valid_notification_formats.keys())
-                                                   }
-
-    # Stuff that shouldn't be available but is just state-storage
-    for v in ['previous_md5', 'last_error', 'has_ldjson_price_data', 'previous_md5_before_filters', 'uuid']:
-        del schema['properties'][v]
-
-    schema['properties']['webdriver_delay']['anyOf'].append({'type': 'integer'})
-
-    schema['properties']['time_between_check'] = build_time_between_check_json_schema()
-
-    # headers ?
-    return schema
-

changedetectionio/api/api_v1.py

@@ -1,24 +1,11 @@
-from flask_expects_json import expects_json
-from changedetectionio import queuedWatchMetaData
 from flask_restful import abort, Resource
 from flask import request, make_response
 import validators
 from . import auth
-import copy
 
-# See docs/README.md for rebuilding the docs/apidoc information
 
-from . import api_schema
 
-# Build a JSON Schema atleast partially based on our Watch model
-from changedetectionio.model.Watch import base_config as watch_base_config
-schema = api_schema.build_watch_json_schema(watch_base_config)
-
-schema_create_watch = copy.deepcopy(schema)
-schema_create_watch['required'] = ['url']
-
-schema_update_watch = copy.deepcopy(schema)
-schema_update_watch['additionalProperties'] = False
+# https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
 
 class Watch(Resource):
     def __init__(self, **kwargs):
@@ -28,100 +15,30 @@ class Watch(Resource):
 
     # Get information about a single watch, excluding the history list (can be large)
     # curl http://localhost:4000/api/v1/watch/<string:uuid>
-    # @todo - version2 - ?muted and ?paused should be able to be called together, return the watch struct not "OK"
     # ?recheck=true
     @auth.check_token
     def get(self, uuid):
-        """
-        @api {get} /api/v1/watch/:uuid Get a single watch data
-        @apiDescription Retrieve watch information and set muted/paused status
-        @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=unmuted"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?paused=unpaused"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-        @apiName Watch
-        @apiGroup Watch
-        @apiParam {uuid} uuid Watch unique ID.
-        @apiQuery {Boolean} [recheck] Recheck this watch `recheck=1`
-        @apiQuery {String} [paused] =`paused` or =`unpaused` , Sets the PAUSED state
-        @apiQuery {String} [muted] =`muted` or =`unmuted` , Sets the MUTE NOTIFICATIONS state
-        @apiSuccess (200) {String} OK When paused/muted/recheck operation OR full JSON object of the watch
-        @apiSuccess (200) {JSON} WatchJSON JSON Full JSON object of the watch
-        """
         from copy import deepcopy
         watch = deepcopy(self.datastore.data['watching'].get(uuid))
         if not watch:
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
 
         if request.args.get('recheck'):
-            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
-            return "OK", 200
-        if request.args.get('paused', '') == 'paused':
-            self.datastore.data['watching'].get(uuid).pause()
-            return "OK", 200
-        elif request.args.get('paused', '') == 'unpaused':
-            self.datastore.data['watching'].get(uuid).unpause()
-            return "OK", 200
-        if request.args.get('muted', '') == 'muted':
-            self.datastore.data['watching'].get(uuid).mute()
-            return "OK", 200
-        elif request.args.get('muted', '') == 'unmuted':
-            self.datastore.data['watching'].get(uuid).unmute()
+            self.update_q.put((1, uuid))
             return "OK", 200
 
         # Return without history, get that via another API call
-        # Properties are not returned as a JSON, so add the required props manually
         watch['history_n'] = watch.history_n
-        watch['last_changed'] = watch.last_changed
-
         return watch
 
     @auth.check_token
     def delete(self, uuid):
-        """
-        @api {delete} /api/v1/watch/:uuid Delete a watch and related history
-        @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-        @apiParam {uuid} uuid Watch unique ID.
-        @apiName Delete
-        @apiGroup Watch
-        @apiSuccess (200) {String} OK Was deleted
-        """
         if not self.datastore.data['watching'].get(uuid):
             abort(400, message='No watch exists with the UUID of {}'.format(uuid))
 
         self.datastore.delete(uuid)
         return 'OK', 204
 
-    @auth.check_token
-    @expects_json(schema_update_watch)
-    def put(self, uuid):
-        """
-        @api {put} /api/v1/watch/:uuid Update watch information
-        @apiExample {curl} Example usage:
-            Update (PUT)
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}'
-
-        @apiDescription Updates an existing watch using JSON, accepts the same structure as returned in <a href="#api-Watch-Watch">get single watch information</a>
-        @apiParam {uuid} uuid Watch unique ID.
-        @apiName Update a watch
-        @apiGroup Watch
-        @apiSuccess (200) {String} OK Was updated
-        @apiSuccess (500) {String} ERR Some other error
-        """
-        watch = self.datastore.data['watching'].get(uuid)
-        if not watch:
-            abort(404, message='No watch exists with the UUID of {}'.format(uuid))
-
-        if request.json.get('proxy'):
-            plist = self.datastore.proxy_list
-            if not request.json.get('proxy') in plist:
-                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
-
-        watch.update(request.json)
-
-        return "OK", 200
-
 
 class WatchHistory(Resource):
     def __init__(self, **kwargs):
@@ -131,21 +48,6 @@ class WatchHistory(Resource):
     # Get a list of available history for a watch by UUID
     # curl http://localhost:4000/api/v1/watch/<string:uuid>/history
     def get(self, uuid):
-        """
-        @api {get} /api/v1/watch/<string:uuid>/history Get a list of all historical snapshots available for a watch
-        @apiDescription Requires `uuid`, returns list
-        @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
-            {
-                "1676649279": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/cb7e9be8258368262246910e6a2a4c30.txt",
-                "1677092785": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/e20db368d6fc633e34f559ff67bb4044.txt",
-                "1677103794": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/02efdd37dacdae96554a8cc85dc9c945.txt"
-            }
-        @apiName Get list of available stored snapshots for watch
-        @apiGroup Watch History
-        @apiSuccess (200) {String} OK
-        @apiSuccess (404) {String} ERR Not found
-        """
         watch = self.datastore.data['watching'].get(uuid)
         if not watch:
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
@@ -157,18 +59,11 @@ class WatchSingleHistory(Resource):
         # datastore is a black box dependency
         self.datastore = kwargs['datastore']
 
+    # Read a given history snapshot and return its content
+    # <string:timestamp> or "latest"
+    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history/<int:timestamp>
     @auth.check_token
     def get(self, uuid, timestamp):
-        """
-        @api {get} /api/v1/watch/<string:uuid>/history/<int:timestamp> Get single snapshot from watch
-        @apiDescription Requires watch `uuid` and `timestamp`. `timestamp` of "`latest`" for latest available snapshot, or <a href="#api-Watch_History-Get_list_of_available_stored_snapshots_for_watch">use the list returned here</a>
-        @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
-        @apiName Get single snapshot content
-        @apiGroup Watch History
-        @apiSuccess (200) {String} OK
-        @apiSuccess (404) {String} ERR Not found
-        """
         watch = self.datastore.data['watching'].get(uuid)
         if not watch:
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
@@ -179,7 +74,8 @@ class WatchSingleHistory(Resource):
         if timestamp == 'latest':
             timestamp = list(watch.history.keys())[-1]
 
-        content = watch.get_history_snapshot(timestamp)
+        with open(watch.history[timestamp], 'r') as f:
+            content = f.read()
 
         response = make_response(content, 200)
         response.mimetype = "text/plain"
@@ -193,97 +89,36 @@ class CreateWatch(Resource):
         self.update_q = kwargs['update_q']
 
     @auth.check_token
-    @expects_json(schema_create_watch)
     def post(self):
-        """
-        @api {post} /api/v1/watch Create a single watch
-        @apiDescription Requires atleast `url` set, can accept the same structure as <a href="#api-Watch-Watch">get single watch information</a> to create.
-        @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
-        @apiName Create
-        @apiGroup Watch
-        @apiSuccess (200) {String} OK Was created
-        @apiSuccess (500) {String} ERR Some other error
-        """
-
+        # curl http://localhost:4000/api/v1/watch -H "Content-Type: application/json" -d '{"url": "https://my-nice.com", "tag": "one, two" }'
         json_data = request.get_json()
-        url = json_data['url'].strip()
+        tag = json_data['tag'].strip() if json_data.get('tag') else ''
 
         if not validators.url(json_data['url'].strip()):
             return "Invalid or unsupported URL", 400
 
-        if json_data.get('proxy'):
-            plist = self.datastore.proxy_list
-            if not json_data.get('proxy') in plist:
-                return "Invalid proxy choice, currently supported proxies are '{}'".format(', '.join(plist)), 400
+        extras = {'title': json_data['title'].strip()} if json_data.get('title') else {}
 
-        extras = copy.deepcopy(json_data)
-
-        # Because we renamed 'tag' to 'tags' but don't want to change the API (can do this in v2 of the API)
-        tags = None
-        if extras.get('tag'):
-            tags = extras.get('tag')
-            del extras['tag']
-
-        del extras['url']
-
-        new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags)
-        if new_uuid:
-            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
-            return {'uuid': new_uuid}, 201
-        else:
-            return "Invalid or unsupported URL", 400
+        new_uuid = self.datastore.add_watch(url=json_data['url'].strip(), tag=tag, extras=extras)
+        self.update_q.put((1, new_uuid))
+        return {'uuid': new_uuid}, 201
 
+    # Return concise list of available watches and some very basic info
+    # curl http://localhost:4000/api/v1/watch|python -mjson.tool
+    # ?recheck_all=1 to recheck all
     @auth.check_token
     def get(self):
-        """
-        @api {get} /api/v1/watch List watches
-        @apiDescription Return concise list of available watches and some very basic info
-        @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            {
-                "6a4b7d5c-fee4-4616-9f43-4ac97046b595": {
-                    "last_changed": 1677103794,
-                    "last_checked": 1677103794,
-                    "last_error": false,
-                    "title": "",
-                    "url": "http://www.quotationspage.com/random.php"
-                },
-                "e6f5fd5c-dbfe-468b-b8f3-f9d6ff5ad69b": {
-                    "last_changed": 0,
-                    "last_checked": 1676662819,
-                    "last_error": false,
-                    "title": "QuickLook",
-                    "url": "https://github.com/QL-Win/QuickLook/tags"
-                }
-            }
-
-        @apiParam {String} [recheck_all]       Optional Set to =1 to force recheck of all watches
-        @apiParam {String} [tag]               Optional name of tag to limit results
-        @apiName ListWatches
-        @apiGroup Watch Management
-        @apiSuccess (200) {String} OK JSON dict
-        """
         list = {}
-
-        tag_limit = request.args.get('tag', '').lower()
-
-
-        for uuid, watch in self.datastore.data['watching'].items():
-            # Watch tags by name (replace the other calls?)
-            tags = self.datastore.get_all_tags_for_watch(uuid=uuid)
-            if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):
-                continue
-
-            list[uuid] = {'url': watch['url'],
-                       'title': watch['title'],
-                       'last_checked': watch['last_checked'],
-                       'last_changed': watch.last_changed,
-                       'last_error': watch['last_error']}
+        for k, v in self.datastore.data['watching'].items():
+            list[k] = {'url': v['url'],
+                       'title': v['title'],
+                       'last_checked': v['last_checked'],
+                       'last_changed': v.last_changed,
+                       'last_error': v['last_error']}
 
         if request.args.get('recheck_all'):
             for uuid in self.datastore.data['watching'].keys():
-                self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
+                self.update_q.put((1, uuid))
             return {'status': "OK"}, 200
 
         return list, 200
@@ -296,22 +131,6 @@ class SystemInfo(Resource):
 
     @auth.check_token
     def get(self):
-        """
-        @api {get} /api/v1/systeminfo Return system info
-        @apiDescription Return some info about the current system state
-        @apiExample {curl} Example usage:
-            curl http://localhost:4000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            HTTP/1.0 200
-            {
-                'queue_size': 10 ,
-                'overdue_watches': ["watch-uuid-list"],
-                'uptime': 38344.55,
-                'watch_count': 800,
-                'version': "0.40.1"
-            }
-        @apiName Get Info
-        @apiGroup System Information
-        """
         import time
         overdue_watches = []
 
@@ -330,11 +149,10 @@ class SystemInfo(Resource):
             # Allow 5 minutes of grace time before we decide it's overdue
             if time_since_check - (5 * 60) > t:
                 overdue_watches.append(uuid)
-        from changedetectionio import __version__ as main_version
+
         return {
                    'queue_size': self.update_q.qsize(),
                    'overdue_watches': overdue_watches,
                    'uptime': round(time.time() - self.datastore.start_time, 2),
-                   'watch_count': len(self.datastore.data.get('watching', {})),
-                   'version': main_version
+                   'watch_count': len(self.datastore.data.get('watching', {}))
                }, 200

changedetectionio/blueprint/browser_steps/__init__.py

@@ -23,110 +23,63 @@
 
 from distutils.util import strtobool
 from flask import Blueprint, request, make_response
+from flask_login import login_required
 import os
 import logging
 from changedetectionio.store import ChangeDetectionStore
-from changedetectionio import login_optionally_required
 
-browsersteps_sessions = {}
-io_interface_context = None
+browsersteps_live_ui_o = {}
+browsersteps_playwright_browser_interface = None
+browsersteps_playwright_browser_interface_browser = None
+browsersteps_playwright_browser_interface_context = None
+browsersteps_playwright_browser_interface_end_time = None
+browsersteps_playwright_browser_interface_start_time = None
 
+def cleanup_playwright_session():
+
+    global browsersteps_live_ui_o
+    global browsersteps_playwright_browser_interface
+    global browsersteps_playwright_browser_interface_browser
+    global browsersteps_playwright_browser_interface_context
+    global browsersteps_playwright_browser_interface_end_time
+    global browsersteps_playwright_browser_interface_start_time
+
+    browsersteps_live_ui_o = {}
+    browsersteps_playwright_browser_interface = None
+    browsersteps_playwright_browser_interface_browser = None
+    browsersteps_playwright_browser_interface_end_time = None
+    browsersteps_playwright_browser_interface_start_time = None
+
+    print("Cleaning up old playwright session because time was up, calling .goodbye()")
+    try:
+        browsersteps_playwright_browser_interface_context.goodbye()
+    except Exception as e:
+        print ("Got exception in shutdown, probably OK")
+        print (str(e))
+
+    browsersteps_playwright_browser_interface_context = None
+
+    print ("Cleaning up old playwright session because time was up - done")
 
 def construct_blueprint(datastore: ChangeDetectionStore):
+
     browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
 
-    def start_browsersteps_session(watch_uuid):
-        from . import nonContext
-        from . import browser_steps
-        import time
-        global browsersteps_sessions
-        global io_interface_context
-
-
-        # We keep the playwright session open for many minutes
-        seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
-
-        browsersteps_start_session = {'start_time': time.time()}
-
-        # You can only have one of these running
-        # This should be very fine to leave running for the life of the application
-        # @idea - Make it global so the pool of watch fetchers can use it also
-        if not io_interface_context:
-            io_interface_context = nonContext.c_sync_playwright()
-            # Start the Playwright context, which is actually a nodejs sub-process and communicates over STDIN/STDOUT pipes
-            io_interface_context = io_interface_context.start()
-
-
-        # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
-        keepalive = "&timeout={}".format(((seconds_keepalive + 3) * 1000))
-        try:
-            browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(
-                os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
-        except Exception as e:
-            if 'ECONNREFUSED' in str(e):
-                return make_response('Unable to start the Playwright Browser session, is it running?', 401)
-            else:
-                return make_response(str(e), 401)
-
-        proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
-        proxy = None
-        if proxy_id:
-            proxy_url = datastore.proxy_list.get(proxy_id).get('url')
-            if proxy_url:
-
-                # Playwright needs separate username and password values
-                from urllib.parse import urlparse
-                parsed = urlparse(proxy_url)
-                proxy = {'server': proxy_url}
-
-                if parsed.username:
-                    proxy['username'] = parsed.username
-
-                if parsed.password:
-                    proxy['password'] = parsed.password
-
-                print("Browser Steps: UUID {} selected proxy {}".format(watch_uuid, proxy_url))
-
-        # Tell Playwright to connect to Chrome and setup a new session via our stepper interface
-        browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
-            playwright_browser=browsersteps_start_session['browser'],
-            proxy=proxy)
-
-        # For test
-        #browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time()))
-
-        return browsersteps_start_session
-
-
-    @login_optionally_required
-    @browser_steps_blueprint.route("/browsersteps_start_session", methods=['GET'])
-    def browsersteps_start_session():
-        # A new session was requested, return sessionID
-
-        import uuid
-        global browsersteps_sessions
-
-        browsersteps_session_id = str(uuid.uuid4())
-        watch_uuid = request.args.get('uuid')
-
-        if not watch_uuid:
-            return make_response('No Watch UUID specified', 500)
-
-        print("Starting connection with playwright")
-        logging.debug("browser_steps.py connecting")
-        browsersteps_sessions[browsersteps_session_id] = start_browsersteps_session(watch_uuid)
-        print("Starting connection with playwright - done")
-        return {'browsersteps_session_id': browsersteps_session_id}
-
-    # A request for an action was received
-    @login_optionally_required
-    @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])
+    @login_required
+    @browser_steps_blueprint.route("/browsersteps_update", methods=['GET', 'POST'])
     def browsersteps_ui_update():
         import base64
         import playwright._impl._api_types
-        global browsersteps_sessions
+        import time
+
         from changedetectionio.blueprint.browser_steps import browser_steps
 
+        global browsersteps_live_ui_o, browsersteps_playwright_browser_interface_end_time
+        global browsersteps_playwright_browser_interface_browser
+        global browsersteps_playwright_browser_interface
+        global browsersteps_playwright_browser_interface_start_time
+
+        step_n = None
         remaining =0
         uuid = request.args.get('uuid')
 
@@ -135,9 +88,13 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if not browsersteps_session_id:
             return make_response('No browsersteps_session_id specified', 500)
 
-        if not browsersteps_sessions.get(browsersteps_session_id):
-            return make_response('No session exists under that ID', 500)
-
+        # Because we don't "really" run in a context manager ( we make the playwright interface global/long-living )
+        # We need to manage the shutdown when the time is up
+        if browsersteps_playwright_browser_interface_end_time:
+            remaining = browsersteps_playwright_browser_interface_end_time-time.time()
+            if browsersteps_playwright_browser_interface_end_time and remaining <= 0:
+                cleanup_playwright_session()
+                return make_response('Browser session expired, please reload the Browser Steps interface', 401)
 
         # Actions - step/apply/etc, do the thing and return state
         if request.method == 'POST':
@@ -150,13 +107,18 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
             if step_operation == 'Goto site':
                 step_operation = 'goto_url'
-                step_optional_value = datastore.data['watching'][uuid].get('url')
-                step_selector = None
+                step_optional_value = None
+                step_selector = datastore.data['watching'][uuid].get('url')
 
             # @todo try.. accept.. nice errors not popups..
             try:
 
-                browsersteps_sessions[browsersteps_session_id]['browserstepper'].call_action(action_name=step_operation,
+                this_session = browsersteps_live_ui_o.get(browsersteps_session_id)
+                if not this_session:
+                    print("Browser exited")
+                    return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+                this_session.call_action(action_name=step_operation,
                                          selector=step_selector,
                                          optional_value=step_optional_value)
 
@@ -168,43 +130,99 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             # Get visual selector ready/update its data (also use the current filter info from the page?)
             # When the last 'apply' button was pressed
             # @todo this adds overhead because the xpath selection is happening twice
-            u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url
+            u = this_session.page.url
             if is_last_step and u:
-                (screenshot, xpath_data) = browsersteps_sessions[browsersteps_session_id]['browserstepper'].request_visualselector_data()
+                (screenshot, xpath_data) = this_session.request_visualselector_data()
                 datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
                 datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
 
-#        if not this_session.page:
-#            cleanup_playwright_session()
-#            return make_response('Browser session ran out of time :( Please reload this page.', 401)
+        # Setup interface
+        if request.method == 'GET':
 
-        # Screenshots and other info only needed on requesting a step (POST)
-        try:
-            state = browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()
-        except playwright._impl._api_types.Error as e:
-            return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
+            if not browsersteps_playwright_browser_interface:
+                print("Starting connection with playwright")
+                logging.debug("browser_steps.py connecting")
 
-        # Use send_file() which is way faster than read/write loop on bytes
-        import json
-        from tempfile import mkstemp
-        from flask import send_file
-        tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
+                global browsersteps_playwright_browser_interface_context
+                from . import nonContext
+                browsersteps_playwright_browser_interface_context = nonContext.c_sync_playwright()
+                browsersteps_playwright_browser_interface = browsersteps_playwright_browser_interface_context.start()
 
-        output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
-            base64.b64encode(state[0]).decode('ascii')),
-            'xpath_data': state[1],
-            'session_age_start': browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,
-            'browser_time_remaining': round(remaining)
-        })
+                time.sleep(1)
+                # At 20 minutes, some other variable is closing it
+                # @todo find out what it is and set it
+                seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
 
-        with os.fdopen(tmp_fd, 'w') as f:
-            f.write(output)
+                # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
+                keepalive = "&timeout={}".format(((seconds_keepalive+3) * 1000))
+                try:
+                    browsersteps_playwright_browser_interface_browser = browsersteps_playwright_browser_interface.chromium.connect_over_cdp(
+                        os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
+                except Exception as e:
+                    if 'ECONNREFUSED' in str(e):
+                        return make_response('Unable to start the Playwright session properly, is it running?', 401)
 
-        response = make_response(send_file(path_or_file=tmp_file,
-                                           mimetype='application/json; charset=UTF-8',
-                                           etag=True))
-        # No longer needed
-        os.unlink(tmp_file)
+                browsersteps_playwright_browser_interface_end_time = time.time() + (seconds_keepalive-3)
+                print("Starting connection with playwright - done")
+
+            if not browsersteps_live_ui_o.get(browsersteps_session_id):
+                # Boot up a new session
+                proxy_id = datastore.get_preferred_proxy_for_watch(uuid=uuid)
+                proxy = None
+                if proxy_id:
+                    proxy_url = datastore.proxy_list.get(proxy_id).get('url')
+                    if proxy_url:
+                        proxy = {'server': proxy_url}
+                        print("Browser Steps: UUID {} Using proxy {}".format(uuid, proxy_url))
+
+                # Begin the new "Playwright Context" that re-uses the playwright interface
+                # Each session is a "Playwright Context" as a list, that uses the playwright interface
+                browsersteps_live_ui_o[browsersteps_session_id] = browser_steps.browsersteps_live_ui(
+                    playwright_browser=browsersteps_playwright_browser_interface_browser,
+                    proxy=proxy)
+                this_session = browsersteps_live_ui_o[browsersteps_session_id]
+
+        if not this_session.page:
+            cleanup_playwright_session()
+            return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+        response = None
+
+        if request.method == 'POST':
+            # Screenshots and other info only needed on requesting a step (POST)
+            try:
+                state = this_session.get_current_state()
+            except playwright._impl._api_types.Error as e:
+                return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
+
+            # Use send_file() which is way faster than read/write loop on bytes
+            import json
+            from tempfile import mkstemp
+            from flask import send_file
+            tmp_fd, tmp_file = mkstemp(text=True, suffix=".json", prefix="changedetectionio-")
+
+            output = json.dumps({'screenshot': "data:image/jpeg;base64,{}".format(
+                base64.b64encode(state[0]).decode('ascii')),
+                'xpath_data': state[1],
+                'session_age_start': this_session.age_start,
+                'browser_time_remaining': round(remaining)
+            })
+
+            with os.fdopen(tmp_fd, 'w') as f:
+                f.write(output)
+
+            response = make_response(send_file(path_or_file=tmp_file,
+                                               mimetype='application/json; charset=UTF-8',
+                                               etag=True))
+            # No longer needed
+            os.unlink(tmp_file)
+
+        elif request.method == 'GET':
+            # Just enough to get the session rolling, it will call for goto-site via POST next
+            response = make_response({
+                'session_age_start': this_session.age_start,
+                'browser_time_remaining': round(remaining)
+            })
 
         return response
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -25,14 +25,12 @@ browser_step_ui_config = {'Choose one': '0 0',
                           'Execute JS': '0 1',
 #                          'Extract text and use as filter': '1 0',
                           'Goto site': '0 0',
-                          'Goto URL': '0 1',
                           'Press Enter': '0 0',
                           'Select by label': '1 1',
                           'Scroll down': '0 0',
                           'Uncheck checkbox': '1 0',
                           'Wait for seconds': '0 1',
                           'Wait for text': '0 1',
-                          'Wait for text in element': '1 1',
                           #                          'Press Page Down': '0 0',
                           #                          'Press Page Up': '0 0',
                           # weird bug, come back to it later
@@ -55,7 +53,7 @@ class steppable_browser_interface():
 
         print("> action calling", call_action_name)
         # https://playwright.dev/python/docs/selectors#xpath-selectors
-        if selector and selector.startswith('/') and not selector.startswith('//'):
+        if selector.startswith('/') and not selector.startswith('//'):
             selector = "xpath=" + selector
 
         action_handler = getattr(self, "action_" + call_action_name)
@@ -71,19 +69,21 @@ class steppable_browser_interface():
             optional_value = str(jinja2_env.from_string(optional_value).render())
 
         action_handler(selector, optional_value)
-        self.page.wait_for_timeout(1.5 * 1000)
+        self.page.wait_for_timeout(3 * 1000)
         print("Call action done in", time.time() - now)
 
-    def action_goto_url(self, selector=None, value=None):
+    def action_goto_url(self, url, optional_value):
         # self.page.set_viewport_size({"width": 1280, "height": 5000})
         now = time.time()
-        response = self.page.goto(value, timeout=0, wait_until='commit')
+        response = self.page.goto(url, timeout=0, wait_until='domcontentloaded')
+        print("Time to goto URL", time.time() - now)
 
         # Wait_until = commit
         # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
         # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
         # This seemed to solve nearly all 'TimeoutErrors'
-        print("Time to goto URL ", time.time() - now)
+        extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))
+        self.page.wait_for_timeout(extra_wait * 1000)
 
     def action_click_element_containing_text(self, selector=None, value=''):
         if not len(value.strip()):
@@ -105,8 +105,7 @@ class steppable_browser_interface():
         print("Clicking element")
         if not len(selector.strip()):
             return
-
-        self.page.click(selector=selector, timeout=30 * 1000, delay=randint(200, 500))
+        self.page.click(selector, timeout=10 * 1000, delay=randint(200, 500))
 
     def action_click_element_if_exists(self, selector, value):
         import playwright._impl._api_types as _api_types
@@ -133,18 +132,7 @@ class steppable_browser_interface():
         self.page.wait_for_timeout(1000)
 
     def action_wait_for_seconds(self, selector, value):
-        self.page.wait_for_timeout(float(value.strip()) * 1000)
-
-    def action_wait_for_text(self, selector, value):
-        import json
-        v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=90000)
-
-    def action_wait_for_text_in_element(self, selector, value):
-        import json
-        s = json.dumps(selector)
-        v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=90000)
+        self.page.wait_for_timeout(int(value) * 1000)
 
     # @todo - in the future make some popout interface to capture what needs to be set
     # https://playwright.dev/python/docs/api/class-keyboard

changedetectionio/blueprint/price_data_follower/__init__.py

@@ -3,28 +3,22 @@ from distutils.util import strtobool
 from flask import Blueprint, flash, redirect, url_for
 from flask_login import login_required
 from changedetectionio.store import ChangeDetectionStore
-from changedetectionio import queuedWatchMetaData
-from queue import PriorityQueue
 
-PRICE_DATA_TRACK_ACCEPT = 'accepted'
-PRICE_DATA_TRACK_REJECT = 'rejected'
-
-def construct_blueprint(datastore: ChangeDetectionStore, update_q: PriorityQueue):
+def construct_blueprint(datastore: ChangeDetectionStore):
 
     price_data_follower_blueprint = Blueprint('price_data_follower', __name__)
 
     @login_required
     @price_data_follower_blueprint.route("/<string:uuid>/accept", methods=['GET'])
     def accept(uuid):
-        datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_ACCEPT
-        update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
+        datastore.data['watching'][uuid]['track_ldjson_price_data'] = 'accepted'
         return redirect(url_for("form_watch_checknow", uuid=uuid))
 
 
     @login_required
     @price_data_follower_blueprint.route("/<string:uuid>/reject", methods=['GET'])
     def reject(uuid):
-        datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_REJECT
+        datastore.data['watching'][uuid]['track_ldjson_price_data'] = 'rejected'
         return redirect(url_for("index"))
 
 

changedetectionio/blueprint/tags/README.md

@@ -1,9 +0,0 @@
-# Groups tags
-
-## How it works
-
-Watch has a list() of tag UUID's, which relate to a config under application.settings.tags
-
-The 'tag' is actually a watch, because they basically will eventually share 90% of the same config.
-
-So a tag is like an abstract of a watch

changedetectionio/blueprint/tags/__init__.py

@@ -1,141 +0,0 @@
-from flask import Blueprint, request, make_response, render_template, flash, url_for, redirect
-from changedetectionio.store import ChangeDetectionStore
-from changedetectionio import login_optionally_required
-
-
-def construct_blueprint(datastore: ChangeDetectionStore):
-    tags_blueprint = Blueprint('tags', __name__, template_folder="templates")
-
-    @tags_blueprint.route("/list", methods=['GET'])
-    @login_optionally_required
-    def tags_overview_page():
-        from .form import SingleTag
-        add_form = SingleTag(request.form)
-        output = render_template("groups-overview.html",
-                                 form=add_form,
-                                 available_tags=datastore.data['settings']['application'].get('tags', {}),
-                                 )
-
-        return output
-
-    @tags_blueprint.route("/add", methods=['POST'])
-    @login_optionally_required
-    def form_tag_add():
-        from .form import SingleTag
-        add_form = SingleTag(request.form)
-
-        if not add_form.validate():
-            for widget, l in add_form.errors.items():
-                flash(','.join(l), 'error')
-            return redirect(url_for('tags.tags_overview_page'))
-
-        title = request.form.get('name').strip()
-
-        if datastore.tag_exists_by_name(title):
-            flash(f'The tag "{title}" already exists', "error")
-            return redirect(url_for('tags.tags_overview_page'))
-
-        datastore.add_tag(title)
-        flash("Tag added")
-
-
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/mute/<string:uuid>", methods=['GET'])
-    @login_optionally_required
-    def mute(uuid):
-        if datastore.data['settings']['application']['tags'].get(uuid):
-            datastore.data['settings']['application']['tags'][uuid]['notification_muted'] = not datastore.data['settings']['application']['tags'][uuid]['notification_muted']
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
-    @login_optionally_required
-    def delete(uuid):
-        removed = 0
-        # Delete the tag, and any tag reference
-        if datastore.data['settings']['application']['tags'].get(uuid):
-            del datastore.data['settings']['application']['tags'][uuid]
-
-        for watch_uuid, watch in datastore.data['watching'].items():
-            if watch.get('tags') and uuid in watch['tags']:
-                removed += 1
-                watch['tags'].remove(uuid)
-
-        flash(f"Tag deleted and removed from {removed} watches")
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/unlink/<string:uuid>", methods=['GET'])
-    @login_optionally_required
-    def unlink(uuid):
-        unlinked = 0
-        for watch_uuid, watch in datastore.data['watching'].items():
-            if watch.get('tags') and uuid in watch['tags']:
-                unlinked += 1
-                watch['tags'].remove(uuid)
-
-        flash(f"Tag unlinked removed from {unlinked} watches")
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/delete_all", methods=['GET'])
-    @login_optionally_required
-    def delete_all():
-        for watch_uuid, watch in datastore.data['watching'].items():
-            watch['tags'] = []
-        datastore.data['settings']['application']['tags'] = {}
-
-        flash(f"All tags deleted")
-        return redirect(url_for('tags.tags_overview_page'))
-
-    @tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
-    @login_optionally_required
-    def form_tag_edit(uuid):
-        from changedetectionio import forms
-
-        if uuid == 'first':
-            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
-
-        default = datastore.data['settings']['application']['tags'].get(uuid)
-
-        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                               data=default,
-                               )
-        form.datastore=datastore # needed?
-
-        output = render_template("edit-tag.html",
-                                 data=default,
-                                 form=form,
-                                 settings_application=datastore.data['settings']['application'],
-                                 )
-
-        return output
-
-
-    @tags_blueprint.route("/edit/<string:uuid>", methods=['POST'])
-    @login_optionally_required
-    def form_tag_edit_submit(uuid):
-        from changedetectionio import forms
-        if uuid == 'first':
-            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()
-
-        default = datastore.data['settings']['application']['tags'].get(uuid)
-
-        form = forms.watchForm(formdata=request.form if request.method == 'POST' else None,
-                               data=default,
-                               )
-        # @todo subclass form so validation works
-        #if not form.validate():
-#            for widget, l in form.errors.items():
-#                flash(','.join(l), 'error')
-#           return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))
-
-        datastore.data['settings']['application']['tags'][uuid].update(form.data)
-        datastore.needs_write_urgent = True
-        flash("Updated")
-
-        return redirect(url_for('tags.tags_overview_page'))
-
-
-    @tags_blueprint.route("/delete/<string:uuid>", methods=['GET'])
-    def form_tag_delete(uuid):
-        return redirect(url_for('tags.tags_overview_page'))
-    return tags_blueprint

changedetectionio/blueprint/tags/form.py

@@ -1,22 +0,0 @@
-from wtforms import (
-    BooleanField,
-    Form,
-    IntegerField,
-    RadioField,
-    SelectField,
-    StringField,
-    SubmitField,
-    TextAreaField,
-    validators,
-)
-
-
-
-class SingleTag(Form):
-
-    name = StringField('Tag name', [validators.InputRequired()], render_kw={"placeholder": "Name"})
-    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-
-
-
-

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -1,131 +0,0 @@
-{% extends 'base.html' %}
-{% block content %}
-{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
-{% from '_common_fields.jinja' import render_common_settings_form %}
-<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
-<script>
-
-/*{% if emailprefix %}*/
-    /*const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');*/
-/*{% endif %}*/
-
-
-</script>
-
-<script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
-<!--<script src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>-->
-<script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
-
-<div class="edit-form monospaced-textarea">
-
-    <div class="tabs collapsable">
-        <ul>
-            <li class="tab" id=""><a href="#general">General</a></li>
-            <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
-            <li class="tab"><a href="#notifications">Notifications</a></li>
-        </ul>
-    </div>
-
-    <div class="box-wrap inner">
-        <form class="pure-form pure-form-stacked"
-              action="{{ url_for('tags.form_tag_edit', uuid=data.uuid) }}" method="POST">
-             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
-
-            <div class="tab-pane-inner" id="general">
-                <fieldset>
-                    <div class="pure-control-group">
-                        {{ render_field(form.title, placeholder="https://...", required=true, class="m-d") }}
-                    </div>
-                </fieldset>
-            </div>
-
-            <div class="tab-pane-inner" id="filters-and-triggers">
-                    <div class="pure-control-group">
-                        {% set field = render_field(form.include_filters,
-                            rows=5,
-                            placeholder="#example
-xpath://body/div/span[contains(@class, 'example-class')]",
-                            class="m-d")
-                        %}
-                        {{ field }}
-                        {% if '/text()' in  field %}
-                          <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
-                        {% endif %}
-                        <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br>
-
-                    <ul>
-                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
-                        <li>JSON - Limit text to this JSON rule, using either <a href="https://pypi.org/project/jsonpath-ng/" target="new">JSONPath</a> or <a href="https://stedolan.github.io/jq/" target="new">jq</a> (if installed).
-                            <ul>
-                                <li>JSONPath: Prefix with <code>json:</code>, use <code>json:$</code> to force re-formatting if required,  <a href="https://jsonpath.com/" target="new">test your JSONPath here</a>.</li>
-                                {% if jq_support %}
-                                <li>jq: Prefix with <code>jq:</code> and <a href="https://jqplay.org/" target="new">test your jq here</a>. Using <a href="https://stedolan.github.io/jq/" target="new">jq</a> allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation <a href="https://stedolan.github.io/jq/manual/" target="new">here</a>.</li>
-                                {% else %}
-                                <li>jq support not installed</li>
-                                {% endif %}
-                            </ul>
-                        </li>
-                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash,
-                            <ul>
-                                <li>Example:  <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a
-                                href="http://xpather.com/" target="new">test your XPath here</a></li>
-                                <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li>
-                            </ul>
-                            </li>
-                    </ul>
-                    Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
-                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br>
-                </span>
-                    </div>
-                <fieldset class="pure-control-group">
-                    {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
-footer
-nav
-.stockticker") }}
-                    <span class="pure-form-message-inline">
-                        <ul>
-                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
-                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
-                        </ul>
-                      </span>
-                </fieldset>
-
-            </div>
-
-            <div class="tab-pane-inner" id="notifications">
-                <fieldset>
-                    <div  class="pure-control-group inline-radio">
-                      {{ render_checkbox_field(form.notification_muted) }}
-                    </div>
-                    {% if is_html_webdriver %}
-                    <div class="pure-control-group inline-radio">
-                      {{ render_checkbox_field(form.notification_screenshot) }}
-                        <span class="pure-form-message-inline">
-                            <strong>Use with caution!</strong> This will easily fill up your email storage quota or flood other storages.
-                        </span>
-                    </div>
-                    {% endif %}
-                    <div class="field-group" id="notification-field-group">
-                        {% if has_default_notification_urls %}
-                        <div class="inline-warning">
-                            <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!" >
-                            There are <a href="{{ url_for('settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
-                        </div>
-                        {% endif %}
-                        <a href="#notifications" id="notification-setting-reset-to-default" class="pure-button button-xsmall" style="right: 20px; top: 20px; position: absolute; background-color: #5f42dd; border-radius: 4px; font-size: 70%; color: #fff">Use system defaults</a>
-
-                        {{ render_common_settings_form(form, emailprefix, settings_application) }}
-                    </div>
-                </fieldset>
-            </div>
-
-            <div id="actions">
-                <div class="pure-control-group">
-                    {{ render_button(form.save_button) }}
-                </div>
-            </div>
-        </form>
-    </div>
-</div>
-
-{% endblock %}

changedetectionio/blueprint/tags/templates/groups-overview.html

@@ -1,60 +0,0 @@
-{% extends 'base.html' %}
-{% block content %}
-{% from '_helpers.jinja' import render_simple_field, render_field %}
-<script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
-
-<div class="box">
-    <form class="pure-form" action="{{ url_for('tags.form_tag_add') }}" method="POST" id="new-watch-form">
-        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
-        <fieldset>
-            <legend>Add a new organisational tag</legend>
-            <div id="watch-add-wrapper-zone">
-                <div>
-                    {{ render_simple_field(form.name, placeholder="watch label / tag") }}
-                </div>
-                <div>
-                    {{ render_simple_field(form.save_button, title="Save" ) }}
-                </div>
-            </div>
-            <br>
-            <div style="color: #fff;">Groups allows you to manage filters and notifications for multiple watches under a single organisational tag.</div>
-        </fieldset>
-    </form>
-    <!-- @todo maybe some overview matrix, 'tick' with which has notification, filter rules etc -->
-    <div id="watch-table-wrapper">
-
-        <table class="pure-table pure-table-striped watch-table group-overview-table">
-            <thead>
-            <tr>
-                <th></th>
-                <th>Tag / Label name</th>
-                <th></th>
-            </tr>
-            </thead>
-            <tbody>
-            <!--
-            @Todo - connect Last checked, Last Changed, Number of Watches etc
-            --->
-            {% if not available_tags|length %}
-            <tr>
-                <td colspan="3">No website organisational tags/groups configured</td>
-            </tr>
-            {% endif %}
-            {% for uuid, tag in available_tags.items()  %}
-            <tr id="{{ uuid }}" class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}">
-                <td class="watch-controls">
-                    <a class="link-mute state-{{'on' if tag.notification_muted else 'off'}}" href="{{url_for('tags.mute', uuid=tag.uuid)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
-                </td>
-                <td class="title-col inline">{{tag.title}}</td>
-                <td>
-                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.form_tag_edit', uuid=uuid) }}">Edit</a>&nbsp;
-                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.delete', uuid=uuid) }}" title="Deletes and removes tag">Delete</a>
-                    <a class="pure-button pure-button-primary" href="{{ url_for('tags.unlink', uuid=uuid) }}" title="Keep the tag but unlink any watches">Unlink</a>
-                </td>
-            </tr>
-            {% endfor %}
-            </tbody>
-        </table>
-    </div>
-</div>
-{% endblock %}

changedetectionio/changedetection.py

@@ -3,14 +3,11 @@
 # Launch as a eventlet.wsgi server instance.
 
 from distutils.util import strtobool
-from json.decoder import JSONDecodeError
-
 import eventlet
 import eventlet.wsgi
 import getopt
 import os
 import signal
-import socket
 import sys
 
 from . import store, changedetection_app, content_fetcher
@@ -31,13 +28,11 @@ def sigterm_handler(_signo, _stack_frame):
 def main():
     global datastore
     global app
-
-    datastore_path = None
-    do_cleanup = False
-    host = ''
-    ipv6_enabled = False
-    port = os.environ.get('PORT') or 5000
     ssl_mode = False
+    host = ''
+    port = os.environ.get('PORT') or 5000
+    do_cleanup = False
+    datastore_path = None
 
     # On Windows, create and use a default path.
     if os.name == 'nt':
@@ -48,7 +43,7 @@ def main():
         datastore_path = os.path.join(os.getcwd(), "../datastore")
 
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:", "port")
+        opts, args = getopt.getopt(sys.argv[1:], "Ccsd:h:p:", "port")
     except getopt.GetoptError:
         print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
         sys.exit(2)
@@ -68,10 +63,6 @@ def main():
         if opt == '-d':
             datastore_path = arg
 
-        if opt == '-6':
-            print ("Enabling IPv6 listen support")
-            ipv6_enabled = True
-
         # Cleanup (remove text files that arent in the index)
         if opt == '-c':
             do_cleanup = True
@@ -92,14 +83,8 @@ def main():
                 "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
             sys.exit(2)
 
-    try:
-        datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
-    except JSONDecodeError as e:
-        # Dont' start if the JSON DB looks corrupt
-        print ("ERROR: JSON DB or Proxy List JSON at '{}' appears to be corrupt, aborting".format(app_config['datastore_path']))
-        print(str(e))
-        return
 
+    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
     app = changedetection_app(app_config, datastore)
 
     signal.signal(signal.SIGTERM, sigterm_handler)
@@ -139,15 +124,13 @@ def main():
         from werkzeug.middleware.proxy_fix import ProxyFix
         app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
 
-    s_type = socket.AF_INET6 if ipv6_enabled else socket.AF_INET
-
     if ssl_mode:
         # @todo finalise SSL config, but this should get you in the right direction if you need it.
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), s_type),
+        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port)),
                                                certfile='cert.pem',
                                                keyfile='privkey.pem',
                                                server_side=True), app)
 
     else:
-        eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app)
+        eventlet.wsgi.server(eventlet.listen((host, int(port))), app)
 

changedetectionio/content_fetcher.py

@@ -1,4 +1,3 @@
-import hashlib
 from abc import abstractmethod
 import chardet
 import json
@@ -10,7 +9,6 @@ import time
 
 visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary'
 
-
 class Non200ErrorCodeReceived(Exception):
     def __init__(self, status_code, url, screenshot=None, xpath_data=None, page_html=None):
         # Set this so we can use it in other parts of the app
@@ -26,11 +24,6 @@ class Non200ErrorCodeReceived(Exception):
         return
 
 
-class checksumFromPreviousCheckWasTheSame(Exception):
-    def __init__(self):
-        return
-
-
 class JSActionExceptions(Exception):
     def __init__(self, status_code, url, screenshot, message=''):
         self.status_code = status_code
@@ -39,7 +32,6 @@ class JSActionExceptions(Exception):
         self.message = message
         return
 
-
 class BrowserStepsStepTimout(Exception):
     def __init__(self, step_n):
         self.step_n = step_n
@@ -47,7 +39,7 @@ class BrowserStepsStepTimout(Exception):
 
 
 class PageUnloadable(Exception):
-    def __init__(self, status_code, url, message, screenshot=False):
+    def __init__(self, status_code, url, screenshot=False, message=False):
         # Set this so we can use it in other parts of the app
         self.status_code = status_code
         self.url = url
@@ -55,7 +47,6 @@ class PageUnloadable(Exception):
         self.message = message
         return
 
-
 class EmptyReply(Exception):
     def __init__(self, status_code, url, screenshot=None):
         # Set this so we can use it in other parts of the app
@@ -64,7 +55,6 @@ class EmptyReply(Exception):
         self.screenshot = screenshot
         return
 
-
 class ScreenshotUnavailable(Exception):
     def __init__(self, status_code, url, page_html=None):
         # Set this so we can use it in other parts of the app
@@ -75,7 +65,6 @@ class ScreenshotUnavailable(Exception):
             self.page_text = html_to_text(page_html)
         return
 
-
 class ReplyWithContentButNoText(Exception):
     def __init__(self, status_code, url, screenshot=None):
         # Set this so we can use it in other parts of the app
@@ -84,20 +73,19 @@ class ReplyWithContentButNoText(Exception):
         self.screenshot = screenshot
         return
 
-
 class Fetcher():
+    error = None
+    status_code = None
+    content = None
+    headers = None
     browser_steps = None
     browser_steps_screenshot_path = None
-    content = None
-    error = None
+
     fetcher_description = "No description"
-    headers = {}
-    status_code = None
     webdriver_js_execute_code = None
-    xpath_data = None
     xpath_element_js = ""
-    instock_data = None
-    instock_data_js = ""
+
+    xpath_data = None
 
     # Will be needed in the future by the VisualSelector, always get this where possible.
     screenshot = False
@@ -111,7 +99,7 @@ class Fetcher():
         from pkg_resources import resource_string
         # The code that scrapes elements and makes a list of elements/size/position to click on in the VisualSelector
         self.xpath_element_js = resource_string(__name__, "res/xpath_element_scraper.js").decode('utf-8')
-        self.instock_data_js = resource_string(__name__, "res/stock-not-in-stock.js").decode('utf-8')
+
 
     @abstractmethod
     def get_error(self):
@@ -125,8 +113,7 @@ class Fetcher():
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
+            current_include_filters=None):
         # Should set self.error, self.status_code and self.content
         pass
 
@@ -147,13 +134,6 @@ class Fetcher():
     def is_ready(self):
         return True
 
-    def get_all_headers(self):
-        """
-        Get all headers but ensure all keys are lowercase
-        :return:
-        """
-        return {k.lower(): v for k, v in self.headers.items()}
-
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._api_types import TimeoutError
@@ -166,15 +146,13 @@ class Fetcher():
             interface = steppable_browser_interface()
             interface.page = self.page
 
-            valid_steps = filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-                self.browser_steps)
+            valid_steps = filter(lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'), self.browser_steps)
 
             for step in valid_steps:
                 step_n += 1
                 print(">> Iterating check - browser Step n {} - {}...".format(step_n, step['operation']))
-                self.screenshot_step("before-" + str(step_n))
-                self.save_step_html("before-" + str(step_n))
+                self.screenshot_step("before-"+str(step_n))
+                self.save_step_html("before-"+str(step_n))
                 try:
                     optional_value = step['optional_value']
                     selector = step['selector']
@@ -189,11 +167,12 @@ class Fetcher():
                                                       optional_value=optional_value)
                     self.screenshot_step(step_n)
                     self.save_step_html(step_n)
-                except TimeoutError as e:
-                    print(str(e))
+                except TimeoutError:
                     # Stop processing here
                     raise BrowserStepsStepTimout(step_n=step_n)
 
+
+
     # It's always good to reset these
     def delete_browser_steps_screenshots(self):
         import glob
@@ -203,7 +182,6 @@ class Fetcher():
             for f in files:
                 os.unlink(f)
 
-
 #   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
 #   the current one would return javascript output (as we use JS to generate the diff)
 #
@@ -221,7 +199,6 @@ def available_fetchers():
 
     return p
 
-
 class base_html_playwright(Fetcher):
     fetcher_description = "Playwright {}/Javascript".format(
         os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').capitalize()
@@ -261,15 +238,10 @@ class base_html_playwright(Fetcher):
         if proxy_override:
             self.proxy = {'server': proxy_override}
 
-        if self.proxy:
-            # Playwright needs separate username and password values
-            from urllib.parse import urlparse
-            parsed = urlparse(self.proxy.get('server'))
-            if parsed.username:
-                self.proxy['username'] = parsed.username
-                self.proxy['password'] = parsed.password
-
     def screenshot_step(self, step_n=''):
+
+        # There's a bug where we need to do it twice or it doesnt take the whole page, dont know why.
+        self.page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024})
         screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=85)
 
         if self.browser_steps_screenshot_path is not None:
@@ -285,119 +257,6 @@ class base_html_playwright(Fetcher):
         with open(destination, 'w') as f:
             f.write(content)
 
-    def run_fetch_browserless_puppeteer(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
-
-        from pkg_resources import resource_string
-
-        extra_wait_ms = (int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay) * 1000
-
-        self.xpath_element_js = self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors)
-        code = resource_string(__name__, "res/puppeteer_fetch.js").decode('utf-8')
-        # In the future inject this is a proper JS package
-        code = code.replace('%xpath_scrape_code%', self.xpath_element_js)
-        code = code.replace('%instock_scrape_code%', self.instock_data_js)
-
-        from requests.exceptions import ConnectTimeout, ReadTimeout
-        wait_browserless_seconds = 240
-
-        browserless_function_url = os.getenv('BROWSERLESS_FUNCTION_URL')
-        from urllib.parse import urlparse
-        if not browserless_function_url:
-            # Convert/try to guess from PLAYWRIGHT_DRIVER_URL
-            o = urlparse(os.getenv('PLAYWRIGHT_DRIVER_URL'))
-            browserless_function_url = o._replace(scheme="http")._replace(path="function").geturl()
-
-
-        # Append proxy connect string
-        if self.proxy:
-            import urllib.parse
-            # Remove username/password if it exists in the URL or you will receive "ERR_NO_SUPPORTED_PROXIES" error
-            # Actual authentication handled by Puppeteer/node
-            o = urlparse(self.proxy.get('server'))
-            proxy_url = urllib.parse.quote(o._replace(netloc="{}:{}".format(o.hostname, o.port)).geturl())
-            browserless_function_url = f"{browserless_function_url}&--proxy-server={proxy_url}&dumpio=true"
-
-
-        try:
-            amp = '&' if '?' in browserless_function_url else '?'
-            response = requests.request(
-                method="POST",
-                json={
-                    "code": code,
-                    "context": {
-                        # Very primitive disk cache - USE WITH EXTREME CAUTION
-                        # Run browserless container  with -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]"
-                        'disk_cache_dir': os.getenv("PUPPETEER_DISK_CACHE", False), # or path to disk cache ending in /, ie /tmp/cache/
-                        'execute_js': self.webdriver_js_execute_code,
-                        'extra_wait_ms': extra_wait_ms,
-                        'include_filters': current_include_filters,
-                        'req_headers': request_headers,
-                        'screenshot_quality': int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)),
-                        'url': url,
-                        'user_agent': request_headers.get('User-Agent', 'Mozilla/5.0'),
-                        'proxy_username': self.proxy.get('username','') if self.proxy else False,
-                        'proxy_password': self.proxy.get('password', '') if self.proxy else False,
-                        'no_cache_list': [
-                            'twitter',
-                            '.pdf'
-                        ],
-                        # Could use https://github.com/easylist/easylist here, or install a plugin
-                        'block_url_list': [
-                            'adnxs.com',
-                            'analytics.twitter.com',
-                            'doubleclick.net',
-                            'google-analytics.com',
-                            'googletagmanager',
-                            'trustpilot.com'
-                        ]
-                    }
-                },
-                # @todo /function needs adding ws:// to http:// rebuild this
-                url=browserless_function_url+f"{amp}--disable-features=AudioServiceOutOfProcess&dumpio=true&--disable-remote-fonts",
-                timeout=wait_browserless_seconds)
-
-        except ReadTimeout:
-            raise PageUnloadable(url=url, status_code=None, message=f"No response from browserless in {wait_browserless_seconds}s")
-        except ConnectTimeout:
-            raise PageUnloadable(url=url, status_code=None, message=f"Timed out connecting to browserless, retrying..")
-        else:
-            # 200 Here means that the communication to browserless worked only, not the page state
-            if response.status_code == 200:
-                import base64
-
-                x = response.json()
-                if not x.get('screenshot'):
-                    # https://github.com/puppeteer/puppeteer/blob/v1.0.0/docs/troubleshooting.md#tips
-                    # https://github.com/puppeteer/puppeteer/issues/1834
-                    # https://github.com/puppeteer/puppeteer/issues/1834#issuecomment-381047051
-                    # Check your memory is shared and big enough
-                    raise ScreenshotUnavailable(url=url, status_code=None)
-
-                if not x.get('content', '').strip():
-                    raise EmptyReply(url=url, status_code=None)
-
-                if x.get('status_code', 200) != 200 and not ignore_status_codes:
-                    raise Non200ErrorCodeReceived(url=url, status_code=x.get('status_code', 200), page_html=x['content'])
-
-                self.content = x.get('content')
-                self.headers = x.get('headers')
-                self.instock_data = x.get('instock_data')
-                self.screenshot = base64.b64decode(x.get('screenshot'))
-                self.status_code = x.get('status_code')
-                self.xpath_data = x.get('xpath_data')
-
-            else:
-                # Some other error from browserless
-                raise PageUnloadable(url=url, status_code=None, message=response.content.decode('utf-8'))
-
     def run(self,
             url,
             timeout,
@@ -405,26 +264,7 @@ class base_html_playwright(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
-
-        # For now, USE_EXPERIMENTAL_PUPPETEER_FETCH is not supported by watches with BrowserSteps (for now!)
-        has_browser_steps = self.browser_steps and list(filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-                self.browser_steps))
-
-        if not has_browser_steps:
-            if os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
-                # Temporary backup solution until we rewrite the playwright code
-                return self.run_fetch_browserless_puppeteer(
-                    url,
-                    timeout,
-                    request_headers,
-                    request_body,
-                    request_method,
-                    ignore_status_codes,
-                    current_include_filters,
-                    is_binary)
+            current_include_filters=None):
 
         from playwright.sync_api import sync_playwright
         import playwright._impl._api_types
@@ -442,12 +282,10 @@ class base_html_playwright(Fetcher):
             # Set user agent to prevent Cloudflare from blocking the browser
             # Use the default one configured in the App.py model that's passed from fetch_site_status.py
             context = browser.new_context(
-                user_agent=request_headers.get('User-Agent', 'Mozilla/5.0'),
+                user_agent=request_headers['User-Agent'] if request_headers.get('User-Agent') else 'Mozilla/5.0',
                 proxy=self.proxy,
                 # This is needed to enable JavaScript execution on GitHub and others
                 bypass_csp=True,
-                # Should be `allow` or `block` - sites like YouTube can transmit large amounts of data via Service Workers
-                service_workers=os.getenv('PLAYWRIGHT_SERVICE_WORKERS', 'allow'),
                 # Should never be needed
                 accept_downloads=False
             )
@@ -456,34 +294,24 @@ class base_html_playwright(Fetcher):
             if len(request_headers):
                 context.set_extra_http_headers(request_headers)
 
+            try:
                 self.page.set_default_navigation_timeout(90000)
                 self.page.set_default_timeout(90000)
 
                 # Listen for all console events and handle errors
                 self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
 
-            # Goto page
-            try:
+                # Bug - never set viewport size BEFORE page.goto
+
+
+                # Waits for the next navigation. Using Python context manager
+                # prevents a race condition between clicking and waiting for a navigation.
+                with self.page.expect_navigation():
+                    response = self.page.goto(url, wait_until='load')
                 # Wait_until = commit
                 # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
                 # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
                 # This seemed to solve nearly all 'TimeoutErrors'
-                response = self.page.goto(url, wait_until='commit')
-            except playwright._impl._api_types.Error as e:
-                # Retry once - https://github.com/browserless/chrome/issues/2485
-                # Sometimes errors related to invalid cert's and other can be random
-                print("Content Fetcher > retrying request got error - ", str(e))
-                time.sleep(1)
-                response = self.page.goto(url, wait_until='commit')
-
-            except Exception as e:
-                print("Content Fetcher > Other exception when page.goto", str(e))
-                context.close()
-                browser.close()
-                raise PageUnloadable(url=url, status_code=None, message=str(e))
-
-            # Execute any browser steps
-            try:
                 extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
                 self.page.wait_for_timeout(extra_wait * 1000)
 
@@ -496,32 +324,43 @@ class base_html_playwright(Fetcher):
                 # This can be ok, we will try to grab what we could retrieve
                 pass
             except Exception as e:
-                print("Content Fetcher > Other exception when executing custom JS code", str(e))
+                print ("other exception when page.goto")
+                print (str(e))
                 context.close()
                 browser.close()
-                raise PageUnloadable(url=url, status_code=None, message=str(e))
+                raise PageUnloadable(url=url, status_code=None)
+
 
             if response is None:
                 context.close()
                 browser.close()
-                print("Content Fetcher > Response object was none")
+                print ("response object was none")
                 raise EmptyReply(url=url, status_code=None)
 
+            # Bug 2(?) Set the viewport size AFTER loading the page
+            self.page.set_viewport_size({"width": 1280, "height": 1024})
+
             # Run Browser Steps here
             self.iterate_browser_steps()
 
             extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
             time.sleep(extra_wait)
 
+
             self.content = self.page.content()
             self.status_code = response.status
+
             if len(self.page.content().strip()) == 0:
                 context.close()
                 browser.close()
-                print("Content Fetcher > Content was empty")
-                raise EmptyReply(url=url, status_code=response.status)
+                print ("Content was empty")
+                raise EmptyReply(url=url, status_code=None)
+
+            # Bug 2(?) Set the viewport size AFTER loading the page
+            self.page.set_viewport_size({"width": 1280, "height": 1024})
 
             self.status_code = response.status
+            self.content = self.page.content()
             self.headers = response.all_headers()
 
             # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
@@ -530,9 +369,7 @@ class base_html_playwright(Fetcher):
             else:
                 self.page.evaluate("var include_filters=''")
 
-            self.xpath_data = self.page.evaluate(
-                "async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
-            self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
+            self.xpath_data = self.page.evaluate("async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
 
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
@@ -542,9 +379,10 @@ class base_html_playwright(Fetcher):
             # which will significantly increase the IO size between the server and client, it's recommended to use the lowest
             # acceptable screenshot quality here
             try:
+                # Quality set to 1 because it's not used, just used as a work-around for a bug, no need to change this.
+                self.page.screenshot(type='jpeg', clip={'x': 1.0, 'y': 1.0, 'width': 1280, 'height': 1024}, quality=1)
                 # The actual screenshot
-                self.screenshot = self.page.screenshot(type='jpeg', full_page=True,
-                                                       quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
+                self.screenshot = self.page.screenshot(type='jpeg', full_page=True, quality=int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)))
             except Exception as e:
                 context.close()
                 browser.close()
@@ -553,7 +391,6 @@ class base_html_playwright(Fetcher):
             context.close()
             browser.close()
 
-
 class base_html_webdriver(Fetcher):
     if os.getenv("WEBDRIVER_URL"):
         fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))
@@ -603,8 +440,7 @@ class base_html_webdriver(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
+            current_include_filters=None):
 
         from selenium import webdriver
         from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
@@ -662,7 +498,7 @@ class base_html_webdriver(Fetcher):
             try:
                 self.driver.quit()
             except Exception as e:
-                print("Content Fetcher > Exception in chrome shutdown/quit" + str(e))
+                print("Exception in chrome shutdown/quit" + str(e))
 
 
 # "html_requests" is listed as the default fetcher in store.py!
@@ -679,8 +515,7 @@ class html_requests(Fetcher):
             request_body,
             request_method,
             ignore_status_codes=False,
-            current_include_filters=None,
-            is_binary=False):
+            current_include_filters=None):
 
         # Make requests use a more modern looking user-agent
         if not 'User-Agent' in request_headers:
@@ -710,12 +545,10 @@ class html_requests(Fetcher):
         # For example - some sites don't tell us it's utf-8, but return utf-8 content
         # This seems to not occur when using webdriver/selenium, it seems to detect the text encoding more reliably.
         # https://github.com/psf/requests/issues/1604 good info about requests encoding detection
-        if not is_binary:
-            # Don't run this for PDF (and requests identified as binary) takes a _long_ time
-            if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
-                encoding = chardet.detect(r.content)['encoding']
-                if encoding:
-                    r.encoding = encoding
+        if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
+            encoding = chardet.detect(r.content)['encoding']
+            if encoding:
+                r.encoding = encoding
 
         if not r.content or not len(r.content):
             raise EmptyReply(url=url, status_code=r.status_code)
@@ -727,14 +560,8 @@ class html_requests(Fetcher):
             raise Non200ErrorCodeReceived(url=url, status_code=r.status_code, page_html=r.text)
 
         self.status_code = r.status_code
-        if is_binary:
-            # Binary files just return their checksum until we add something smarter
-            self.content = hashlib.md5(r.content).hexdigest()
-        else:
-            self.content = r.text
-
+        self.content = r.text
         self.headers = r.headers
-        self.raw_content = r.content
 
 
 # Decide which is the 'real' HTML webdriver, this is more a system wide config

changedetectionio/dev-docker/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.8-slim
+
+# https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+RUN [ ! -d "/datastore" ] && mkdir /datastore
+
+COPY sleep.py /
+CMD [ "python", "/sleep.py" ]
+
+
+

changedetectionio/dev-docker/sleep.py

@@ -0,0 +1,7 @@
+import time
+
+print ("Sleep loop, you should run your script from the console")
+
+while True: 
+    # Wait for 5 seconds
+    time.sleep(2)

changedetectionio/diff.py

@@ -10,7 +10,7 @@ def same_slicer(l, a, b):
         return l[a:b]
 
 # like .compare but a little different output
-def customSequenceMatcher(before, after, include_equal=False, include_removed=True, include_added=True, include_replaced=True, include_change_type_prefix=True):
+def customSequenceMatcher(before, after, include_equal=False):
     cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
 
     # @todo Line-by-line mode instead of buncghed, including `after` that is not in `before` (maybe unset?)
@@ -18,39 +18,34 @@ def customSequenceMatcher(before, after, include_equal=False, include_removed=Tr
         if include_equal and tag == 'equal':
             g = before[alo:ahi]
             yield g
-        elif include_removed and tag == 'delete':
-            row_prefix = "(removed) " if include_change_type_prefix else ''
-            g = [ row_prefix + i for i in same_slicer(before, alo, ahi)]
+        elif tag == 'delete':
+            g = ["(removed) " + i for i in same_slicer(before, alo, ahi)]
             yield g
-        elif include_replaced and tag == 'replace':
-            row_prefix = "(changed) " if include_change_type_prefix else ''
-            g = [row_prefix + i for i in same_slicer(before, alo, ahi)]
-            row_prefix = "(into) " if include_change_type_prefix else ''
-            g += [row_prefix + i for i in same_slicer(after, blo, bhi)]
+        elif tag == 'replace':
+            g = ["(changed) " + i for i in same_slicer(before, alo, ahi)]
+            g += ["(into   ) " + i for i in same_slicer(after, blo, bhi)]
             yield g
-        elif include_added and tag == 'insert':
-            row_prefix = "(added) " if include_change_type_prefix else ''
-            g = [row_prefix + i for i in same_slicer(after, blo, bhi)]
+        elif tag == 'insert':
+            g = ["(added  ) " + i for i in same_slicer(after, blo, bhi)]
             yield g
 
 # only_differences - only return info about the differences, no context
-# line_feed_sep could be "<br>" or "<li>" or "\n" etc
-def render_diff(previous_version_file_contents, newest_version_file_contents, include_equal=False, include_removed=True, include_added=True, include_replaced=True, line_feed_sep="\n", include_change_type_prefix=True):
+# line_feed_sep could be "<br/>" or "<li>" or "\n" etc
+def render_diff(previous_file, newest_file, include_equal=False, line_feed_sep="\n"):
+    with open(newest_file, 'r') as f:
+        newest_version_file_contents = f.read()
+        newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
 
-    newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
-
-    if previous_version_file_contents:
+    if previous_file:
+        with open(previous_file, 'r') as f:
+            previous_version_file_contents = f.read()
             previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
     else:
         previous_version_file_contents = ""
 
-    rendered_diff = customSequenceMatcher(before=previous_version_file_contents,
-                                          after=newest_version_file_contents,
-                                          include_equal=include_equal,
-                                          include_removed=include_removed,
-                                          include_added=include_added,
-                                          include_replaced=include_replaced,
-                                          include_change_type_prefix=include_change_type_prefix)
+    rendered_diff = customSequenceMatcher(previous_version_file_contents,
+                                          newest_version_file_contents,
+                                          include_equal)
 
     # Recursively join lists
     f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])

changedetectionio/fetch_site_status.py

@@ -1,35 +1,22 @@
-# HTML to TEXT/JSON DIFFERENCE FETCHER
-
 import hashlib
-import json
 import logging
 import os
 import re
 import urllib3
 
 from changedetectionio import content_fetcher, html_tools
-from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
-from copy import deepcopy
-from . import difference_detection_processor
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 
-name =  'Webpage Text/HTML, JSON and PDF changes'
-description = 'Detects all text changes where possible'
-
 class FilterNotFoundInResponse(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
 
-class PDFToHTMLToolNotFound(ValueError):
-    def __init__(self, msg):
-        ValueError.__init__(self, msg)
-
 
 # Some common stuff here that can be moved to a base class
 # (set_proxy_from_list)
-class perform_site_check(difference_detection_processor):
+class perform_site_check():
     screenshot = None
     xpath_data = None
 
@@ -50,15 +37,17 @@ class perform_site_check(difference_detection_processor):
 
         return regex
 
-    def run(self, uuid, skip_when_checksum_same=True):
+    def run(self, uuid):
+        from copy import deepcopy
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
 
         # DeepCopy so we can be sure we don't accidently change anything by reference
         watch = deepcopy(self.datastore.data['watching'].get(uuid))
+
         if not watch:
-            raise Exception("Watch no longer exists.")
+            return
 
         # Protect against file:// access
         if re.search(r'^file', watch.get('url', ''), re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
@@ -69,10 +58,11 @@ class perform_site_check(difference_detection_processor):
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
 
+        extra_headers = watch.get('headers', [])
+
         # Tweak the base config with the per-watch ones
-        request_headers = watch.get('headers', [])
-        request_headers.update(self.datastore.get_all_base_headers())
-        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
+        request_headers = deepcopy(self.datastore.data['settings']['headers'])
+        request_headers.update(extra_headers)
 
         # https://github.com/psf/requests/issues/4525
         # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
@@ -95,10 +85,7 @@ class perform_site_check(difference_detection_processor):
             is_source = True
 
         # Pluggable content fetcher
-        prefer_backend = watch.get_fetch_backend
-        if not prefer_backend or prefer_backend == 'system':
-            prefer_backend = self.datastore.data['settings']['application']['fetch_backend']
-
+        prefer_backend = watch.get('fetch_backend')
         if hasattr(content_fetcher, prefer_backend):
             klass = getattr(content_fetcher, prefer_backend)
         else:
@@ -128,26 +115,12 @@ class perform_site_check(difference_detection_processor):
         if watch.get('webdriver_js_execute_code') is not None and watch.get('webdriver_js_execute_code').strip():
             fetcher.webdriver_js_execute_code = watch.get('webdriver_js_execute_code')
 
-        # requests for PDF's, images etc should be passwd the is_binary flag
-        is_binary = watch.is_pdf
-
-        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'), is_binary=is_binary)
+        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'))
         fetcher.quit()
 
         self.screenshot = fetcher.screenshot
         self.xpath_data = fetcher.xpath_data
 
-        # Track the content type
-        update_obj['content_type'] = fetcher.get_all_headers().get('content-type', '').lower()
-
-        # Watches added automatically in the queue manager will skip if its the same checksum as the previous run
-        # Saves a lot of CPU
-        update_obj['previous_md5_before_filters'] = hashlib.md5(fetcher.content.encode('utf-8')).hexdigest()
-        if skip_when_checksum_same:
-            if update_obj['previous_md5_before_filters'] == watch.get('previous_md5_before_filters'):
-                raise content_fetcher.checksumFromPreviousCheckWasTheSame()
-
-
         # Fetching complete, now filters
         # @todo move to class / maybe inside of fetcher abstract base?
 
@@ -158,7 +131,7 @@ class perform_site_check(difference_detection_processor):
         # https://stackoverflow.com/questions/41817578/basic-method-chaining ?
         # return content().textfilter().jsonextract().checksumcompare() ?
 
-        is_json = 'application/json' in fetcher.get_all_headers().get('content-type', '').lower()
+        is_json = 'application/json' in fetcher.headers.get('Content-Type', '')
         is_html = not is_json
 
         # source: support, basically treat it as plaintext
@@ -166,60 +139,25 @@ class perform_site_check(difference_detection_processor):
             is_html = False
             is_json = False
 
-        if watch.is_pdf or 'application/pdf' in fetcher.get_all_headers().get('content-type', '').lower():
-            from shutil import which
-            tool = os.getenv("PDF_TO_HTML_TOOL", "pdftohtml")
-            if not which(tool):
-                raise PDFToHTMLToolNotFound("Command-line `{}` tool was not found in system PATH, was it installed?".format(tool))
-
-            import subprocess
-            proc = subprocess.Popen(
-                [tool, '-stdout', '-', '-s', 'out.pdf', '-i'],
-                stdout=subprocess.PIPE,
-                stdin=subprocess.PIPE)
-            proc.stdin.write(fetcher.raw_content)
-            proc.stdin.close()
-            fetcher.content = proc.stdout.read().decode('utf-8')
-            proc.wait(timeout=60)
-
-            # Add a little metadata so we know if the file changes (like if an image changes, but the text is the same
-            # @todo may cause problems with non-UTF8?
-            metadata = "<p>Added by changedetection.io: Document checksum - {} Filesize - {} bytes</p>".format(
-                hashlib.md5(fetcher.raw_content).hexdigest().upper(),
-                len(fetcher.content))
-
-            fetcher.content = fetcher.content.replace('</body>', metadata + '</body>')
-
-        # Better would be if Watch.model could access the global data also
-        # and then use getattr https://docs.python.org/3/reference/datamodel.html#object.__getitem__
-        # https://realpython.com/inherit-python-dict/ instead of doing it procedurely
-        include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='include_filters')
-        include_filters_rule = [*watch.get('include_filters', []), *include_filters_from_tags]
-
-        subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='subtractive_selectors'),
-                                 *watch.get("subtractive_selectors", []),
-                                 *self.datastore.data["settings"]["application"].get("global_subtractive_selectors", [])
-                                 ]
+        include_filters_rule = deepcopy(watch.get('include_filters', []))
+        # include_filters_rule = watch['include_filters']
+        subtractive_selectors = watch.get(
+            "subtractive_selectors", []
+        ) + self.datastore.data["settings"]["application"].get(
+            "global_subtractive_selectors", []
+        )
 
         # Inject a virtual LD+JSON price tracker rule
-        if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
-            include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
+        if watch.get('track_ldjson_price_data'):
+            include_filters_rule.append('json:$..price')
 
-        has_filter_rule = len(include_filters_rule) and len(include_filters_rule[0].strip())
-        has_subtractive_selectors = len(subtractive_selectors) and len(subtractive_selectors[0].strip())
+        has_filter_rule = include_filters_rule and len("".join(include_filters_rule).strip())
+        has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
 
         if is_json and not has_filter_rule:
             include_filters_rule.append("json:$")
             has_filter_rule = True
 
-        if is_json:
-            # Sort the JSON so we dont get false alerts when the content is just re-ordered
-            try:
-                fetcher.content = json.dumps(json.loads(fetcher.content), sort_keys=True)
-            except Exception as e:
-                # Might have just been a snippet, or otherwise bad JSON, continue
-                pass
-
         if has_filter_rule:
             json_filter_prefixes = ['json:', 'jq:']
             for filter in include_filters_rule:
@@ -227,8 +165,6 @@ class perform_site_check(difference_detection_processor):
                     stripped_text_from_html += html_tools.extract_json_as_string(content=fetcher.content, json_filter=filter)
                     is_html = False
 
-
-
         if is_html or is_source:
 
             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
@@ -236,7 +172,7 @@ class perform_site_check(difference_detection_processor):
             html_content = fetcher.content
 
             # If not JSON,  and if it's not text/plain..
-            if 'text/plain' in fetcher.get_all_headers().get('content-type', '').lower():
+            if 'text/plain' in fetcher.headers.get('Content-Type', '').lower():
                 # Don't run get_text or xpath/css filters on plaintext
                 stripped_text_from_html = html_content
             else:
@@ -279,34 +215,6 @@ class perform_site_check(difference_detection_processor):
         # Re #340 - return the content before the 'ignore text' was applied
         text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
-
-        # @todo whitespace coming from missing rtrim()?
-        # stripped_text_from_html could be based on their preferences, replace the processed text with only that which they want to know about.
-        # Rewrite's the processing text based on only what diff result they want to see
-        if watch.has_special_diff_filter_options_set() and len(watch.history.keys()):
-            # Now the content comes from the diff-parser and not the returned HTTP traffic, so could be some differences
-            from .. import diff
-            # needs to not include (added) etc or it may get used twice
-            # Replace the processed text with the preferred result
-            rendered_diff = diff.render_diff(previous_version_file_contents=watch.get_last_fetched_before_filters(),
-                                                       newest_version_file_contents=stripped_text_from_html,
-                                                       include_equal=False,  # not the same lines
-                                                       include_added=watch.get('filter_text_added', True),
-                                                       include_removed=watch.get('filter_text_removed', True),
-                                                       include_replaced=watch.get('filter_text_replaced', True),
-                                                       line_feed_sep="\n",
-                                                       include_change_type_prefix=False)
-
-            watch.save_last_fetched_before_filters(text_content_before_ignored_filter)
-
-            if not rendered_diff and stripped_text_from_html:
-                # We had some content, but no differences were found
-                # Store our new file as the MD5 so it will trigger in the future
-                c = hashlib.md5(text_content_before_ignored_filter.translate(None, b'\r\n\t ')).hexdigest()
-                return False, {'previous_md5': c}, stripped_text_from_html.encode('utf-8')
-            else:
-                stripped_text_from_html = rendered_diff
-
         # Treat pages with no renderable text content as a change? No by default
         empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
         if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
@@ -365,7 +273,6 @@ class perform_site_check(difference_detection_processor):
             blocked = True
             # Filter and trigger works the same, so reuse it
             # It should return the line numbers that match
-            # Unblock flow if the trigger was found (some text remained after stripped what didnt match)
             result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),
                                                   wordlist=trigger_text,
                                                   mode="line numbers")

changedetectionio/forms.py

@@ -28,8 +28,6 @@ from changedetectionio.notification import (
 
 from wtforms.fields import FormField
 
-dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
-
 valid_method = {
     'GET',
     'POST',
@@ -92,29 +90,6 @@ class SaltyPasswordField(StringField):
         else:
             self.data = False
 
-class StringTagUUID(StringField):
-
-   # process_formdata(self, valuelist) handled manually in POST handler
-
-    # Is what is shown when field <input> is rendered
-    def _value(self):
-        # Tag UUID to name, on submit it will convert it back (in the submit handler of init.py)
-        if self.data and type(self.data) is list:
-            tag_titles = []
-            for i in self.data:
-                tag = self.datastore.data['settings']['application']['tags'].get(i)
-                if tag:
-                    tag_title = tag.get('title')
-                    if tag_title:
-                        tag_titles.append(tag_title)
-
-            return ', '.join(tag_titles)
-
-        if not self.data:
-            return ''
-
-        return 'error'
-
 class TimeBetweenCheckForm(Form):
     weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
     days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
@@ -163,7 +138,7 @@ class ValidateContentFetcherIsReady(object):
         from changedetectionio import content_fetcher
 
         # Better would be a radiohandler that keeps a reference to each class
-        if field.data is not None and field.data != 'system':
+        if field.data is not None:
             klass = getattr(content_fetcher, field.data)
             some_object = klass()
             try:
@@ -172,12 +147,12 @@ class ValidateContentFetcherIsReady(object):
             except urllib3.exceptions.MaxRetryError as e:
                 driver_url = some_object.command_executor
                 message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data))
-                message += '<br>' + field.gettext(
+                message += '<br/>' + field.gettext(
                     'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.')
-                message += '<br>' + field.gettext('Did you follow the instructions in the wiki?')
-                message += '<br><br>' + field.gettext('WebDriver Host: %s' % (driver_url))
-                message += '<br><a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">Go here for more information</a>'
-                message += '<br>'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e)))
+                message += '<br/>' + field.gettext('Did you follow the instructions in the wiki?')
+                message += '<br/><br/>' + field.gettext('WebDriver Host: %s' % (driver_url))
+                message += '<br/><a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">Go here for more information</a>'
+                message += '<br/>'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e)))
 
                 raise ValidationError(message)
 
@@ -257,17 +232,12 @@ class validateURL(object):
 
     def __call__(self, form, field):
         import validators
-
         try:
             validators.url(field.data.strip())
         except validators.ValidationFailure:
             message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
             raise ValidationError(message)
 
-        from .model.Watch import is_safe_url
-        if not is_safe_url(field.data):
-            raise ValidationError('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX')
-
 
 class ValidateListRegex(object):
     """
@@ -369,18 +339,14 @@ class ValidateCSSJSONXPATHInput(object):
                     raise ValidationError("A system-error occurred when validating your jq expression")
 
 class quickWatchForm(Form):
-    from . import processors
-
     url = fields.URLField('URL', validators=[validateURL()])
-    tags = StringTagUUID('Group tag', [validators.Optional()])
+    tag = StringField('Group tag', [validators.Optional()])
     watch_submit_button = SubmitField('Watch', render_kw={"class": "pure-button pure-button-primary"})
-    processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff")
     edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"})
 
 
 # Common to a single watch and the global settings
 class commonSettingsForm(Form):
-
     notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
     notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
     notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
@@ -389,10 +355,6 @@ class commonSettingsForm(Form):
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
     webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1,
                                                                                                                                     message="Should contain one or more seconds")])
-class importForm(Form):
-    from . import processors
-    processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff")
-    urls = TextAreaField('URLs')
 
 class SingleBrowserStep(Form):
 
@@ -408,7 +370,7 @@ class SingleBrowserStep(Form):
 class watchForm(commonSettingsForm):
 
     url = fields.URLField('URL', validators=[validateURL()])
-    tags = StringTagUUID('Group tag', [validators.Optional()], default='')
+    tag = StringField('Group tag', [validators.Optional()], default='')
 
     time_between_check = FormField(TimeBetweenCheckForm)
 
@@ -425,19 +387,11 @@ class watchForm(commonSettingsForm):
     body = TextAreaField('Request body', [validators.Optional()])
     method = SelectField('Request method', choices=valid_method, default=default_method)
     ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
-    check_unique_lines = BooleanField('Only trigger when unique lines appear', default=False)
-
-    filter_text_added = BooleanField('Added lines', default=True)
-    filter_text_replaced = BooleanField('Replaced/changed lines', default=True)
-    filter_text_removed = BooleanField('Removed lines', default=True)
-
-    # @todo this class could be moved to its own text_json_diff_watchForm and this goes to restock_diff_Watchform perhaps
-    in_stock_only = BooleanField('Only trigger when product goes BACK to in-stock', default=True)
-
+    check_unique_lines = BooleanField('Only trigger when new lines appear', default=False)
     trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
     if os.getenv("PLAYWRIGHT_DRIVER_URL"):
         browser_steps = FieldList(FormField(SingleBrowserStep), min_entries=10)
-    text_should_not_be_present = StringListField('Block change-detection while text matches', [validators.Optional(), ValidateListRegex()])
+    text_should_not_be_present = StringListField('Block change-detection if text matches', [validators.Optional(), ValidateListRegex()])
     webdriver_js_execute_code = TextAreaField('Execute JavaScript before change detection', render_kw={"rows": "5"}, validators=[validators.Optional()])
 
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
@@ -472,13 +426,6 @@ class watchForm(commonSettingsForm):
         return result
 
 
-class SingleExtraProxy(Form):
-
-    # maybe better to set some <script>var..
-    proxy_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
-    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "http://user:pass@...:3128", "size":50})
-    # @todo do the validation here instead
-
 # datastore.data['settings']['requests']..
 class globalSettingsRequestForm(Form):
     time_between_check = FormField(TimeBetweenCheckForm)
@@ -486,34 +433,21 @@ class globalSettingsRequestForm(Form):
     jitter_seconds = IntegerField('Random jitter seconds ± check',
                                   render_kw={"style": "width: 5em;"},
                                   validators=[validators.NumberRange(min=0, message="Should contain zero or more seconds")])
-    extra_proxies = FieldList(FormField(SingleExtraProxy), min_entries=5)
-
-    def validate_extra_proxies(self, extra_validators=None):
-        for e in self.data['extra_proxies']:
-            if e.get('proxy_name') or e.get('proxy_url'):
-                if not e.get('proxy_name','').strip() or not e.get('proxy_url','').strip():
-                    self.extra_proxies.errors.append('Both a name, and a Proxy URL is required.')
-                    return False
-
 
 # datastore.data['settings']['application']..
 class globalSettingsApplicationForm(commonSettingsForm):
 
-    api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
     base_url = StringField('Base URL', validators=[validators.Optional()])
-    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
-    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
-    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
     global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
+    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
     ignore_whitespace = BooleanField('Ignore whitespace')
-    password = SaltyPasswordField()
-    pager_size = IntegerField('Pager size',
-                              render_kw={"style": "width: 5em;"},
-                              validators=[validators.NumberRange(min=0,
-                                                                 message="Should be atleast zero (disabled)")])
     removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
+    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
     render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
-    shared_diff_access = BooleanField('Allow access to view diff page when password is enabled', default=False, validators=[validators.Optional()])
+    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
+    password = SaltyPasswordField()
+
     filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
                                                                   render_kw={"style": "width: 5em;"},
                                                                   validators=[validators.NumberRange(min=0,

changedetectionio/html_tools.py

@@ -8,11 +8,7 @@ import json
 import re
 
 # HTML added to be sure each result matching a filter (.example) gets converted to a new line by Inscriptis
-TEXT_FILTER_LIST_LINE_SUFFIX = "<br>"
-
-# 'price' , 'lowPrice', 'highPrice' are usually under here
-# all of those may or may not appear on different websites
-LD_JSON_PRODUCT_OFFER_SELECTOR = "json:$..offers"
+TEXT_FILTER_LIST_LINE_SUFFIX = "<br/>"
 
 class JSONNotFound(ValueError):
     def __init__(self, msg):
@@ -137,13 +133,12 @@ def _get_stripped_text_from_json_match(match):
 def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None):
     stripped_text_from_html = False
 
-    # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags
+    # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded <script type=ldjson>
     try:
         stripped_text_from_html = _parse_json(json.loads(content), json_filter)
     except json.JSONDecodeError:
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
-        # As a last resort, try to parse the whole <body>
         s = []
         soup = BeautifulSoup(content, 'html.parser')
 
@@ -151,34 +146,30 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
             bs_result = soup.findAll('script', {"type": "application/ld+json"})
         else:
             bs_result = soup.findAll('script')
-        bs_result += soup.findAll('body')
 
-        bs_jsons = []
+
+        if not bs_result:
+            raise JSONNotFound("No parsable JSON found in this document")
+
         for result in bs_result:
             # Skip empty tags, and things that dont even look like JSON
-            if not result.text or '{' not in result.text:
+            if not result.string or not '{' in result.string:
                 continue
+                
             try:
-                json_data = json.loads(result.text)
-                bs_jsons.append(json_data)
+                json_data = json.loads(result.string)
             except json.JSONDecodeError:
-                # Skip objects which cannot be parsed
+                # Just skip it
                 continue
-
-        if not bs_jsons:
-            raise JSONNotFound("No parsable JSON found in this document")
-        
-        for json_data in bs_jsons:
-            stripped_text_from_html = _parse_json(json_data, json_filter)
-            if ensure_is_ldjson_info_type:
-                # Could sometimes be list, string or something else random
-                if isinstance(json_data, dict):
-                    # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
-                    # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
-                    if json_data.get('@type', False) and json_data.get('@type','').lower() == ensure_is_ldjson_info_type.lower() and stripped_text_from_html:
-                        break
-            elif stripped_text_from_html:
-                break
+            else:
+                stripped_text_from_html = _parse_json(json_data, json_filter)
+                if ensure_is_ldjson_info_type:
+                    # Could sometimes be list, string or something else random
+                    if isinstance(json_data, dict):
+                        if json_data.get('@type', False) and json_data.get('@type','').lower() == ensure_is_ldjson_info_type.lower():
+                            break
+                elif stripped_text_from_html:
+                    break
 
     if not stripped_text_from_html:
         # Re 265 - Just return an empty string when filter not found
@@ -268,7 +259,7 @@ def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
 # Does LD+JSON exist with a @type=='product' and a .price set anywhere?
 def has_ldjson_product_info(content):
     try:
-        pricing_data = extract_json_as_string(content=content, json_filter=LD_JSON_PRODUCT_OFFER_SELECTOR, ensure_is_ldjson_info_type="product")
+        pricing_data = extract_json_as_string(content=content, json_filter='json:$..price', ensure_is_ldjson_info_type="product")
     except JSONNotFound as e:
         # Totally fine
         return False
@@ -290,18 +281,3 @@ def workarounds_for_obfuscations(content):
     content = re.sub('<!--\s+-->', '', content)
 
     return content
-
-
-def get_triggered_text(content, trigger_text):
-    triggered_text = []
-    result = strip_ignore_text(content=content,
-                               wordlist=trigger_text,
-                               mode="line numbers")
-
-    i = 1
-    for p in content.splitlines():
-        if i in result:
-            triggered_text.append(p)
-        i += 1
-
-    return triggered_text

changedetectionio/importer.py

@@ -29,7 +29,6 @@ class import_url_list(Importer):
             data,
             flash,
             datastore,
-            processor=None
             ):
 
         urls = data.split("\n")
@@ -52,13 +51,8 @@ class import_url_list(Importer):
 
             # Flask wtform validators wont work with basic auth, use validators package
             # Up to 5000 per batch so we dont flood the server
-            # @todo validators.url failed on local hostnames (such as referring to ourself when using browserless)
-            if len(url) and 'http' in url.lower() and good < 5000:
-                extras = None
-                if processor:
-                    extras = {'processor': processor}
-                new_uuid = datastore.add_watch(url=url.strip(), tag=tags, write_to_disk_now=False, extras=extras)
-
+            if len(url) and validators.url(url.replace('source:', '')) and good < 5000:
+                new_uuid = datastore.add_watch(url=url.strip(), tag=tags, write_to_disk_now=False)
                 if new_uuid:
                     # Straight into the queue.
                     self.new_uuids.append(new_uuid)
@@ -85,8 +79,7 @@ class import_distill_io_json(Importer):
         now = time.time()
         self.new_uuids=[]
 
-        # @todo Use JSONSchema like in the API to validate here.
-        
+
         try:
             data = json.loads(data.strip())
         except json.decoder.JSONDecodeError:
@@ -121,8 +114,11 @@ class import_distill_io_json(Importer):
                 except IndexError:
                     pass
 
+
+                if d.get('tags', False):
+                    extras['tag'] = ", ".join(d['tags'])
+
                 new_uuid = datastore.add_watch(url=d['uri'].strip(),
-                                               tag=",".join(d.get('tags', [])),
                                                extras=extras,
                                                write_to_disk_now=False)
 

changedetectionio/model/App.py

@@ -15,35 +15,32 @@ class model(dict):
                 'headers': {
                 },
                 'requests': {
-                    'extra_proxies': [], # Configurable extra proxies via the UI
-                    'jitter_seconds': 0,
-                    'proxy': None, # Preferred proxy connection
-                    'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
                     'timeout': int(getenv("DEFAULT_SETTINGS_REQUESTS_TIMEOUT", "45")),  # Default 45 seconds
+                    'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
+                    'jitter_seconds': 0,
                     'workers': int(getenv("DEFAULT_SETTINGS_REQUESTS_WORKERS", "10")),  # Number of threads, lower is better for slow connections
+                    'proxy': None # Preferred proxy connection
                 },
                 'application': {
-                    # Custom notification content
                     'api_access_token_enabled': True,
+                    'password': False,
                     'base_url' : None,
-                    'empty_pages_are_a_change': False,
                     'extract_title_as_title': False,
+                    'empty_pages_are_a_change': False,
+                    'css_dark_mode': False,
                     'fetch_backend': getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
                     'filter_failure_notification_threshold_attempts': _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT,
                     'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
                     'global_subtractive_selectors': [],
                     'ignore_whitespace': True,
+                    'render_anchor_tag_content': False,
+                    'notification_urls': [], # Apprise URL list
+                    # Custom notification content
+                    'notification_title': default_notification_title,
                     'notification_body': default_notification_body,
                     'notification_format': default_notification_format,
-                    'notification_title': default_notification_title,
-                    'notification_urls': [], # Apprise URL list
-                    'pager_size': 50,
-                    'password': False,
-                    'render_anchor_tag_content': False,
                     'schema_version' : 0,
-                    'shared_diff_access': False,
-                    'webdriver_delay': None , # Extra delay in seconds before extracting text
-                    'tags': {} #@todo use Tag.model initialisers
+                    'webdriver_delay': None  # Extra delay in seconds before extracting text
                 }
             }
         }
@@ -51,15 +48,3 @@ class model(dict):
     def __init__(self, *arg, **kw):
         super(model, self).__init__(*arg, **kw)
         self.update(self.base_config)
-
-
-def parse_headers_from_text_file(filepath):
-    headers = {}
-    with open(filepath, 'r') as f:
-        for l in f.readlines():
-            l = l.strip()
-            if not l.startswith('#') and ':' in l:
-                (k, v) = l.split(':')
-                headers[k.strip()] = v.strip()
-
-    return headers

changedetectionio/model/Tag.py

@@ -1,19 +0,0 @@
-from .Watch import base_config
-import uuid
-
-class model(dict):
-
-    def __init__(self, *arg, **kw):
-
-        self.update(base_config)
-
-        self['uuid'] = str(uuid.uuid4())
-
-        if kw.get('default'):
-            self.update(kw['default'])
-            del kw['default']
-
-
-        # Goes at the end so we update the default object with the initialiser
-        super(model, self).__init__(*arg, **kw)
-

changedetectionio/model/Watch.py

@@ -1,14 +1,9 @@
 from distutils.util import strtobool
 import logging
 import os
-import re
 import time
 import uuid
 
-# Allowable protocols, protects against javascript: etc
-# file:// is further checked by ALLOW_FILE_URI
-SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):'
-
 minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
 mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
 
@@ -16,81 +11,59 @@ from changedetectionio.notification import (
     default_notification_format_for_watch
 )
 
-base_config = {
-    'body': None,
-    'check_unique_lines': False,  # On change-detected, compare against all history if its something new
-    'check_count': 0,
-    'date_created': None,
-    'consecutive_filter_failures': 0,  # Every time the CSS/xPath filter cannot be located, reset when all is fine.
-    'extract_text': [],  # Extract text by regex after filters
-    'extract_title_as_title': False,
-    'fetch_backend': 'system', # plaintext, playwright etc
-    'processor': 'text_json_diff', # could be restock_diff or others from .processors
-    'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
-    'filter_text_added': True,
-    'filter_text_replaced': True,
-    'filter_text_removed': True,
-    'has_ldjson_price_data': None,
-    'track_ldjson_price_data': None,
-    'headers': {},  # Extra headers to send
-    'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
-    'in_stock_only' : True, # Only trigger change on going to instock from out-of-stock
-    'include_filters': [],
-    'last_checked': 0,
-    'last_error': False,
-    'last_viewed': 0,  # history key value of the last viewed via the [diff] link
-    'method': 'GET',
-    # Custom notification content
-    'notification_body': None,
-    'notification_format': default_notification_format_for_watch,
-    'notification_muted': False,
-    'notification_title': None,
-    'notification_screenshot': False,  # Include the latest screenshot if available and supported by the apprise URL
-    'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
-    'paused': False,
-    'previous_md5': False,
-    'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
-    'proxy': None,  # Preferred proxy connection
-    'subtractive_selectors': [],
-    'tag': '', # Old system of text name for a tag, to be removed
-    'tags': [], # list of UUIDs to App.Tags
-    'text_should_not_be_present': [],  # Text that should not present
-    # Re #110, so then if this is set to None, we know to use the default value instead
-    # Requires setting to None on submit if it's the same as the default
-    # Should be all None by default, so we use the system default in this case.
-    'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
-    'title': None,
-    'trigger_text': [],  # List of text or regex to wait for until a change is detected
-    'url': '',
-    'uuid': str(uuid.uuid4()),
-    'webdriver_delay': None,
-    'webdriver_js_execute_code': None,  # Run before change-detection
-}
-
-
-def is_safe_url(test_url):
-    # See https://github.com/dgtlmoon/changedetection.io/issues/1358
-
-    # Remove 'source:' prefix so we dont get 'source:javascript:' etc
-    # 'source:' is a valid way to tell us to return the source
-
-    r = re.compile(re.escape('source:'), re.IGNORECASE)
-    test_url = r.sub('', test_url)
-
-    pattern = re.compile(os.getenv('SAFE_PROTOCOL_REGEX', SAFE_PROTOCOL_REGEX), re.IGNORECASE)
-    if not pattern.match(test_url.strip()):
-        return False
-
-    return True
 
 class model(dict):
     __newest_history_key = None
-    __history_n = 0
+    __history_n=0
+    __base_config = {
+            #'history': {},  # Dict of timestamp and output stripped filename (removed)
+            #'newest_history_key': 0, (removed, taken from history.txt index)
+            'body': None,
+            'check_unique_lines': False, # On change-detected, compare against all history if its something new
+            'check_count': 0,
+            'consecutive_filter_failures': 0, # Every time the CSS/xPath filter cannot be located, reset when all is fine.
+            'extract_text': [],  # Extract text by regex after filters
+            'extract_title_as_title': False,
+            'fetch_backend': None,
+            'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
+            'has_ldjson_price_data': None,
+            'track_ldjson_price_data': None,
+            'headers': {},  # Extra headers to send
+            'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
+            'include_filters': [],
+            'last_checked': 0,
+            'last_error': False,
+            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
+            'method': 'GET',
+             # Custom notification content
+            'notification_body': None,
+            'notification_format': default_notification_format_for_watch,
+            'notification_muted': False,
+            'notification_title': None,
+            'notification_screenshot': False, # Include the latest screenshot if available and supported by the apprise URL
+            'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
+            'paused': False,
+            'previous_md5': False,
+            'proxy': None, # Preferred proxy connection
+            'subtractive_selectors': [],
+            'tag': None,
+            'text_should_not_be_present': [], # Text that should not present
+            # Re #110, so then if this is set to None, we know to use the default value instead
+            # Requires setting to None on submit if it's the same as the default
+            # Should be all None by default, so we use the system default in this case.
+            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
+            'title': None,
+            'trigger_text': [],  # List of text or regex to wait for until a change is detected
+            'url': None,
+            'uuid': str(uuid.uuid4()),
+            'webdriver_delay': None,
+            'webdriver_js_execute_code': None, # Run before change-detection
+        }
     jitter_seconds = 0
 
     def __init__(self, *arg, **kw):
 
-        self.update(base_config)
+        self.update(self.__base_config)
         self.__datastore_path = kw['datastore_path']
 
         self['uuid'] = str(uuid.uuid4())
@@ -121,11 +94,7 @@ class model(dict):
 
     @property
     def link(self):
-
         url = self.get('url', '')
-        if not is_safe_url(url):
-            return 'DISABLED'
-
         ready_url = url
         if '{%' in url or '{{' in url:
             from jinja2 import Environment
@@ -144,26 +113,6 @@ class model(dict):
 
         return ready_url
 
-    @property
-    def get_fetch_backend(self):
-        """
-        Like just using the `fetch_backend` key but there could be some logic
-        :return:
-        """
-        # Maybe also if is_image etc?
-        # This is because chrome/playwright wont render the PDF in the browser and we will just fetch it and use pdf2html to see the text.
-        if self.is_pdf:
-            return 'html_requests'
-
-        return self.get('fetch_backend')
-
-    @property
-    def is_pdf(self):
-        # content_type field is set in the future
-        # https://github.com/dgtlmoon/changedetection.io/issues/1392
-        # Not sure the best logic here
-        return self.get('url', '').lower().endswith('.pdf') or 'pdf' in self.get('content_type', '').lower()
-
     @property
     def label(self):
         # Used for sorting
@@ -246,32 +195,9 @@ class model(dict):
         bump = self.history
         return self.__newest_history_key
 
-    def get_history_snapshot(self, timestamp):
-        import brotli
-        filepath = self.history[timestamp]
-
-        # See if a brotli versions exists and switch to that
-        if not filepath.endswith('.br') and os.path.isfile(f"{filepath}.br"):
-            filepath = f"{filepath}.br"
-
-        # OR in the backup case that the .br does not exist, but the plain one does
-        if filepath.endswith('.br') and not os.path.isfile(filepath):
-            if os.path.isfile(filepath.replace('.br', '')):
-                filepath = filepath.replace('.br', '')
-
-        if filepath.endswith('.br'):
-            # Brotli doesnt have a fileheader to detect it, so we rely on filename
-            # https://www.rfc-editor.org/rfc/rfc7932
-            with open(filepath, 'rb') as f:
-                return(brotli.decompress(f.read()).decode('utf-8'))
-
-        with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
-            return f.read()
-
     # Save some text file to the appropriate path and bump the history
     # result_obj from fetch_site_status.run()
-    def save_history_text(self, contents, timestamp, snapshot_id):
-        import brotli
+    def save_history_text(self, contents, timestamp):
 
         self.ensure_data_dir_exists()
 
@@ -280,21 +206,13 @@ class model(dict):
         if self.__newest_history_key and int(timestamp) == int(self.__newest_history_key):
             time.sleep(timestamp - self.__newest_history_key)
 
-        threshold = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD', 1024))
-        skip_brotli = strtobool(os.getenv('DISABLE_BROTLI_TEXT_SNAPSHOT', 'False'))
+        snapshot_fname = "{}.txt".format(str(uuid.uuid4()))
 
-        if not skip_brotli and len(contents) > threshold:
-            snapshot_fname = f"{snapshot_id}.txt.br"
-            dest = os.path.join(self.watch_data_dir, snapshot_fname)
-            if not os.path.exists(dest):
-                with open(dest, 'wb') as f:
-                    f.write(brotli.compress(contents, mode=brotli.MODE_TEXT))
-        else:
-            snapshot_fname = f"{snapshot_id}.txt"
-            dest = os.path.join(self.watch_data_dir, snapshot_fname)
-            if not os.path.exists(dest):
-                with open(dest, 'wb') as f:
-                    f.write(contents)
+        # in /diff/ and /preview/ we are going to assume for now that it's UTF-8 when reading
+        # most sites are utf-8 and some are even broken utf-8
+        with open(os.path.join(self.watch_data_dir, snapshot_fname), 'wb') as f:
+            f.write(contents)
+            f.close()
 
         # Append to index
         # @todo check last char was \n
@@ -331,8 +249,7 @@ class model(dict):
         # Compare each lines (set) against each history text file (set) looking for something new..
         existing_history = set({})
         for k, v in self.history.items():
-            content = self.get_history_snapshot(k)
-            alist = set([line.strip().lower() for line in content.splitlines()])
+            alist = set([line.decode('utf-8').strip().lower() for line in open(v, 'rb')])
             existing_history = existing_history.union(alist)
 
         # Check that everything in local_lines(new stuff) already exists in existing_history - it should
@@ -347,6 +264,17 @@ class model(dict):
         # False is not an option for AppRise, must be type None
         return None
 
+    def get_screenshot_as_jpeg(self):
+
+        # Created by save_screenshot()
+        fname = os.path.join(self.watch_data_dir, "last-screenshot.jpg")
+        if os.path.isfile(fname):
+            return fname
+
+        # False is not an option for AppRise, must be type None
+        return None
+
+
     def __get_file_ctime(self, filename):
         fname = os.path.join(self.watch_data_dir, filename)
         if os.path.isfile(fname):
@@ -393,25 +321,6 @@ class model(dict):
             return fname
         return False
 
-
-    def pause(self):
-        self['paused'] = True
-
-    def unpause(self):
-        self['paused'] = False
-
-    def toggle_pause(self):
-        self['paused'] ^= True
-
-    def mute(self):
-        self['notification_muted'] = True
-
-    def unmute(self):
-        self['notification_muted'] = False
-
-    def toggle_mute(self):
-        self['notification_muted'] ^= True
-
     def extract_regex_from_all_history(self, regex):
         import csv
         import re
@@ -423,8 +332,8 @@ class model(dict):
         # self.history will be keyed with the full path
         for k, fname in self.history.items():
             if os.path.isfile(fname):
-                if True:
-                    contents = self.get_history_snapshot(k)
+                with open(fname, "r") as f:
+                    contents = f.read()
                     res = re.findall(regex, contents, re.MULTILINE)
                     if res:
                         if not csv_writer:
@@ -455,39 +364,3 @@ class model(dict):
             f.close()
 
         return csv_output_filename
-
-
-    def has_special_diff_filter_options_set(self):
-
-        # All False - nothing would be done, so act like it's not processable
-        if not self.get('filter_text_added', True) and not self.get('filter_text_replaced', True) and not self.get('filter_text_removed', True):
-            return False
-
-        # Or one is set
-        if not self.get('filter_text_added', True) or not self.get('filter_text_replaced', True) or not self.get('filter_text_removed', True):
-            return True
-
-        # None is set
-        return False
-
-
-    def get_last_fetched_before_filters(self):
-        import brotli
-        filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
-
-        if not os.path.isfile(filepath):
-            # If a previous attempt doesnt yet exist, just snarf the previous snapshot instead
-            dates = list(self.history.keys())
-            if len(dates):
-                return self.get_history_snapshot(dates[-1])
-            else:
-                return ''
-
-        with open(filepath, 'rb') as f:
-            return(brotli.decompress(f.read()).decode('utf-8'))
-
-    def save_last_fetched_before_filters(self, contents):
-        import brotli
-        filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
-        with open(filepath, 'wb') as f:
-            f.write(brotli.compress(contents, mode=brotli.MODE_TEXT))

changedetectionio/notification.py

@@ -5,18 +5,15 @@ import json
 
 valid_tokens = {
     'base_url': '',
-    'current_snapshot': '',
-    'diff': '',
-    'diff_added': '',
-    'diff_full': '',
-    'diff_removed': '',
-    'diff_url': '',
-    'preview_url': '',
-    'triggered_text': '',
-    'watch_tag': '',
-    'watch_title': '',
     'watch_url': '',
     'watch_uuid': '',
+    'watch_title': '',
+    'watch_tag': '',
+    'diff': '',
+    'diff_full': '',
+    'diff_url': '',
+    'preview_url': '',
+    'current_snapshot': ''
 }
 
 default_notification_format_for_watch = 'System default'
@@ -89,15 +86,9 @@ def process_notification(n_object, datastore):
     n_body = jinja2_env.from_string(n_object.get('notification_body', default_notification_body)).render(**notification_parameters)
     n_title = jinja2_env.from_string(n_object.get('notification_title', default_notification_title)).render(**notification_parameters)
     n_format = valid_notification_formats.get(
-        n_object.get('notification_format', default_notification_format),
+        n_object['notification_format'],
         valid_notification_formats[default_notification_format],
     )
-
-    # If we arrived with 'System default' then look it up
-    if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
-        # Initially text or whatever
-        n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
-
     
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
@@ -129,10 +120,10 @@ def process_notification(n_object, datastore):
                     url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
 
                 if url.startswith('tgram://'):
-                    # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
+                    # Telegram only supports a limit subset of HTML, remove the '<br/>' we place in.
                     # re https://github.com/dgtlmoon/changedetection.io/issues/555
                     # @todo re-use an existing library we have already imported to strip all non-allowed tags
-                    n_body = n_body.replace('<br>', '\n')
+                    n_body = n_body.replace('<br/>', '\n')
                     n_body = n_body.replace('</br>', '\n')
                     # real limit is 4096, but minus some for extra metadata
                     payload_max_size = 3600
@@ -192,13 +183,8 @@ def create_notification_parameters(n_object, datastore):
     uuid = n_object['uuid'] if 'uuid' in n_object else ''
 
     if uuid != '':
-        watch_title = datastore.data['watching'][uuid].get('title', '')
-        tag_list = []
-        tags = datastore.get_all_tags_for_watch(uuid)
-        if tags:
-            for tag_uuid, tag in tags.items():
-                tag_list.append(tag.get('title'))
-        watch_tag = ', '.join(tag_list)
+        watch_title = datastore.data['watching'][uuid]['title']
+        watch_tag = datastore.data['watching'][uuid]['tag']
     else:
         watch_title = 'Change Detection'
         watch_tag = ''
@@ -223,18 +209,15 @@ def create_notification_parameters(n_object, datastore):
     tokens.update(
         {
             'base_url': base_url if base_url is not None else '',
-            'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '',
-            'diff': n_object.get('diff', ''),  # Null default in the case we use a test
-            'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test
-            'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
-            'diff_removed': n_object.get('diff_removed', ''),  # Null default in the case we use a test
-            'diff_url': diff_url,
-            'preview_url': preview_url,
-            'triggered_text': n_object.get('triggered_text', ''),
-            'watch_tag': watch_tag if watch_tag is not None else '',
-            'watch_title': watch_title if watch_title is not None else '',
             'watch_url': watch_url,
             'watch_uuid': uuid,
+            'watch_title': watch_title if watch_title is not None else '',
+            'watch_tag': watch_tag if watch_tag is not None else '',
+            'diff_url': diff_url,
+            'diff': n_object.get('diff', ''),  # Null default in the case we use a test
+            'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
+            'preview_url': preview_url,
+            'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
         })
 
     return tokens

changedetectionio/processors/README.md

@@ -1,11 +0,0 @@
-# Change detection post-processors
-
-The concept here is to be able to switch between different domain specific problems to solve.
-
-- `text_json_diff` The traditional text and JSON comparison handler
-- `restock_diff` Only cares about detecting if a product looks like it has some text that suggests that it's out of stock, otherwise assumes that it's in stock.
-
-Some suggestions for the future
-
-- `graphical` 
-- `restock_and_price` - extract price AND stock text

changedetectionio/processors/__init__.py

@@ -1,24 +0,0 @@
-from abc import abstractmethod
-import hashlib
-
-
-class difference_detection_processor():
-
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    @abstractmethod
-    def run(self, uuid, skip_when_checksum_same=True):
-        update_obj = {'last_notification_error': False, 'last_error': False}
-        some_data = 'xxxxx'
-        update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()
-        changed_detected = False
-        return changed_detected, update_obj, ''.encode('utf-8')
-
-
-def available_processors():
-    from . import restock_diff, text_json_diff
-    x=[('text_json_diff', text_json_diff.name), ('restock_diff', restock_diff.name)]
-    # @todo Make this smarter with introspection of sorts.
-    return x

changedetectionio/processors/restock_diff.py

@@ -1,131 +0,0 @@
-
-import hashlib
-import os
-import re
-import urllib3
-from . import difference_detection_processor
-from changedetectionio import content_fetcher
-from copy import deepcopy
-
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-name = 'Re-stock detection for single product pages'
-description = 'Detects if the product goes back to in-stock'
-
-class UnableToExtractRestockData(Exception):
-    def __init__(self, status_code):
-        # Set this so we can use it in other parts of the app
-        self.status_code = status_code
-        return
-
-class perform_site_check(difference_detection_processor):
-    screenshot = None
-    xpath_data = None
-
-    def __init__(self, *args, datastore, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.datastore = datastore
-
-    def run(self, uuid, skip_when_checksum_same=True):
-
-        # DeepCopy so we can be sure we don't accidently change anything by reference
-        watch = deepcopy(self.datastore.data['watching'].get(uuid))
-
-        if not watch:
-            raise Exception("Watch no longer exists.")
-
-        # Protect against file:// access
-        if re.search(r'^file', watch.get('url', ''), re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
-            raise Exception(
-                "file:// type access is denied for security reasons."
-            )
-
-        # Unset any existing notification error
-        update_obj = {'last_notification_error': False, 'last_error': False}
-
-        request_headers = watch.get('headers', [])
-        request_headers.update(self.datastore.get_all_base_headers())
-        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid))
-
-        # https://github.com/psf/requests/issues/4525
-        # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
-        # do this by accident.
-        if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
-            request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
-
-        timeout = self.datastore.data['settings']['requests'].get('timeout')
-
-        url = watch.link
-
-        request_body = self.datastore.data['watching'][uuid].get('body')
-        request_method = self.datastore.data['watching'][uuid].get('method')
-        ignore_status_codes = self.datastore.data['watching'][uuid].get('ignore_status_codes', False)
-
-        # Pluggable content fetcher
-        prefer_backend = watch.get_fetch_backend
-        if not prefer_backend or prefer_backend == 'system':
-            prefer_backend = self.datastore.data['settings']['application']['fetch_backend']
-
-        if hasattr(content_fetcher, prefer_backend):
-            klass = getattr(content_fetcher, prefer_backend)
-        else:
-            # If the klass doesnt exist, just use a default
-            klass = getattr(content_fetcher, "html_requests")
-
-        proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
-        proxy_url = None
-        if proxy_id:
-            proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')
-            print("UUID {} Using proxy {}".format(uuid, proxy_url))
-
-        fetcher = klass(proxy_override=proxy_url)
-
-        # Configurable per-watch or global extra delay before extracting text (for webDriver types)
-        system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)
-        if watch['webdriver_delay'] is not None:
-            fetcher.render_extract_delay = watch.get('webdriver_delay')
-        elif system_webdriver_delay is not None:
-            fetcher.render_extract_delay = system_webdriver_delay
-
-        # Could be removed if requests/plaintext could also return some info?
-        if prefer_backend != 'html_webdriver':
-            raise Exception("Re-stock detection requires Chrome or compatible webdriver/playwright fetcher to work")
-
-        if watch.get('webdriver_js_execute_code') is not None and watch.get('webdriver_js_execute_code').strip():
-            fetcher.webdriver_js_execute_code = watch.get('webdriver_js_execute_code')
-
-        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'))
-        fetcher.quit()
-
-        self.screenshot = fetcher.screenshot
-        self.xpath_data = fetcher.xpath_data
-
-        # Track the content type
-        update_obj['content_type'] = fetcher.headers.get('Content-Type', '')
-        update_obj["last_check_status"] = fetcher.get_last_status_code()
-
-        # Main detection method
-        fetched_md5 = None
-        if fetcher.instock_data:
-            fetched_md5 = hashlib.md5(fetcher.instock_data.encode('utf-8')).hexdigest()
-            # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
-            update_obj["in_stock"] = True if fetcher.instock_data == 'Possibly in stock' else False
-        else:
-            raise UnableToExtractRestockData(status_code=fetcher.status_code)
-
-        # The main thing that all this at the moment comes down to :)
-        changed_detected = False
-
-        if watch.get('previous_md5') and watch.get('previous_md5') != fetched_md5:
-            # Yes if we only care about it going to instock, AND we are in stock
-            if watch.get('in_stock_only') and update_obj["in_stock"]:
-                changed_detected = True
-
-            if not watch.get('in_stock_only'):
-                # All cases
-                changed_detected = True
-
-        # Always record the new checksum
-        update_obj["previous_md5"] = fetched_md5
-
-        return changed_detected, update_obj, fetcher.instock_data.encode('utf-8')

changedetectionio/queuedWatchMetaData.py

@@ -1,10 +0,0 @@
-from dataclasses import dataclass, field
-from typing import Any
-
-# So that we can queue some metadata in `item`
-# https://docs.python.org/3/library/queue.html#queue.PriorityQueue
-#
-@dataclass(order=True)
-class PrioritizedItem:
-    priority: int
-    item: Any=field(compare=False)

changedetectionio/res/puppeteer_fetch.js

@@ -1,183 +0,0 @@
-module.exports = async ({page, context}) => {
-
-    var {
-        url,
-        execute_js,
-        user_agent,
-        extra_wait_ms,
-        req_headers,
-        include_filters,
-        xpath_element_js,
-        screenshot_quality,
-        proxy_username,
-        proxy_password,
-        disk_cache_dir,
-        no_cache_list,
-        block_url_list,
-    } = context;
-
-    await page.setBypassCSP(true)
-    await page.setExtraHTTPHeaders(req_headers);
-    await page.setUserAgent(user_agent);
-    // https://ourcodeworld.com/articles/read/1106/how-to-solve-puppeteer-timeouterror-navigation-timeout-of-30000-ms-exceeded
-
-    await page.setDefaultNavigationTimeout(0);
-
-    if (proxy_username) {
-        await page.authenticate({
-            username: proxy_username,
-            password: proxy_password
-        });
-    }
-
-    await page.setViewport({
-        width: 1024,
-        height: 768,
-        deviceScaleFactor: 1,
-    });
-
-    await page.setRequestInterception(true);
-    if (disk_cache_dir) {
-        console.log(">>>>>>>>>>>>>>> LOCAL DISK CACHE ENABLED <<<<<<<<<<<<<<<<<<<<<");
-    }
-    const fs = require('fs');
-    const crypto = require('crypto');
-
-    function file_is_expired(file_path) {
-        if (!fs.existsSync(file_path)) {
-            return true;
-        }
-        var stats = fs.statSync(file_path);
-        const now_date = new Date();
-        const expire_seconds = 300;
-        if ((now_date / 1000) - (stats.mtime.getTime() / 1000) > expire_seconds) {
-            console.log("CACHE EXPIRED: " + file_path);
-            return true;
-        }
-        return false;
-
-    }
-
-    page.on('request', async (request) => {
-        // General blocking of requests that waste traffic
-        if (block_url_list.some(substring => request.url().toLowerCase().includes(substring))) return request.abort();
-
-        if (disk_cache_dir) {
-            const url = request.url();
-            const key = crypto.createHash('md5').update(url).digest("hex");
-            const dir_path = disk_cache_dir + key.slice(0, 1) + '/' + key.slice(1, 2) + '/' + key.slice(2, 3) + '/';
-
-            // https://stackoverflow.com/questions/4482686/check-synchronously-if-file-directory-exists-in-node-js
-
-            if (fs.existsSync(dir_path + key)) {
-                console.log("* CACHE HIT , using - " + dir_path + key + " - " + url);
-                const cached_data = fs.readFileSync(dir_path + key);
-                // @todo headers can come from dir_path+key+".meta" json file
-                request.respond({
-                    status: 200,
-                    //contentType: 'text/html', //@todo
-                    body: cached_data
-                });
-                return;
-            }
-        }
-        request.continue();
-    });
-
-
-    if (disk_cache_dir) {
-        page.on('response', async (response) => {
-            const url = response.url();
-            // Basic filtering for sane responses
-            if (response.request().method() != 'GET' || response.request().resourceType() == 'xhr' || response.request().resourceType() == 'document' || response.status() != 200) {
-                console.log("Skipping (not useful) - Status:" + response.status() + " Method:" + response.request().method() + " ResourceType:" + response.request().resourceType() + " " + url);
-                return;
-            }
-            if (no_cache_list.some(substring => url.toLowerCase().includes(substring))) {
-                console.log("Skipping (no_cache_list) - " + url);
-                return;
-            }
-            if (url.toLowerCase().includes('data:')) {
-                console.log("Skipping (embedded-data) - " + url);
-                return;
-            }
-            response.buffer().then(buffer => {
-                if (buffer.length > 100) {
-                    console.log("Cache - Saving " + response.request().method() + " - " + url + " - " + response.request().resourceType());
-
-                    const key = crypto.createHash('md5').update(url).digest("hex");
-                    const dir_path = disk_cache_dir + key.slice(0, 1) + '/' + key.slice(1, 2) + '/' + key.slice(2, 3) + '/';
-
-                    if (!fs.existsSync(dir_path)) {
-                        fs.mkdirSync(dir_path, {recursive: true})
-                    }
-
-                    if (fs.existsSync(dir_path + key)) {
-                        if (file_is_expired(dir_path + key)) {
-                            fs.writeFileSync(dir_path + key, buffer);
-                        }
-                    } else {
-                        fs.writeFileSync(dir_path + key, buffer);
-                    }
-                }
-            });
-        });
-    }
-
-    const r = await page.goto(url, {
-        waitUntil: 'load'
-    });
-
-    await page.waitForTimeout(1000);
-    await page.waitForTimeout(extra_wait_ms);
-
-    if (execute_js) {
-        await page.evaluate(execute_js);
-        await page.waitForTimeout(200);
-    }
-
-    var xpath_data;
-    var instock_data;
-    try {
-        // Not sure the best way here, in the future this should be a new package added to npm then run in browserless
-        // (Once the old playwright is removed)
-        xpath_data = await page.evaluate((include_filters) => {%xpath_scrape_code%}, include_filters);
-        instock_data = await page.evaluate(() => {%instock_scrape_code%});
-    } catch (e) {
-        console.log(e);
-    }
-
-    // Protocol error (Page.captureScreenshot): Cannot take screenshot with 0 width can come from a proxy auth failure
-    // Wrap it here (for now)
-
-    var b64s = false;
-    try {
-        b64s = await page.screenshot({encoding: "base64", fullPage: true, quality: screenshot_quality, type: 'jpeg'});
-    } catch (e) {
-        console.log(e);
-    }
-
-    // May fail on very large pages with 'WARNING: tile memory limits exceeded, some content may not draw'
-    if (!b64s) {
-        // @todo after text extract, we can place some overlay text with red background to say 'croppped'
-        console.error('ERROR: content-fetcher page was maybe too large for a screenshot, reverting to viewport only screenshot');
-        try {
-            b64s = await page.screenshot({encoding: "base64", quality: screenshot_quality, type: 'jpeg'});
-        } catch (e) {
-            console.log(e);
-        }
-    }
-
-    var html = await page.content();
-    return {
-        data: {
-            'content': html,
-            'headers': r.headers(),
-            'instock_data': instock_data,
-            'screenshot': b64s,
-            'status_code': r.status(),
-            'xpath_data': xpath_data
-        },
-        type: 'application/json',
-    };
-};

changedetectionio/res/stock-not-in-stock.js

@@ -1,102 +0,0 @@
-function isItemInStock() {
-  // @todo Pass these in so the same list can be used in non-JS fetchers
-  const outOfStockTexts = [
-    '0 in stock',
-    'agotado',
-    'artikel zurzeit vergriffen',
-    'as soon as stock is available',
-    'available for back order',
-    'backordered',
-    'brak na stanie',
-    'brak w magazynie',
-    'coming soon',
-    'currently have any tickets for this',
-    'currently unavailable',
-    'en rupture de stock',
-    'item is no longer available',
-    'message if back in stock',
-    'nachricht bei',
-    'nicht auf lager',
-    'nicht lieferbar',
-    'nicht zur verfügung',
-    'no disponible temporalmente',
-    'no longer in stock',
-    'no tickets available',
-    'not available',
-    'not currently available',
-    'not in stock',
-    'notify me when available',
-    'não estamos a aceitar encomendas',
-    'out of stock',
-    'out-of-stock',
-    'produkt niedostępny',
-    'sold out',
-    'temporarily out of stock',
-    'temporarily unavailable',
-    'tickets unavailable',
-    'unavailable tickets',
-    'we do not currently have an estimate of when this product will be back in stock.',
-    'zur zeit nicht an lager',
-  ];
-
-
-  const negateOutOfStockRegexs = [
-      '[0-9] in stock'
-  ]
-  var negateOutOfStockRegexs_r = [];
-  for (let i = 0; i < negateOutOfStockRegexs.length; i++) {
-    negateOutOfStockRegexs_r.push(new RegExp(negateOutOfStockRegexs[0], 'g'));
-  }
-
-
-  const elementsWithZeroChildren = Array.from(document.getElementsByTagName('*')).filter(element => element.children.length === 0);
-
-  // REGEXS THAT REALLY MEAN IT'S IN STOCK
-  for (let i = elementsWithZeroChildren.length - 1; i >= 0; i--) {
-    const element = elementsWithZeroChildren[i];
-    if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
-      var elementText="";
-      if (element.tagName.toLowerCase() === "input") {
-        elementText = element.value.toLowerCase();
-      } else {
-        elementText = element.textContent.toLowerCase();
-      }
-
-      if (elementText.length) {
-        // try which ones could mean its in stock
-        for (let i = 0; i < negateOutOfStockRegexs.length; i++) {
-          if (negateOutOfStockRegexs_r[i].test(elementText)) {
-            return 'Possibly in stock';
-          }
-        }
-      }
-    }
-  }
-
-  // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
-  for (let i = elementsWithZeroChildren.length - 1; i >= 0; i--) {
-    const element = elementsWithZeroChildren[i];
-    if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
-      var elementText="";
-      if (element.tagName.toLowerCase() === "input") {
-        elementText = element.value.toLowerCase();
-      } else {
-        elementText = element.textContent.toLowerCase();
-      }
-
-      if (elementText.length) {
-        // and these mean its out of stock
-        for (const outOfStockText of outOfStockTexts) {
-          if (elementText.includes(outOfStockText)) {
-            return elementText; // item is out of stock
-          }
-        }
-      }
-    }
-  }
-
-  return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
-}
-
-// returns the element text that makes it think it's out of stock
-return isItemInStock();

changedetectionio/res/xpath_element_scraper.js

@@ -1,6 +1,3 @@
-// Copyright (C) 2021 Leigh Morresi (dgtlmoon@gmail.com)
-// All rights reserved.
-
 // @file Scrape the page looking for elements of concern (%ELEMENTS%)
 // http://matatk.agrip.org.uk/tests/position-and-width/
 // https://stackoverflow.com/questions/26813480/when-is-element-getboundingclientrect-guaranteed-to-be-updated-accurate
@@ -8,14 +5,8 @@
 // Some pages like https://www.londonstockexchange.com/stock/NCCL/ncondezi-energy-limited/analysis
 // will automatically force a scroll somewhere, so include the position offset
 // Lets hope the position doesnt change while we iterate the bbox's, but this is better than nothing
-var scroll_y = 0;
-try {
-    scroll_y = +document.documentElement.scrollTop || document.body.scrollTop
-} catch (e) {
-    console.log(e);
-}
-
 
+var scroll_y=+document.documentElement.scrollTop || document.body.scrollTop
 
 // Include the getXpath script directly, easier than fetching
 function getxpath(e) {
@@ -44,15 +35,15 @@ const findUpTag = (el) => {
     if (el.name !== undefined && el.name.length) {
         var proposed = el.tagName + "[name=" + el.name + "]";
         var proposed_element = window.document.querySelectorAll(proposed);
-        if (proposed_element.length) {
+        if(proposed_element.length) {
             if (proposed_element.length === 1) {
                 return proposed;
             } else {
                 // Some sites change ID but name= stays the same, we can hit it if we know the index
                 // Find all the elements that match and work out the input[n]
-                var n = Array.from(proposed_element).indexOf(el);
+                var n=Array.from(proposed_element).indexOf(el);
                 // Return a Playwright selector for nthinput[name=zipcode]
-                return proposed + " >> nth=" + n;
+                return proposed+" >> nth="+n;
             }
         }
     }
@@ -90,16 +81,8 @@ var bbox;
 for (var i = 0; i < elements.length; i++) {
     bbox = elements[i].getBoundingClientRect();
 
-    // Exclude items that are not interactable or visible
-    if(elements[i].style.opacity === "0") {
-        continue
-    }
-    if(elements[i].style.display === "none" || elements[i].style.pointerEvents === "none" ) {
-        continue
-    }
-
-    // Skip really small ones, and where width or height ==0
-    if (bbox['width'] * bbox['height'] < 100) {
+    // Forget really small ones
+    if (bbox['width'] < 10 && bbox['height'] < 10) {
         continue;
     }
 
@@ -155,6 +138,7 @@ for (var i = 0; i < elements.length; i++) {
 
 }
 
+
 // Inject the current one set in the include_filters, which may be a CSS rule
 // used for displaying the current one in VisualSelector, where its not one we generated.
 if (include_filters.length) {
@@ -182,23 +166,10 @@ if (include_filters.length) {
         }
 
         if (q) {
-            // #1231 - IN the case XPath attribute filter is applied, we will have to traverse up and find the element.
-            if (q.hasOwnProperty('getBoundingClientRect')) {
-                bbox = q.getBoundingClientRect();
-                console.log("xpath_element_scraper: Got filter element, scroll from top was " + scroll_y)
-            } else {
-                try {
-                    // Try and see we can find its ownerElement
-                    bbox = q.ownerElement.getBoundingClientRect();
-                    console.log("xpath_element_scraper: Got filter by ownerElement element, scroll from top was " + scroll_y)
-                } catch (e) {
-                    console.log("xpath_element_scraper: error looking up ownerElement")
-                }
-            }
-        }
-        
-        if(!q) {
-            console.log("xpath_element_scraper: filter element " + f + " was not found");
+            bbox = q.getBoundingClientRect();
+            console.log("xpath_element_scraper: Got filter element, scroll from top was "+scroll_y)
+        } else {
+            console.log("xpath_element_scraper: filter element "+f+" was not found");
         }
 
         if (bbox && bbox['width'] > 0 && bbox['height'] > 0) {
@@ -213,9 +184,5 @@ if (include_filters.length) {
     }
 }
 
-// Sort the elements so we find the smallest one first, in other words, we find the smallest one matching in that area
-// so that we dont select the wrapping element by mistake and be unable to select what we want
-size_pos.sort((a, b) => (a.width*a.height > b.width*b.height) ? 1 : -1)
-
 // Window.width required for proper scaling in the frontend
 return {'size_pos': size_pos, 'browser_width': window.innerWidth};

changedetectionio/run_all_tests.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+
+# live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions
+# and I like to restart the server for each test (and have the test cleanup after each test)
+# merge request welcome :)
+
+
+# exit when any command fails
+set -e
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+find tests/test_*py -type f|while read test_name
+do
+  echo "TEST RUNNING $test_name"
+  pytest $test_name
+done
+
+echo "RUNNING WITH BASE_URL SET"
+
+# Now re-run some tests with BASE_URL enabled
+# Re #65 - Ability to include a link back to the installation, in the notification.
+export BASE_URL="https://really-unique-domain.io"
+pytest tests/test_notification.py
+
+
+# Re-run with HIDE_REFERER set - could affect login
+export HIDE_REFERER=True
+pytest tests/test_access_control.py
+
+
+# Now for the selenium and playwright/browserless fetchers
+# Note - this is not UI functional tests - just checking that each one can fetch the content
+
+echo "TESTING WEBDRIVER FETCH > SELENIUM/WEBDRIVER..."
+docker run -d --name $$-test_selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome-debug:3.141.59
+# takes a while to spin up
+sleep 5
+export WEBDRIVER_URL=http://localhost:4444/wd/hub
+pytest tests/fetchers/test_content.py
+pytest tests/test_errorhandling.py
+unset WEBDRIVER_URL
+docker kill $$-test_selenium
+
+echo "TESTING WEBDRIVER FETCH > PLAYWRIGHT/BROWSERLESS..."
+# Not all platforms support playwright (not ARM/rPI), so it's not packaged in requirements.txt
+PLAYWRIGHT_VERSION=$(grep -i -E "RUN pip install.+" "$SCRIPT_DIR/../Dockerfile" | grep --only-matching -i -E "playwright[=><~+]+[0-9\.]+")
+echo "using $PLAYWRIGHT_VERSION"
+pip3 install "$PLAYWRIGHT_VERSION"
+docker run -d --name $$-test_browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
+# takes a while to spin up
+sleep 5
+export PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000
+pytest tests/fetchers/test_content.py
+pytest tests/test_errorhandling.py
+pytest tests/visualselector/test_fetch_data.py
+
+unset PLAYWRIGHT_DRIVER_URL
+docker kill $$-test_browserless
+
+# Test proxy list handling, starting two squids on different ports
+# Each squid adds a different header to the response, which is the main thing we test for.
+docker run -d --name $$-squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf -p 3128:3128 ubuntu/squid:4.13-21.10_edge
+docker run -d --name $$-squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf -p 3129:3128 ubuntu/squid:4.13-21.10_edge
+
+
+# So, basic HTTP as env var test
+export HTTP_PROXY=http://localhost:3128
+export HTTPS_PROXY=http://localhost:3128
+pytest tests/proxy_list/test_proxy.py
+docker logs $$-squid-one 2>/dev/null|grep one.changedetection.io
+if [ $? -ne 0 ]
+then
+  echo "Did not see a request to one.changedetection.io in the squid logs (while checking env vars HTTP_PROXY/HTTPS_PROXY)"
+fi
+unset HTTP_PROXY
+unset HTTPS_PROXY
+
+
+# 2nd test actually choose the preferred proxy from proxies.json
+cp tests/proxy_list/proxies.json-example ./test-datastore/proxies.json
+# Makes a watch use a preferred proxy
+pytest tests/proxy_list/test_multiple_proxy.py
+
+# Should be a request in the default "first" squid
+docker logs $$-squid-one 2>/dev/null|grep chosen.changedetection.io
+if [ $? -ne 0 ]
+then
+  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy)"
+fi
+
+# And one in the 'second' squid (user selects this as preferred)
+docker logs $$-squid-two 2>/dev/null|grep chosen.changedetection.io
+if [ $? -ne 0 ]
+then
+  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy)"
+fi
+
+# @todo - test system override proxy selection and watch defaults, setup a 3rd squid?
+docker kill $$-squid-one
+docker kill $$-squid-two
+
+

changedetectionio/run_basic_tests.sh

@@ -1,38 +0,0 @@
-#!/bin/bash
-
-
-# live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions
-# and I like to restart the server for each test (and have the test cleanup after each test)
-# merge request welcome :)
-
-
-# exit when any command fails
-set -e
-
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-find tests/test_*py -type f|while read test_name
-do
-  echo "TEST RUNNING $test_name"
-  pytest $test_name
-done
-
-echo "RUNNING WITH BASE_URL SET"
-
-# Now re-run some tests with BASE_URL enabled
-# Re #65 - Ability to include a link back to the installation, in the notification.
-export BASE_URL="https://really-unique-domain.io"
-pytest tests/test_notification.py
-
-
-# Re-run with HIDE_REFERER set - could affect login
-export HIDE_REFERER=True
-pytest tests/test_access_control.py
-
-# Re-run a few tests that will trigger brotli based storage
-export SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD=5
-pytest tests/test_access_control.py
-pytest tests/test_notification.py
-pytest tests/test_backend.py
-pytest tests/test_rss.py
-pytest tests/test_unique_lines.py

changedetectionio/run_proxy_tests.sh

@@ -1,61 +0,0 @@
-#!/bin/bash
-
-# exit when any command fails
-set -e
-
-# Test proxy list handling, starting two squids on different ports
-# Each squid adds a different header to the response, which is the main thing we test for.
-docker run --network changedet-network -d --name squid-one --hostname squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
-docker run --network changedet-network -d --name squid-two --hostname squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
-
-# Used for configuring a custom proxy URL via the UI
-docker run --network changedet-network -d \
-  --name squid-custom \
-  --hostname squid-custom \
-  --rm \
-  -v `pwd`/tests/proxy_list/squid-auth.conf:/etc/squid/conf.d/debian.conf \
-  -v `pwd`/tests/proxy_list/squid-passwords.txt:/etc/squid3/passwords \
-  ubuntu/squid:4.13-21.10_edge
-
-
-## 2nd test actually choose the preferred proxy from proxies.json
-
-docker run --network changedet-network \
-  -v `pwd`/tests/proxy_list/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
-  test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_multiple_proxy.py'
-
-
-## Should be a request in the default "first" squid
-docker logs squid-one 2>/dev/null|grep chosen.changedetection.io
-if [ $? -ne 0 ]
-then
-  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy - squid one)"
-  exit 1
-fi
-
-# And one in the 'second' squid (user selects this as preferred)
-docker logs squid-two 2>/dev/null|grep chosen.changedetection.io
-if [ $? -ne 0 ]
-then
-  echo "Did not see a request to chosen.changedetection.io in the squid logs (while checking preferred proxy - squid two)"
-  exit 1
-fi
-
-
-# Test the UI configurable proxies
-
-docker run --network changedet-network \
-  test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py'
-
-
-# Should see a request for one.changedetection.io in there
-docker logs squid-custom 2>/dev/null|grep "TCP_TUNNEL.200.*changedetection.io"
-if [ $? -ne 0 ]
-then
-  echo "Did not see a valid request to changedetection.io in the squid logs (while checking preferred proxy - squid two)"
-  exit 1
-fi
-
-docker kill squid-one squid-two squid-custom

changedetectionio/static/images/email.svg

@@ -1,37 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
-
-<svg
-   fill="#FFFFFF"
-   height="7.5005589"
-   width="11.248507"
-   version="1.1"
-   id="Layer_1"
-   viewBox="0 0 7.1975545 4.7993639"
-   xml:space="preserve"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg"><defs
-   id="defs19" />
-<g
-   id="g14"
-   transform="matrix(-0.01406065,0,0,0.01406065,7.1975543,-1.1990922)">
-	<g
-   id="g12">
-		<g
-   id="g10">
-			<path
-   d="M 468.373,85.28 H 45.333 C 21.227,85.28 0,105.76 0,129.014 V 383.2 c 0,23.147 21.227,43.413 45.333,43.413 h 422.933 c 23.68,0 43.627,-19.84 43.627,-43.413 V 129.014 C 512,105.334 492.053,85.28 468.373,85.28 Z m 0,320 H 45.333 c -12.373,0 -24,-10.773 -24,-22.08 V 129.014 c 0,-11.307 11.84,-22.4 24,-22.4 h 422.933 c 11.733,0 22.293,10.667 22.293,22.4 V 383.2 h 0.107 c 10e-4,11.734 -10.453,22.08 -22.293,22.08 z"
-   id="path2" />
-			<path
-   d="m 440.853,153.974 c -3.307,-4.907 -9.92,-6.187 -14.827,-2.987 L 256,264.48 85.973,151.094 c -4.907,-3.2 -11.52,-1.707 -14.72,3.2 -3.093,4.8 -1.813,11.307 2.88,14.507 l 176,117.333 c 3.627,2.347 8.213,2.347 11.84,0 l 176,-117.333 c 4.8,-3.201 6.187,-9.921 2.88,-14.827 z"
-   id="path4" />
-			<path
-   d="m 143.573,257.654 c -0.107,0.107 -0.32,0.213 -0.427,0.32 L 68.48,311.307 c -4.907,3.307 -6.187,9.92 -2.88,14.827 3.307,4.907 9.92,6.187 14.827,2.88 0.107,-0.107 0.32,-0.213 0.427,-0.32 l 74.667,-53.333 c 4.907,-3.307 6.187,-9.92 2.88,-14.827 -3.308,-4.907 -9.921,-6.187 -14.828,-2.88 z"
-   id="path6" />
-			<path
-   d="m 443.947,311.627 c -0.107,-0.107 -0.32,-0.213 -0.427,-0.32 l -74.667,-53.333 c -4.693,-3.52 -11.413,-2.56 -14.933,2.133 -3.52,4.693 -2.56,11.413 2.133,14.933 0.107,0.107 0.32,0.213 0.427,0.32 l 74.667,53.333 c 4.693,3.52 11.413,2.56 14.933,-2.133 3.52,-4.693 2.56,-11.413 -2.133,-14.933 z"
-   id="path8" />
-		</g>
-	</g>
-</g>
-</svg>

changedetectionio/static/images/generic-icon.svg

@@ -1,3 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Created with Inkscape (http://www.inkscape.org/) -->
-<svg width="61.649mm" height="61.649mm" version="1.1" viewBox="0 0 61.649 61.649" xml:space="preserve" xmlns="http://www.w3.org/2000/svg"><g transform="translate(66.269 -15.463)" fill="#3056d3"><g transform="matrix(1.423 0 0 1.423 101.16 69.23)" fill="#3056d3"><g transform="matrix(.8229 0 0 .8229 -23.378 -2.3935)" fill="#3056d3"><path d="m-88.248-43.007a26.323 26.323 0 0 0-26.323 26.323 26.323 26.323 0 0 0 26.323 26.323 26.323 26.323 0 0 0 26.323-26.323 26.323 26.323 0 0 0-26.323-26.323zm0 2.8417a23.482 23.482 0 0 1 23.482 23.482 23.482 23.482 0 0 1-23.482 23.482 23.482 23.482 0 0 1-23.482-23.482 23.482 23.482 0 0 1 23.482-23.482z"/><g transform="matrix(.26458 0 0 .26458 -115.65 -44.085)"><path d="m33.02 64.43c0.35-0.05 2.04-0.13 2.04-0.13h25.53s3.17 0.32 3.67 0.53c2.5 1.05 3.98 1.89 6.04 3.57 0.72 0.58 4.12 4.01 4.12 4.01l51.67 57.39s1.61 1.65 1.97 1.94c1.2 0.97 2.48 1.96 3.98 2.32 0.5 0.12 2.72 0.21 2.72 0.21h27.32l-8.83-9.04s-1.31-1.65-1.44-1.94c-0.45-0.93-0.59-2.59-0.13-3.51 0.35-0.69 1.46-1.87 2.23-1.98 1.03-0.14 2.12-0.39 3.02 0.14 0.33 0.2 1.64 1.32 1.64 1.32l17.49 17.49s1.35 1.09 1.6 1.6c0.17 0.34 0.29 0.82 0.15 1.18-0.17 0.42-1.42 1.63-1.42 1.63l-0.94 0.98-15.69 16.37s-1.44 1.4-1.79 1.67c-0.76 0.6-1.99 0.89-2.96 0.9-1.03 0-2.62-1.11-3.26-1.91-0.6-0.76-1.1-2.22-0.77-3.13 0.16-0.45 1.28-1.85 1.28-1.85l11.36-11.3-29.47-0.02-1.68 0.09s-4.16-0.66-5.26-1.03c-1.63-0.56-3.44-1.82-4.75-2.93-0.39-0.33-1.8-1.92-1.8-1.92l-51.7-59.28s-2-2.06-2.43-2.43c-1.37-1.17-2-1.62-3.76-2.34-0.44-0.18-3.45-0.55-3.45-0.55l-24.13-0.22s-2.23-0.15-2.61-0.22c-1.08-0.21-2.16-1.07-2.81-1.83-0.79-0.92-0.59-3.06 0.06-4.09 0.57-0.89 2.14-1.52 3.19-1.66z"/><path d="m86.1 109.7-17.13 19.65s-2 2.06-2.43 2.43c-1.37 1.17-2 1.62-3.76 2.34-0.44 0.18-3.45 0.55-3.45 0.55l-24.13 0.22s-2.23 0.15-2.61 0.22c-1.08 0.21-2.16 1.07-2.81 1.83-0.79 0.92-0.59 3.06 0.06 4.09 0.57 0.89 2.14 1.52 3.19 1.66 0.35 0.05 2.04 0.13 2.04 0.13h25.53s3.17-0.32 3.67-0.53c2.5-1.05 3.98-1.89 6.04-3.57 0.72-0.58 4.12-4.01 4.12-4.01l17.38-19.3z"/><path d="m177.81 67.6c-0.17-0.42-1.42-1.63-1.42-1.63l-0.94-0.98-15.69-16.37s-1.44-1.4-1.79-1.67c-0.76-0.6-1.99-0.89-2.96-0.9-1.03 0-2.62 1.11-3.26 1.91-0.6 0.76-1.1 2.22-0.77 3.13 0.16 0.45 1.28 1.85 1.28 1.85l11.36 11.3-29.47 0.02-1.68-0.09s-4.16 0.66-5.26 1.03c-1.63 0.56-3.44 1.82-4.75 2.93-0.39 0.33-1.8 1.92-1.8 1.92l-18.91 21.69 5.98 5.98 18.38-20.41s1.61-1.65 1.97-1.94c1.2-0.97 2.48-1.96 3.98-2.32 0.5-0.12 2.72-0.21 2.72-0.21h27.32l-8.83 9.04s-1.31 1.65-1.44 1.94c-0.45 0.93-0.59 2.59-0.13 3.51 0.35 0.69 1.46 1.87 2.23 1.98 1.03 0.14 2.12 0.39 3.02-0.14 0.33-0.2 1.64-1.32 1.64-1.32l17.49-17.49s1.35-1.09 1.6-1.6c0.17-0.34 0.29-0.82 0.15-1.18z"/></g></g></g></g></svg>

changedetectionio/static/images/pdf-icon.svg

@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg xmlns="http://www.w3.org/2000/svg" width="75.320129mm" height="92.604164mm" viewBox="0 0 75.320129 92.604164">
-  <g transform="translate(53.548057 -183.975276) scale(1.4843)">
-    <path fill="#ff2116" d="M-29.632812 123.94727c-3.551967 0-6.44336 2.89347-6.44336 6.44531v49.49804c0 3.55185 2.891393 6.44532 6.44336 6.44532H8.2167969c3.5519661 0 6.4433591-2.89335 6.4433591-6.44532v-40.70117s.101353-1.19181-.416015-2.35156c-.484969-1.08711-1.275391-1.84375-1.275391-1.84375a1.0584391 1.0584391 0 0 0-.0059-.008l-9.3906254-9.21094a1.0584391 1.0584391 0 0 0-.015625-.0156s-.8017392-.76344-1.9902344-1.27344c-1.39939552-.6005-2.8417968-.53711-2.8417968-.53711l.021484-.002z" color="#000" font-family="sans-serif" overflow="visible" paint-order="markers fill stroke" style="line-height:normal;font-variant-ligatures:normal;font-variant-position:normal;font-variant-caps:normal;font-variant-numeric:normal;font-variant-alternates:normal;font-feature-settings:normal;text-indent:0;text-align:start;text-decoration-line:none;text-decoration-style:solid;text-decoration-color:#000000;text-transform:none;text-orientation:mixed;white-space:normal;shape-padding:0;isolation:auto;mix-blend-mode:normal;solid-color:#000000;solid-opacity:1"/>
-    <path fill="#f5f5f5" d="M-29.632812 126.06445h28.3789058a1.0584391 1.0584391 0 0 0 .021484 0s1.13480448.011 1.96484378.36719c.79889772.34282 1.36536982.86176 1.36914062.86524.0000125.00001.00391.004.00391.004l9.3671868 9.18945s.564354.59582.837891 1.20899c.220779.49491.234375 1.40039.234375 1.40039a1.0584391 1.0584391 0 0 0-.002.0449v40.74609c0 2.41592-1.910258 4.32813-4.3261717 4.32813H-29.632812c-2.415914 0-4.326172-1.91209-4.326172-4.32813v-49.49804c0-2.41603 1.910258-4.32813 4.326172-4.32813z" color="#000" font-family="sans-serif" overflow="visible" paint-order="markers fill stroke" style="line-height:normal;font-variant-ligatures:normal;font-variant-position:normal;font-variant-caps:normal;font-variant-numeric:normal;font-variant-alternates:normal;font-feature-settings:normal;text-indent:0;text-align:start;text-decoration-line:none;text-decoration-style:solid;text-decoration-color:#000000;text-transform:none;text-orientation:mixed;white-space:normal;shape-padding:0;isolation:auto;mix-blend-mode:normal;solid-color:#000000;solid-opacity:1"/>
-    <path fill="#ff2116" d="M-23.40766 161.09299c-1.45669-1.45669.11934-3.45839 4.39648-5.58397l2.69124-1.33743 1.04845-2.29399c.57665-1.26169 1.43729-3.32036 1.91254-4.5748l.8641-2.28082-.59546-1.68793c-.73217-2.07547-.99326-5.19438-.52872-6.31588.62923-1.51909 2.69029-1.36323 3.50626.26515.63727 1.27176.57212 3.57488-.18329 6.47946l-.6193 2.38125.5455.92604c.30003.50932 1.1764 1.71867 1.9475 2.68743l1.44924 1.80272 1.8033728-.23533c5.72900399-.74758 7.6912472.523 7.6912472 2.34476 0 2.29921-4.4984914 2.48899-8.2760865-.16423-.8499666-.59698-1.4336605-1.19001-1.4336605-1.19001s-2.3665326.48178-3.531704.79583c-1.202707.32417-1.80274.52719-3.564509 1.12186 0 0-.61814.89767-1.02094 1.55026-1.49858 2.4279-3.24833 4.43998-4.49793 5.1723-1.3991.81993-2.86584.87582-3.60433.13733zm2.28605-.81668c.81883-.50607 2.47616-2.46625 3.62341-4.28553l.46449-.73658-2.11497 1.06339c-3.26655 1.64239-4.76093 3.19033-3.98386 4.12664.43653.52598.95874.48237 2.01093-.16792zm21.21809-5.95578c.80089-.56097.68463-1.69142-.22082-2.1472-.70466-.35471-1.2726074-.42759-3.1031574-.40057-1.1249.0767-2.9337647.3034-3.2403347.37237 0 0 .993716.68678 1.434896.93922.58731.33544 2.0145161.95811 3.0565161 1.27706 1.02785.31461 1.6224.28144 2.0729-.0409zm-8.53152-3.54594c-.4847-.50952-1.30889-1.57296-1.83152-2.3632-.68353-.89643-1.02629-1.52887-1.02629-1.52887s-.4996 1.60694-.90948 2.57394l-1.27876 3.16076-.37075.71695s1.971043-.64627 2.97389-.90822c1.0621668-.27744 3.21787-.70134 3.21787-.70134zm-2.74938-11.02573c.12363-1.0375.1761-2.07346-.15724-2.59587-.9246-1.01077-2.04057-.16787-1.85154 2.23517.0636.8084.26443 2.19033.53292 3.04209l.48817 1.54863.34358-1.16638c.18897-.64151.47882-2.02015.64411-3.06364z"/>
-    <path fill="#2c2c2c" d="M-20.930423 167.83862h2.364986q1.133514 0 1.840213.2169.706698.20991 1.189489.9446.482795.72769.482795 1.75625 0 .94459-.391832 1.6233-.391833.67871-1.056548.97958-.65772.30087-2.02913.30087h-.818651v3.72941h-1.581322zm1.581322 1.22447v3.33058h.783664q1.049552 0 1.44838-.39184.405826-.39183.405826-1.27345 0-.65772-.265887-1.06355-.265884-.41282-.587747-.50378-.314866-.098-1.000572-.098zm5.50664-1.22447h2.148082q1.560333 0 2.4909318.55276.9375993.55276 1.4133973 1.6443.482791 1.09153.482791 2.42096 0 1.3994-.4338151 2.49793-.4268149 1.09153-1.3154348 1.76324-.8816233.67172-2.5189212.67172h-2.267031zm1.581326 1.26645v7.018h.657715q1.378411 0 2.001144-.9516.6227329-.95858.6227329-2.5539 0-3.5125-2.6238769-3.5125zm6.4722254-1.26645h5.30372941v1.26645H-4.2075842v2.85478h2.9807225v1.26646h-2.9807225v4.16322h-1.5813254z" font-family="Franklin Gothic Medium Cond" letter-spacing="0" style="line-height:125%;-inkscape-font-specification:'Franklin Gothic Medium Cond'" word-spacing="4.26000023"/>
-  </g>
-</svg>

changedetectionio/static/images/price-tag-icon.svg

@@ -1,2 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg width="83.39" height="89.648" enable-background="new 0 0 122.406 122.881" version="1.1" viewBox="0 0 83.39 89.648" xml:space="preserve" xmlns="http://www.w3.org/2000/svg"><g transform="translate(5e-4 -33.234)"><path d="m44.239 42.946-39.111 39.896 34.908 34.91 39.09-39.876-1.149-34.931zm-0.91791 42.273c0.979-0.979 1.507-1.99 1.577-3.027 0.077-1.043-0.248-2.424-0.967-4.135-0.725-1.717-1.348-3.346-1.87-4.885s-0.814-3.014-0.897-4.432c-0.07-1.42 0.134-2.768 0.624-4.045 0.477-1.279 1.348-2.545 2.607-3.804 2.099-2.099 4.535-3.123 7.314-3.065 2.773 0.063 5.457 1.158 8.04 3.294l2.881 3.034c1.946 2.607 2.799 5.33 2.557 8.166-0.235 2.83-1.532 5.426-3.893 7.785l-6.296-6.297c1.291-1.291 2.035-2.531 2.238-3.727 0.191-1.197-0.165-2.252-1.081-3.168-0.821-0.82-1.717-1.195-2.69-1.139-0.967 0.064-1.908 0.547-2.817 1.457-0.922 0.922-1.393 1.914-1.412 2.977s0.306 2.416 0.973 4.064c0.661 1.652 1.24 3.25 1.736 4.801 0.496 1.553 0.782 3.035 0.858 4.445 0.076 1.426-0.127 2.787-0.591 4.104-0.477 1.316-1.336 2.596-2.588 3.848-2.125 2.125-4.522 3.186-7.212 3.18s-5.311-1.063-7.855-3.16l-3.747 3.746-2.964-2.965 3.766-3.764c-2.423-2.996-3.568-5.998-3.447-9.02 0.127-3.014 1.476-5.813 4.045-8.383l6.278 6.277c-1.412 1.412-2.175 2.799-2.277 4.16-0.108 1.367 0.414 2.627 1.571 3.783 0.839 0.84 1.755 1.26 2.741 1.242 0.985-0.017 1.92-0.47 2.798-1.347zm21.127-46.435h17.457c-0.0269 2.2368 0.69936 16.025 0.69936 16.025l0.785 23.858c0.019 0.609-0.221 1.164-0.619 1.564l5e-3 4e-3 -41.236 42.022c-0.82213 0.8378-2.175 0.83-3.004 0l-37.913-37.91c-0.83-0.83-0.83-2.176 0-3.006l41.236-42.021c0.39287-0.42671 1.502-0.53568 1.502-0.53568zm18.011 11.59c-59.392-29.687-29.696-14.843 0 0z"/></g></svg>

changedetectionio/static/js/browser-steps.js

@@ -114,11 +114,11 @@ $(document).ready(function () {
             e.preventDefault()
         });
 
-        // When the mouse moves we know which element it should be above
-        // mousedown will link that to the UI (select the right action, highlight etc)
         $('#browsersteps-selector-canvas').bind('mousedown', function (e) {
             // https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent
             e.preventDefault()
+            console.log(e);
+            console.log("current xpath in index is " + current_selected_i);
             last_click_xy = {'x': parseInt((1 / x_scale) * e.offsetX), 'y': parseInt((1 / y_scale) * e.offsetY)}
             process_selected(current_selected_i);
             current_selected_i = false;
@@ -132,7 +132,6 @@ $(document).ready(function () {
             }
         });
 
-        // Debounce and find the current most 'interesting' element we are hovering above
         $('#browsersteps-selector-canvas').bind('mousemove', function (e) {
             if (!xpath_data) {
                 return;
@@ -152,40 +151,41 @@ $(document).ready(function () {
             current_selected_i = false;
             // Reverse order - the most specific one should be deeper/"laster"
             // Basically, find the most 'deepest'
-            var possible_elements = [];
-            xpath_data['size_pos'].forEach(function (item, index) {
+            //$('#browsersteps-selector-canvas').css('cursor', 'pointer');
+            for (var i = xpath_data['size_pos'].length; i !== 0; i--) {
+                // draw all of them? let them choose somehow?
+                var sel = xpath_data['size_pos'][i - 1];
                 // If we are in a bounding-box
-                if (e.offsetY > item.top * y_scale && e.offsetY < item.top * y_scale + item.height * y_scale
+                if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
                     &&
-                    e.offsetX > item.left * y_scale && e.offsetX < item.left * y_scale + item.width * y_scale
+                    e.offsetX > sel.left * y_scale && e.offsetX < sel.left * y_scale + sel.width * y_scale
 
                 ) {
-                    // There could be many elements here, record them all and then we'll find out which is the most 'useful'
-                    // (input, textarea, button, A etc)
-                    if (item.width < xpath_data['browser_width']) {
-                        possible_elements.push(item);
+                    // Only highlight these interesting types
+                    if (1) {
+                        ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                        ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                        current_selected_i = i - 1;
+                        break;
+
+                        // find the smallest one at this x,y
+                        // does it mean sort the xpath list by size (w*h) i think so!
+                    } else {
+
+                        if (include_text_elements[0].checked === true) {
+                            // blue one with background instead?
+                            ctx.fillStyle = 'rgba(0,0,255, 0.1)';
+                            ctx.strokeStyle = 'rgba(0,0,200, 0.7)';
+                            $('#browsersteps-selector-canvas').css('cursor', 'grab');
+                            ctx.strokeRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                            ctx.fillRect(sel.left * x_scale, sel.top * y_scale, sel.width * x_scale, sel.height * y_scale);
+                            current_selected_i = i - 1;
+                            break;
+                        }
                     }
                 }
-            });
-
-            // Find the best one
-            if (possible_elements.length) {
-                possible_elements.forEach(function (item, index) {
-                  if (["a", "input", "textarea", "button"].includes(item['tagName'])) {
-                      current_selected_i = item;
-                  }
-                });
-
-                if (!current_selected_i) {
-                    current_selected_i = possible_elements[0];
-                }
-
-                sel = xpath_data['size_pos'][current_selected_i];
-                ctx.strokeRect(current_selected_i.left * x_scale, current_selected_i.top * y_scale, current_selected_i.width * x_scale, current_selected_i.height * y_scale);
-                ctx.fillRect(current_selected_i.left * x_scale, current_selected_i.top * y_scale, current_selected_i.width * x_scale, current_selected_i.height * y_scale);
             }
 
-
         }.debounce(10));
     });
 
@@ -195,16 +195,16 @@ $(document).ready(function () {
 
 
     // callback for clicking on an xpath on the canvas
-    function process_selected(selected_in_xpath_list) {
+    function process_selected(xpath_data_index) {
         found_something = false;
         var first_available = $("ul#browser_steps li.empty").first();
 
 
-        if (selected_in_xpath_list !== false) {
+        if (xpath_data_index !== false) {
             // Nothing focused, so fill in a new one
             // if inpt type button or <button>
             // from the top, find the next not used one and use it
-            var x = selected_in_xpath_list;
+            var x = xpath_data['size_pos'][xpath_data_index];
             console.log(x);
             if (x && first_available.length) {
                 // @todo will it let you click shit that has a layer ontop? probably not.
@@ -214,18 +214,26 @@ $(document).ready(function () {
                     $('input[placeholder="Value"]', first_available).addClass('ok').click().focus();
                     found_something = true;
                 } else {
-                    // There's no good way (that I know) to find if this
-                    // see https://stackoverflow.com/questions/446892/how-to-find-event-listeners-on-a-dom-node-in-javascript-or-in-debugging
-                    // https://codepen.io/azaslavsky/pen/DEJVWv
-
-                    // So we dont know if its really a clickable element or not :-(
-                    // Assume it is - then we dont fill the pages with unreliable "Click X,Y" selections
-                    // If you switch to "Click X,y" after an element here is setup, it will give the last co-ords anyway
-                    //if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
+                    if (x['isClickable'] || x['tagName'].startsWith('h') || x['tagName'] === 'a' || x['tagName'] === 'button' || x['tagtype'] === 'submit' || x['tagtype'] === 'checkbox' || x['tagtype'] === 'radio' || x['tagtype'] === 'li') {
                         $('select', first_available).val('Click element').change();
                         $('input[type=text]', first_available).first().val(x['xpath']);
                         found_something = true;
-                    //}
+                    }
+                }
+
+                first_available.xpath_data_index = xpath_data_index;
+
+                if (!found_something) {
+                    if (include_text_elements[0].checked === true) {
+                        // Suggest that we use as filter?
+                        // @todo filters should always be in the last steps, nothing non-filter after it
+                        found_something = true;
+                        ctx.strokeStyle = 'rgba(0,0,255, 0.9)';
+                        ctx.fillStyle = 'rgba(0,0,255, 0.1)';
+                        $('select', first_available).val('Extract text and use as filter').change();
+                        $('input[type=text]', first_available).first().val(x['xpath']);
+                        include_text_elements[0].checked = false;
+                    }
                 }
             }
         }
@@ -240,7 +248,7 @@ $(document).ready(function () {
 
     function start() {
         console.log("Starting browser-steps UI");
-        browsersteps_session_id = false;
+        browsersteps_session_id = Date.now();
         // @todo This setting of the first one should be done at the datalayer but wtforms doesnt wanna play nice
         $('#browser_steps >li:first-child').removeClass('empty');
         set_first_gotosite_disabled();
@@ -248,7 +256,7 @@ $(document).ready(function () {
         $('.clear,.remove', $('#browser_steps >li:first-child')).hide();
         $.ajax({
             type: "GET",
-            url: browser_steps_start_url,
+            url: browser_steps_sync_url + "&browsersteps_session_id=" + browsersteps_session_id,
             statusCode: {
                 400: function () {
                     // More than likely the CSRF token was lost when the server restarted
@@ -256,12 +264,12 @@ $(document).ready(function () {
                 }
             }
         }).done(function (data) {
+            xpath_data = data.xpath_data;
             $("#loading-status-text").fadeIn();
-            browsersteps_session_id = data.browsersteps_session_id;
             // This should trigger 'Goto site'
             console.log("Got startup response, requesting Goto-Site (first) step fake click");
             $('#browser_steps >li:first-child .apply').click();
-            browserless_seconds_remaining = 500;
+            browserless_seconds_remaining = data.browser_time_remaining;
             set_first_gotosite_disabled();
         }).fail(function (data) {
             console.log(data);
@@ -422,6 +430,7 @@ $(document).ready(function () {
             apply_buttons_disabled = false;
             $("#browsersteps-img").css('opacity', 1);
             $('ul#browser_steps li .control .apply').css('opacity', 1);
+            browserless_seconds_remaining = data.browser_time_remaining;
             $("#loading-status-text").hide();
             set_first_gotosite_disabled();
         }).fail(function (data) {

changedetectionio/static/js/notifications.js

@@ -26,6 +26,9 @@ $(document).ready(function() {
     data = {
         window_url : window.location.href,
         notification_urls : $('.notification-urls').val(),
+        notification_title : $('.notification-title').val(),
+        notification_body : $('.notification-body').val(),
+        notification_format : $('.notification-format').val(),
     }
     for (key in data) {
       if (!data[key].length) {

changedetectionio/static/js/tabs.js

@@ -12,7 +12,7 @@ window.addEventListener('hashchange', function () {
 var has_errors = document.querySelectorAll(".messages .error");
 if (!has_errors.length) {
     if (document.location.hash == "") {
-        location.replace(document.querySelector(".tabs ul li:first-child a").hash);
+        document.querySelector(".tabs ul li:first-child a").click();
     } else {
         set_active_tab();
     }

changedetectionio/static/js/toggle-theme.js

@@ -3,45 +3,22 @@
  * Toggles theme between light and dark mode.
  */
 $(document).ready(function () {
-  const button = document.getElementById("toggle-light-mode");
+  const button = document.getElementsByClassName("toggle-theme")[0];
 
   button.onclick = () => {
     const htmlElement = document.getElementsByTagName("html");
     const isDarkMode = htmlElement[0].dataset.darkmode === "true";
     htmlElement[0].dataset.darkmode = !isDarkMode;
-    setCookieValue(!isDarkMode);
+    if (isDarkMode) {
+      button.classList.remove("dark");
+      setCookieValue(false);
+    } else {
+      button.classList.add("dark");
+      setCookieValue(true);
+    }
   };
 
   const setCookieValue = (value) => {
     document.cookie = `css_dark_mode=${value};max-age=31536000;path=/`
   }
-
-  // Search input box behaviour
-    const toggle_search = document.getElementById("toggle-search");
-  const search_q = document.getElementById("search-q");
-  window.addEventListener('keydown', function (e) {
-
-    if (e.altKey == true && e.keyCode == 83)
-      search_q.classList.toggle('expanded');
-      search_q.focus();
-  });
-
-
-  search_q.onkeydown = (e) => {
-    var key = e.keyCode || e.which;
-    if (key === 13) {
-      document.searchForm.submit();
-    }
-  };
-  toggle_search.onclick = () => {
-    // Could be that they want to search something once text is in there
-    if (search_q.value.length) {
-      document.searchForm.submit();
-    } else {
-      // If not..
-      search_q.classList.toggle('expanded');
-      search_q.focus();
-    }
-  };
-
 });

changedetectionio/static/js/visual-selector.js

@@ -1,5 +1,4 @@
-// Copyright (C) 2021 Leigh Morresi (dgtlmoon@gmail.com)
-// All rights reserved.
+// Horrible proof of concept code :)
 // yes - this is really a hack, if you are a front-ender and want to help, please get in touch!
 
 $(document).ready(function () {
@@ -61,12 +60,7 @@ $(document).ready(function () {
     function bootstrap_visualselector() {
         if (1) {
             // bootstrap it, this will trigger everything else
-            $("img#selector-background").on("error", function () {
-                $('.fetching-update-notice').html("<strong>Ooops!</strong> The VisualSelector tool needs atleast one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.");
-                $('.fetching-update-notice').css('color','#bb0000');
-                $('#selector-current-xpath').hide();
-                $('#clear-selector').hide();
-            }).bind('load', function () {
+            $("img#selector-background").bind('load', function () {
                 console.log("Loaded background...");
                 c = document.getElementById("selector-canvas");
                 // greyed out fill context
@@ -84,11 +78,10 @@ $(document).ready(function () {
             }).attr("src", screenshot_url);
         }
         // Tell visualSelector that the image should update
-        var s = $("img#selector-background").attr('src') + "?" + new Date().getTime();
-        $("img#selector-background").attr('src', s)
+        var s = $("img#selector-background").attr('src')+"?"+ new Date().getTime();
+        $("img#selector-background").attr('src',s)
     }
 
-    // This is fired once the img src is loaded in bootstrap_visualselector()
     function fetch_data() {
         // Image is ready
         $('.fetching-update-notice').html("Fetching element data..");
@@ -105,8 +98,7 @@ $(document).ready(function () {
             reflow_selector();
             $('.fetching-update-notice').fadeOut();
         });
-
-    }
+    };
 
 
     function set_scale() {
@@ -185,10 +177,9 @@ $(document).ready(function () {
             // Basically, find the most 'deepest'
             var found = 0;
             ctx.fillStyle = 'rgba(205,0,0,0.35)';
-            // Will be sorted by smallest width*height first
-            for (var i = 0; i <= selector_data['size_pos'].length; i++) {
+            for (var i = selector_data['size_pos'].length; i !== 0; i--) {
                 // draw all of them? let them choose somehow?
-                var sel = selector_data['size_pos'][i];
+                var sel = selector_data['size_pos'][i - 1];
                 // If we are in a bounding-box
                 if (e.offsetY > sel.top * y_scale && e.offsetY < sel.top * y_scale + sel.height * y_scale
                     &&
@@ -204,7 +195,7 @@ $(document).ready(function () {
                     // no need to keep digging
                     // @todo or, O to go out/up, I to go in
                     // or double click to go up/out the selector?
-                    current_selected_i = i;
+                    current_selected_i = i - 1;
                     found += 1;
                     break;
                 }

changedetectionio/static/js/watch-overview.js

@@ -1,45 +1,31 @@
 $(function () {
-    // Remove unviewed status when normally clicked
-    $('.diff-link').click(function () {
-        $(this).closest('.unviewed').removeClass('unviewed');
-    });
+  // Remove unviewed status when normally clicked
+  $('.diff-link').click(function () {
+    $(this).closest('.unviewed').removeClass('unviewed');
+  });
 
-    $("#checkbox-assign-tag").click(function (e) {
-        $('#op_extradata').val(prompt("Enter a tag name"));
-    });
 
-    $('.with-share-link > *').click(function () {
-        $("#copied-clipboard").remove();
+  $('.with-share-link > *').click(function () {
+      $("#copied-clipboard").remove();
 
-        var range = document.createRange();
-        var n = $("#share-link")[0];
-        range.selectNode(n);
-        window.getSelection().removeAllRanges();
-        window.getSelection().addRange(range);
-        document.execCommand("copy");
-        window.getSelection().removeAllRanges();
+      var range = document.createRange();
+      var n=$("#share-link")[0];
+      range.selectNode(n);
+      window.getSelection().removeAllRanges();
+      window.getSelection().addRange(range);
+      document.execCommand("copy");
+      window.getSelection().removeAllRanges();
 
-        $('.with-share-link').append('<span style="font-size: 80%; color: #fff;" id="copied-clipboard">Copied to clipboard</span>');
-        $("#copied-clipboard").fadeOut(2500, function () {
-            $(this).remove();
-        });
-    });
-
-    $(".watch-table tr").click(function (event) {
-        var tagName = event.target.tagName.toLowerCase();
-        if (tagName === 'tr' || tagName === 'td') {
-            var x = $('input[type=checkbox]', this);
-            if (x) {
-                $(x).click();
-            }
-        }
-    });
+      $('.with-share-link').append('<span style="font-size: 80%; color: #fff;" id="copied-clipboard">Copied to clipboard</span>');
+      $("#copied-clipboard").fadeOut(2500, function() {
+       $(this).remove();
+      });
+  });
 
     // checkboxes - check all
     $("#check-all").click(function (e) {
         $('input[type=checkbox]').not(this).prop('checked', this.checked);
     });
-
     // checkboxes - show/hide buttons
     $("input[type=checkbox]").click(function (e) {
         if ($('input[type=checkbox]:checked').length) {

changedetectionio/static/js/watch-settings.js

@@ -1,7 +1,7 @@
-$(document).ready(function () {
+$(document).ready(function() {
     function toggle() {
         if ($('input[name="fetch_backend"]:checked').val() == 'html_webdriver') {
-            if (playwright_enabled) {
+            if(playwright_enabled) {
                 // playwright supports headers, so hide everything else
                 // See #664
                 $('#requests-override-options #request-method').hide();
@@ -14,14 +14,9 @@ $(document).ready(function () {
                 $('#requests-override-options').hide();
             }
 
+
             $('#webdriver-override-options').show();
 
-        } else if ($('input[name="fetch_backend"]:checked').val() == 'system') {
-            $('#requests-override-options #request-method').hide();
-            $('#requests-override-options #request-body').hide();
-            $('#ignore-status-codes-option').hide();
-            $('#requests-override-options').hide();
-            $('#webdriver-override-options').hide();
         } else {
 
             $('#requests-override-options').show();

changedetectionio/static/styles/.dockerignore

@@ -1,3 +0,0 @@
-node_modules
-package-lock.json
-

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -1,17 +0,0 @@
-ul#requests-extra_proxies {
-  list-style: none;
-  /* tidy up the table to look more "inline" */
-  li {
-    > label {
-      display: none;
-    }
-
-  }
-  /* each proxy entry is a `table` */
-  table {
-    tr {
-      display: inline;
-    }
-  }
-}
-

changedetectionio/static/styles/scss/parts/_pagination.scss

@@ -1,37 +0,0 @@
-.pagination-page-info {
-  color: #fff;
-  font-size: 0.85rem;
-  text-transform: capitalize;
-}
-
-.pagination.menu {
-  > * {
-    display: inline-block;
-  }
-
-  li {
-    display: inline-block;
-  }
-
-  a {
-    padding: 0.65rem;
-    margin: 3px;
-    border: none;
-    background: #444;
-    border-radius: 2px;
-    color: var(--color-text-button);
-    &.disabled {
-      display: none;
-    }
-    &.active {
-      font-weight: bold;
-      background: #888;
-    }
-
-    &:hover {
-      background: #999;
-    }
-  }
-
-
-}

changedetectionio/static/styles/scss/styles.scss

@@ -2,12 +2,10 @@
  * -- BASE STYLES --
  */
 
-@import "parts/_arrows";
-@import "parts/_browser-steps";
-@import "parts/_extra_proxies";
-@import "parts/_pagination";
-@import "parts/_spinners";
 @import "parts/_variables";
+@import "parts/_spinners";
+@import "parts/_browser-steps";
+@import "parts/_arrows";
 
 body {
   color: var(--color-text);
@@ -24,13 +22,6 @@ body {
   width: 1px;
 }
 
-// Row icons like chrome, pdf, share, etc
-.status-icon {
-  display: inline-block;
-  height: 1rem;
-  vertical-align: middle;
-}
-
 .pure-table-even {
   background: var(--color-background);
 }
@@ -54,47 +45,8 @@ a.github-link {
   }
 }
 
-#toggle-light-mode {
-  width: 3rem;
-  .icon-dark {
-    display: none;
-  }
-
-  &.dark {
-    .icon-light {
-      display: none;
-    }
-
-    .icon-dark {
-      display: block;
-    }
-  }
-}
-
-#toggle-search {
-  width: 2rem;
-}
-
-#search-q {
-  opacity: 0;
-  -webkit-transition: all .9s ease;
-  -moz-transition: all .9s ease;
-  transition: all .9s ease;
-  width: 0;
-  display: none;
-  &.expanded {
-    width: auto;
-    display: inline-block;
-
-    opacity: 1;
-  }
-}
-#search-result-info {
-  color: #fff;
-}
-
-button.toggle-button {
-  vertical-align: middle;
+button.toggle-theme {
+  width: 4rem;
   background: transparent;
   border: none;
   cursor: pointer;
@@ -113,7 +65,19 @@ button.toggle-button {
     display: block;
   }
 
+  .icon-dark {
+    display: none;
+  }
 
+  &.dark {
+    .icon-light {
+      display: none;
+    }
+
+    .icon-dark {
+      display: block;
+    }
+  }
 }
 
 .pure-menu-horizontal {
@@ -269,10 +233,6 @@ body:before {
   font-size: 85%;
 }
 
-.button-xsmall {
-  font-size: 70%;
-}
-
 .fetch-error {
   padding-top: 1em;
   font-size: 80%;
@@ -921,21 +881,6 @@ body.full-width {
       font-size: .875em;
     }
   }
-  .text-filtering {
-    h3 {
-      margin-top: 0;
-    }
-    border: 1px solid #ccc;
-    padding: 1rem;
-    border-radius: 5px;
-    margin-bottom: 1rem;
-    fieldset:last-of-type {
-      padding-bottom: 0;
-      .pure-control-group {
-        padding-bottom: 0;
-      }
-    }
-  }
 }
 
 ul {
@@ -1080,41 +1025,4 @@ ul {
     padding: 3px;
     background-color: var(--color-background-button-green);
   }
-
-  font-weight: bold;
-  font-style: italic;
-}
-
-.price-follow-tag-icon {
-  display: inline-block;
-  height: 0.8rem;
-  vertical-align: middle;
-}
-
-
-#quick-watch-processor-type {
-  color: #fff;
-  ul {
-    padding: 0.3rem;
-
-    li {
-      list-style: none;
-      font-size: 0.8rem;
-    }
-  }
-
-}
-
-.restock-label {
-  &.in-stock {
-    background-color: var(--color-background-button-green);
-    color: #fff;
-  }
-  &.not-in-stock {
-    background-color: var(--color-background-button-cancel);
-    color: #777;
-  }
-  padding: 3px;
-  border-radius: 3px;
-  white-space: nowrap;
 }

changedetectionio/static/styles/styles.css

@@ -1,165 +1,6 @@
 /*
  * -- BASE STYLES --
  */
-.arrow {
-  border: solid #1b98f8;
-  border-width: 0 2px 2px 0;
-  display: inline-block;
-  padding: 3px; }
-  .arrow.right {
-    transform: rotate(-45deg);
-    -webkit-transform: rotate(-45deg); }
-  .arrow.left {
-    transform: rotate(135deg);
-    -webkit-transform: rotate(135deg); }
-  .arrow.up, .arrow.asc {
-    transform: rotate(-135deg);
-    -webkit-transform: rotate(-135deg); }
-  .arrow.down, .arrow.desc {
-    transform: rotate(45deg);
-    -webkit-transform: rotate(45deg); }
-
-#browser_steps {
-  /* convert rows to horizontal cells */ }
-  #browser_steps th {
-    display: none; }
-  #browser_steps li {
-    list-style: decimal;
-    padding: 5px; }
-    #browser_steps li:not(:first-child):hover {
-      opacity: 1.0; }
-    #browser_steps li .control {
-      padding-left: 5px;
-      padding-right: 5px; }
-      #browser_steps li .control a {
-        font-size: 70%; }
-    #browser_steps li.empty {
-      padding: 0px;
-      opacity: 0.35; }
-      #browser_steps li.empty .control {
-        display: none; }
-    #browser_steps li:hover {
-      background: #eee; }
-    #browser_steps li > label {
-      display: none; }
-
-#browser-steps-fieldlist {
-  height: 100%;
-  overflow-y: scroll; }
-
-#browser-steps .flex-wrapper {
-  display: flex;
-  flex-flow: row;
-  height: 600px;
-  /*@todo make this dynamic */ }
-
-/*  this is duplicate :( */
-#browsersteps-selector-wrapper {
-  height: 100%;
-  width: 100%;
-  overflow-y: scroll;
-  position: relative;
-  /* nice tall skinny one */ }
-  #browsersteps-selector-wrapper > img {
-    position: absolute;
-    max-width: 100%; }
-  #browsersteps-selector-wrapper > canvas {
-    position: relative;
-    max-width: 100%; }
-    #browsersteps-selector-wrapper > canvas:hover {
-      cursor: pointer; }
-  #browsersteps-selector-wrapper .loader {
-    position: absolute;
-    left: 50%;
-    top: 50%;
-    transform: translate(-50%, -50%);
-    margin-left: -40px;
-    z-index: 100;
-    max-width: 350px;
-    text-align: center; }
-  #browsersteps-selector-wrapper .spinner, #browsersteps-selector-wrapper .spinner:after {
-    width: 80px;
-    height: 80px;
-    font-size: 3px; }
-  #browsersteps-selector-wrapper #browsersteps-click-start {
-    color: var(--color-grey-400); }
-    #browsersteps-selector-wrapper #browsersteps-click-start:hover {
-      cursor: pointer; }
-
-ul#requests-extra_proxies {
-  list-style: none;
-  /* tidy up the table to look more "inline" */
-  /* each proxy entry is a `table` */ }
-  ul#requests-extra_proxies li > label {
-    display: none; }
-  ul#requests-extra_proxies table tr {
-    display: inline; }
-
-.pagination-page-info {
-  color: #fff;
-  font-size: 0.85rem;
-  text-transform: capitalize; }
-
-.pagination.menu > * {
-  display: inline-block; }
-
-.pagination.menu li {
-  display: inline-block; }
-
-.pagination.menu a {
-  padding: 0.65rem;
-  margin: 3px;
-  border: none;
-  background: #444;
-  border-radius: 2px;
-  color: var(--color-text-button); }
-  .pagination.menu a.disabled {
-    display: none; }
-  .pagination.menu a.active {
-    font-weight: bold;
-    background: #888; }
-  .pagination.menu a:hover {
-    background: #999; }
-
-/* spinner */
-.spinner,
-.spinner:after {
-  border-radius: 50%;
-  width: 10px;
-  height: 10px; }
-
-.spinner {
-  margin: 0px auto;
-  font-size: 3px;
-  vertical-align: middle;
-  display: inline-block;
-  text-indent: -9999em;
-  border-top: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-right: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
-  border-left: 1.1em solid #2668ed;
-  -webkit-transform: translateZ(0);
-  -ms-transform: translateZ(0);
-  transform: translateZ(0);
-  -webkit-animation: load8 1.1s infinite linear;
-  animation: load8 1.1s infinite linear; }
-
-@-webkit-keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg); }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg); } }
-
-@keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg); }
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg); } }
-
 /**
  * CSS custom properties (aka variables).
  */
@@ -283,10 +124,6 @@ html[data-darkmode="true"] {
   --color-icon-github-hover: var(--color-grey-700);
   --color-watch-table-error: var(--color-light-red);
   --color-watch-table-row-text: var(--color-grey-800); }
-  html[data-darkmode="true"] #toggle-light-mode .icon-light {
-    display: none; }
-  html[data-darkmode="true"] #toggle-light-mode .icon-dark {
-    display: block; }
   html[data-darkmode="true"] .icon-spread {
     filter: hue-rotate(-10deg) brightness(1.5); }
   html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
@@ -301,6 +138,130 @@ html[data-darkmode="true"] {
     html[data-darkmode="true"] .watch-table .unviewed.error {
       color: var(--color-watch-table-error); }
 
+/* spinner */
+.spinner,
+.spinner:after {
+  border-radius: 50%;
+  width: 10px;
+  height: 10px; }
+
+.spinner {
+  margin: 0px auto;
+  font-size: 3px;
+  vertical-align: middle;
+  display: inline-block;
+  text-indent: -9999em;
+  border-top: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-right: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-bottom: 1.1em solid rgba(38, 104, 237, 0.2);
+  border-left: 1.1em solid #2668ed;
+  -webkit-transform: translateZ(0);
+  -ms-transform: translateZ(0);
+  transform: translateZ(0);
+  -webkit-animation: load8 1.1s infinite linear;
+  animation: load8 1.1s infinite linear; }
+
+@-webkit-keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg); }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg); } }
+
+@keyframes load8 {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg); }
+  100% {
+    -webkit-transform: rotate(360deg);
+    transform: rotate(360deg); } }
+
+#browser_steps {
+  /* convert rows to horizontal cells */ }
+  #browser_steps th {
+    display: none; }
+  #browser_steps li {
+    list-style: decimal;
+    padding: 5px; }
+    #browser_steps li:not(:first-child):hover {
+      opacity: 1.0; }
+    #browser_steps li .control {
+      padding-left: 5px;
+      padding-right: 5px; }
+      #browser_steps li .control a {
+        font-size: 70%; }
+    #browser_steps li.empty {
+      padding: 0px;
+      opacity: 0.35; }
+      #browser_steps li.empty .control {
+        display: none; }
+    #browser_steps li:hover {
+      background: #eee; }
+    #browser_steps li > label {
+      display: none; }
+
+#browser-steps-fieldlist {
+  height: 100%;
+  overflow-y: scroll; }
+
+#browser-steps .flex-wrapper {
+  display: flex;
+  flex-flow: row;
+  height: 600px;
+  /*@todo make this dynamic */ }
+
+/*  this is duplicate :( */
+#browsersteps-selector-wrapper {
+  height: 100%;
+  width: 100%;
+  overflow-y: scroll;
+  position: relative;
+  /* nice tall skinny one */ }
+  #browsersteps-selector-wrapper > img {
+    position: absolute;
+    max-width: 100%; }
+  #browsersteps-selector-wrapper > canvas {
+    position: relative;
+    max-width: 100%; }
+    #browsersteps-selector-wrapper > canvas:hover {
+      cursor: pointer; }
+  #browsersteps-selector-wrapper .loader {
+    position: absolute;
+    left: 50%;
+    top: 50%;
+    transform: translate(-50%, -50%);
+    margin-left: -40px;
+    z-index: 100;
+    max-width: 350px;
+    text-align: center; }
+  #browsersteps-selector-wrapper .spinner, #browsersteps-selector-wrapper .spinner:after {
+    width: 80px;
+    height: 80px;
+    font-size: 3px; }
+  #browsersteps-selector-wrapper #browsersteps-click-start {
+    color: var(--color-grey-400); }
+    #browsersteps-selector-wrapper #browsersteps-click-start:hover {
+      cursor: pointer; }
+
+.arrow {
+  border: solid #1b98f8;
+  border-width: 0 2px 2px 0;
+  display: inline-block;
+  padding: 3px; }
+  .arrow.right {
+    transform: rotate(-45deg);
+    -webkit-transform: rotate(-45deg); }
+  .arrow.left {
+    transform: rotate(135deg);
+    -webkit-transform: rotate(135deg); }
+  .arrow.up, .arrow.asc {
+    transform: rotate(-135deg);
+    -webkit-transform: rotate(-135deg); }
+  .arrow.down, .arrow.desc {
+    transform: rotate(45deg);
+    -webkit-transform: rotate(45deg); }
+
 body {
   color: var(--color-text);
   background: var(--color-background-page); }
@@ -314,11 +275,6 @@ body {
   white-space: nowrap;
   width: 1px; }
 
-.status-icon {
-  display: inline-block;
-  height: 1rem;
-  vertical-align: middle; }
-
 .pure-table-even {
   background: var(--color-background); }
 
@@ -335,40 +291,23 @@ a.github-link {
   a.github-link:hover {
     color: var(--color-icon-github-hover); }
 
-#toggle-light-mode {
-  width: 3rem; }
-  #toggle-light-mode .icon-dark {
-    display: none; }
-
-#toggle-search {
-  width: 2rem; }
-
-#search-q {
-  opacity: 0;
-  -webkit-transition: all .9s ease;
-  -moz-transition: all .9s ease;
-  transition: all .9s ease;
-  width: 0;
-  display: none; }
-  #search-q.expanded {
-    width: auto;
-    display: inline-block;
-    opacity: 1; }
-
-#search-result-info {
-  color: #fff; }
-
-button.toggle-button {
-  vertical-align: middle;
+button.toggle-theme {
+  width: 4rem;
   background: transparent;
   border: none;
   cursor: pointer;
   color: var(--color-icon-github); }
-  button.toggle-button:hover {
+  button.toggle-theme:hover {
     color: var(--color-icon-github-hover); }
-  button.toggle-button svg {
+  button.toggle-theme svg {
     fill: currentColor; }
-  button.toggle-button .icon-light {
+  button.toggle-theme .icon-light {
+    display: block; }
+  button.toggle-theme .icon-dark {
+    display: none; }
+  button.toggle-theme.dark .icon-light {
+    display: none; }
+  button.toggle-theme.dark .icon-dark {
     display: block; }
 
 .pure-menu-horizontal {
@@ -479,9 +418,6 @@ body:before {
 .button-small {
   font-size: 85%; }
 
-.button-xsmall {
-  font-size: 70%; }
-
 .fetch-error {
   padding-top: 1em;
   font-size: 80%;
@@ -919,17 +855,6 @@ body.full-width .edit-form {
     color: var(--color-text-input-description); }
     .edit-form .pure-form-message-inline code {
       font-size: .875em; }
-  .edit-form .text-filtering {
-    border: 1px solid #ccc;
-    padding: 1rem;
-    border-radius: 5px;
-    margin-bottom: 1rem; }
-    .edit-form .text-filtering h3 {
-      margin-top: 0; }
-    .edit-form .text-filtering fieldset:last-of-type {
-      padding-bottom: 0; }
-      .edit-form .text-filtering fieldset:last-of-type .pure-control-group {
-        padding-bottom: 0; }
 
 ul {
   padding-left: 1em;
@@ -1029,34 +954,7 @@ ul {
   border-radius: 3px;
   white-space: nowrap; }
 
-.ldjson-price-track-offer {
-  font-weight: bold;
-  font-style: italic; }
-  .ldjson-price-track-offer a.pure-button {
-    border-radius: 3px;
-    padding: 3px;
-    background-color: var(--color-background-button-green); }
-
-.price-follow-tag-icon {
-  display: inline-block;
-  height: 0.8rem;
-  vertical-align: middle; }
-
-#quick-watch-processor-type {
-  color: #fff; }
-  #quick-watch-processor-type ul {
-    padding: 0.3rem; }
-    #quick-watch-processor-type ul li {
-      list-style: none;
-      font-size: 0.8rem; }
-
-.restock-label {
-  padding: 3px;
+.ldjson-price-track-offer a.pure-button {
   border-radius: 3px;
-  white-space: nowrap; }
-  .restock-label.in-stock {
-    background-color: var(--color-background-button-green);
-    color: #fff; }
-  .restock-label.not-in-stock {
-    background-color: var(--color-background-button-cancel);
-    color: #777; }
+  padding: 3px;
+  background-color: var(--color-background-button-green); }

changedetectionio/store.py

@@ -1,22 +1,20 @@
 from flask import (
     flash
 )
-
-from . model import App, Watch
-from copy import deepcopy, copy
-from os import path, unlink
-from threading import Lock
 import json
 import logging
 import os
-import re
-import requests
-import secrets
 import threading
 import time
 import uuid as uuid_builder
+from copy import deepcopy
+from os import path, unlink
+from threading import Lock
+import re
+import requests
+import secrets
 
-dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
+from . model import App, Watch
 
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
@@ -38,6 +36,7 @@ class ChangeDetectionStore:
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
         self.needs_write = False
+        self.proxy_list = None
         self.start_time = time.time()
         self.stop_thread = False
         # Base definition for all watchers
@@ -79,10 +78,10 @@ class ChangeDetectionStore:
                     self.__data['watching'][uuid] = Watch.model(datastore_path=self.datastore_path, default=watch)
                     print("Watching:", uuid, self.__data['watching'][uuid]['url'])
 
-        # First time ran, Create the datastore.
-        except (FileNotFoundError):
+        # First time ran, doesnt exist.
+        except (FileNotFoundError, json.decoder.JSONDecodeError):
             if include_default_watches:
-                print("No JSON DB found at {}, creating JSON store at {}".format(self.json_store_path, self.datastore_path))
+                print("Creating JSON store at", self.datastore_path)
                 self.add_watch(url='https://news.ycombinator.com/',
                                tag='Tech news',
                                extras={'fetch_backend': 'html_requests'})
@@ -90,10 +89,8 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://changedetection.io/CHANGELOG.txt',
                                tag='changedetection.io',
                                extras={'fetch_backend': 'html_requests'})
-        self.__data['version_tag'] = version_tag
 
-        # Just to test that proxies.json if it exists, doesnt throw a parsing error on startup
-        test_list = self.proxy_list
+        self.__data['version_tag'] = version_tag
 
         # Helper to remove password protection
         password_reset_lockfile = "{}/removepassword.lock".format(self.datastore_path)
@@ -119,6 +116,11 @@ class ChangeDetectionStore:
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['api_access_token'] = secret
 
+        # Proxy list support - available as a selection in settings when text file is imported
+        proxy_list_file = "{}/proxies.json".format(self.datastore_path)
+        if path.isfile(proxy_list_file):
+            self.import_proxy_list(proxy_list_file)
+
         # Bump the update version by running updates
         self.run_updates()
 
@@ -173,6 +175,14 @@ class ChangeDetectionStore:
 
     @property
     def data(self):
+        has_unviewed = False
+        for uuid, watch in self.__data['watching'].items():
+            # #106 - Be sure this is None on empty string, False, None, etc
+            # Default var for fetch_backend
+            # @todo this may not be needed anymore, or could be easily removed
+            if not self.__data['watching'][uuid]['fetch_backend']:
+                self.__data['watching'][uuid]['fetch_backend'] = self.__data['settings']['application']['fetch_backend']
+
         # Re #152, Return env base_url if not overriden, @todo also prefer the proxy pass url
         env_base_url = os.getenv('BASE_URL','')
         if not self.__data['settings']['application']['base_url']:
@@ -180,34 +190,51 @@ class ChangeDetectionStore:
 
         return self.__data
 
+    def get_all_tags(self):
+        tags = []
+        for uuid, watch in self.data['watching'].items():
+            if watch['tag'] is None:
+                continue
+            # Support for comma separated list of tags.
+            for tag in watch['tag'].split(','):
+                tag = tag.strip()
+                if tag not in tags:
+                    tags.append(tag)
+
+        tags.sort()
+        return tags
+
+    def unlink_history_file(self, path):
+        try:
+            unlink(path)
+        except (FileNotFoundError, IOError):
+            pass
+
     # Delete a single watch by UUID
     def delete(self, uuid):
-        import pathlib
-        import shutil
-
         with self.lock:
             if uuid == 'all':
                 self.__data['watching'] = {}
 
                 # GitHub #30 also delete history records
                 for uuid in self.data['watching']:
-                    path = pathlib.Path(os.path.join(self.datastore_path, uuid))
-                    if os.path.exists(path):
-                        shutil.rmtree(path)
+                    for path in self.data['watching'][uuid].history.values():
+                        self.unlink_history_file(path)
 
             else:
-                path = pathlib.Path(os.path.join(self.datastore_path, uuid))
-                if os.path.exists(path):
-                    shutil.rmtree(path)
+                for path in self.data['watching'][uuid].history.values():
+                    self.unlink_history_file(path)
+
                 del self.data['watching'][uuid]
 
-        self.needs_write_urgent = True
+            self.needs_write_urgent = True
 
     # Clone a watch by UUID
     def clone(self, uuid):
-        url = self.data['watching'][uuid].get('url')
+        url = self.data['watching'][uuid]['url']
+        tag = self.data['watching'][uuid]['tag']
         extras = self.data['watching'][uuid]
-        new_uuid = self.add_watch(url=url, extras=extras)
+        new_uuid = self.add_watch(url=url, tag=tag, extras=extras)
         return new_uuid
 
     def url_exists(self, url):
@@ -242,14 +269,16 @@ class ChangeDetectionStore:
 
         self.needs_write_urgent = True
 
-    def add_watch(self, url, tag='', extras=None, tag_uuids=None, write_to_disk_now=True):
+    def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
 
         if extras is None:
             extras = {}
+        # should always be str
+        if tag is None or not tag:
+            tag = ''
 
         # Incase these are copied across, assume it's a reference and deepcopy()
         apply_extras = deepcopy(extras)
-        apply_extras['tags'] = [] if not apply_extras.get('tags') else apply_extras.get('tags')
 
         # Was it a share link? try to fetch the data
         if (url.startswith("https://changedetection.io/share/")):
@@ -273,10 +302,8 @@ class ChangeDetectionStore:
                     'method',
                     'paused',
                     'previous_md5',
-                    'processor',
                     'subtractive_selectors',
                     'tag',
-                    'tags',
                     'text_should_not_be_present',
                     'title',
                     'trigger_text',
@@ -294,41 +321,26 @@ class ChangeDetectionStore:
                 logging.error("Error fetching metadata for shared watch link", url, str(e))
                 flash("Error fetching metadata for {}".format(url), 'error')
                 return False
-        from .model.Watch import is_safe_url
-        if not is_safe_url(url):
-            flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error')
-            return None
 
-        if tag and type(tag) == str:
-            # Then it's probably a string of the actual tag by name, split and add it
-            for t in tag.split(','):
-                # for each stripped tag, add tag as UUID
-                for a_t in t.split(','):
-                    tag_uuid = self.add_tag(a_t)
-                    apply_extras['tags'].append(tag_uuid)
+        with self.lock:
 
-        # Or if UUIDs given directly
-        if tag_uuids:
-            apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids))
+            # #Re 569
+            new_watch = Watch.model(datastore_path=self.datastore_path, default={
+                'url': url,
+                'tag': tag
+            })
 
-        # Make any uuids unique
-        if apply_extras.get('tags'):
-            apply_extras['tags'] = list(set(apply_extras.get('tags')))
+            new_uuid = new_watch['uuid']
+            logging.debug("Added URL {} - {}".format(url, new_uuid))
 
-        new_watch = Watch.model(datastore_path=self.datastore_path, url=url)
+            for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
+                if k in apply_extras:
+                    del apply_extras[k]
 
-        new_uuid = new_watch.get('uuid')
-
-        logging.debug("Added URL {} - {}".format(url, new_uuid))
-
-        for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
-            if k in apply_extras:
-                del apply_extras[k]
-
-        new_watch.update(apply_extras)
-        new_watch.ensure_data_dir_exists()
-        self.__data['watching'][new_uuid] = new_watch
+            new_watch.update(apply_extras)
+            self.__data['watching'][new_uuid] = new_watch
 
+        self.__data['watching'][new_uuid].ensure_data_dir_exists()
 
         if write_to_disk_now:
             self.sync_to_json()
@@ -360,25 +372,28 @@ class ChangeDetectionStore:
             f.write(screenshot)
             f.close()
 
+        # Make a JPEG that's used in notifications (due to being a smaller size) available
+        from PIL import Image
+        im1 = Image.open(target_path)
+        im1.convert('RGB').save(target_path.replace('.png','.jpg'), quality=int(os.getenv("NOTIFICATION_SCREENSHOT_JPG_QUALITY", 75)))
+
 
     def save_error_text(self, watch_uuid, contents):
         if not self.data['watching'].get(watch_uuid):
             return
-
-        self.data['watching'][watch_uuid].ensure_data_dir_exists()
         target_path = os.path.join(self.datastore_path, watch_uuid, "last-error.txt")
+
         with open(target_path, 'w') as f:
             f.write(contents)
 
     def save_xpath_data(self, watch_uuid, data, as_error=False):
-
         if not self.data['watching'].get(watch_uuid):
             return
         if as_error:
             target_path = os.path.join(self.datastore_path, watch_uuid, "elements-error.json")
         else:
             target_path = os.path.join(self.datastore_path, watch_uuid, "elements.json")
-        self.data['watching'][watch_uuid].ensure_data_dir_exists()
+
         with open(target_path, 'w') as f:
             f.write(json.dumps(data))
             f.close()
@@ -448,28 +463,10 @@ class ChangeDetectionStore:
                     print ("Removing",item)
                     unlink(item)
 
-    @property
-    def proxy_list(self):
-        proxy_list = {}
-        proxy_list_file = os.path.join(self.datastore_path, 'proxies.json')
-
-        # Load from external config file
-        if path.isfile(proxy_list_file):
-            with open("{}/proxies.json".format(self.datastore_path)) as f:
-                proxy_list = json.load(f)
-
-        # Mapping from UI config if available
-        extras = self.data['settings']['requests'].get('extra_proxies')
-        if extras:
-            i=0
-            for proxy in extras:
-                i += 0
-                if proxy.get('proxy_name') and proxy.get('proxy_url'):
-                    k = "ui-" + str(i) + proxy.get('proxy_name')
-                    proxy_list[k] = {'label': proxy.get('proxy_name'), 'url': proxy.get('proxy_url')}
-
-
-        return proxy_list if len(proxy_list) else None
+    def import_proxy_list(self, filename):
+        with open(filename) as f:
+            self.proxy_list = json.load(f)
+            print ("Registered proxy list", list(self.proxy_list.keys()))
 
 
     def get_preferred_proxy_for_watch(self, uuid):
@@ -479,10 +476,11 @@ class ChangeDetectionStore:
         :return: proxy "key" id
         """
 
+        proxy_id = None
         if self.proxy_list is None:
             return None
 
-        # If it's a valid one
+        # If its a valid one
         watch = self.data['watching'].get(uuid)
 
         if watch.get('proxy') and watch.get('proxy') in list(self.proxy_list.keys()):
@@ -495,113 +493,13 @@ class ChangeDetectionStore:
             if self.proxy_list.get(system_proxy_id):
                 return system_proxy_id
 
-
-        # Fallback - Did not resolve anything, or doesnt exist, use the first available
-        if system_proxy_id is None or not self.proxy_list.get(system_proxy_id):
+        # Fallback - Did not resolve anything, use the first available
+        if system_proxy_id is None:
             first_default = list(self.proxy_list)[0]
             return first_default
 
         return None
 
-    @property
-    def has_extra_headers_file(self):
-        filepath = os.path.join(self.datastore_path, 'headers.txt')
-        return os.path.isfile(filepath)
-
-    def get_all_base_headers(self):
-        from .model.App import parse_headers_from_text_file
-        headers = {}
-        # Global app settings
-        headers.update(self.data['settings'].get('headers', {}))
-
-        return headers
-
-    def get_all_headers_in_textfile_for_watch(self, uuid):
-        from .model.App import parse_headers_from_text_file
-        headers = {}
-
-        # Global in /datastore/headers.txt
-        filepath = os.path.join(self.datastore_path, 'headers.txt')
-        try:
-            if os.path.isfile(filepath):
-                headers.update(parse_headers_from_text_file(filepath))
-        except Exception as e:
-            print(f"ERROR reading headers.txt at {filepath}", str(e))
-
-        watch = self.data['watching'].get(uuid)
-        if watch:
-
-            # In /datastore/xyz-xyz/headers.txt
-            filepath = os.path.join(watch.watch_data_dir, 'headers.txt')
-            try:
-                if os.path.isfile(filepath):
-                    headers.update(parse_headers_from_text_file(filepath))
-            except Exception as e:
-                print(f"ERROR reading headers.txt at {filepath}", str(e))
-
-            # In /datastore/tag-name.txt
-            tags = self.get_all_tags_for_watch(uuid=uuid)
-            for tag_uuid, tag in tags.items():
-                fname = "headers-"+re.sub(r'[\W_]', '', tag.get('title')).lower().strip() + ".txt"
-                filepath = os.path.join(self.datastore_path, fname)
-                try:
-                    if os.path.isfile(filepath):
-                        headers.update(parse_headers_from_text_file(filepath))
-                except Exception as e:
-                    print(f"ERROR reading headers.txt at {filepath}", str(e))
-
-        return headers
-
-    def get_tag_overrides_for_watch(self, uuid, attr):
-        tags = self.get_all_tags_for_watch(uuid=uuid)
-        ret = []
-
-        if tags:
-            for tag_uuid, tag in tags.items():
-                if attr in tag and tag[attr]:
-                    ret=[*ret, *tag[attr]]
-
-        return ret
-
-    def add_tag(self, name):
-        # If name exists, return that
-        n = name.strip().lower()
-        print (f">>> Adding new tag - '{n}'")
-        if not n:
-            return False
-
-        for uuid, tag in self.__data['settings']['application'].get('tags', {}).items():
-            if n == tag.get('title', '').lower().strip():
-                print (f">>> Tag {name} already exists")
-                return uuid
-
-        # Eventually almost everything todo with a watch will apply as a Tag
-        # So we use the same model as a Watch
-        with self.lock:
-            new_tag = Watch.model(datastore_path=self.datastore_path, default={
-                'title': name.strip(),
-                'date_created': int(time.time())
-            })
-
-            new_uuid = new_tag.get('uuid')
-
-            self.__data['settings']['application']['tags'][new_uuid] = new_tag
-
-        return new_uuid
-
-    def get_all_tags_for_watch(self, uuid):
-        """This should be in Watch model but Watch doesn't have access to datastore, not sure how to solve that yet"""
-        watch = self.data['watching'].get(uuid)
-
-        # Should return a dict of full tag info linked by UUID
-        if watch:
-            return dictfilt(self.__data['settings']['application']['tags'], watch.get('tags', []))
-
-        return {}
-
-    def tag_exists_by_name(self, tag_name):
-        return any(v.get('title', '').lower() == tag_name.lower() for k, v in self.__data['settings']['application']['tags'].items())
-
     # Run all updates
     # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
     #             So therefor - each `update_n` should be very careful about checking if it needs to actually run
@@ -767,36 +665,3 @@ class ChangeDetectionStore:
                 self.data['settings']['application']['notification_urls'][i] = re.sub(r, r'{{\1}}', url)
 
         return
-
-    # Some setups may have missed the correct default, so it shows the wrong config in the UI, although it will default to system-wide
-    def update_10(self):
-        for uuid, watch in self.data['watching'].items():
-            try:
-                if not watch.get('fetch_backend', ''):
-                    watch['fetch_backend'] = 'system'
-            except:
-                continue
-        return
-
-    # We don't know when the date_created was in the past until now, so just add an index number for now.
-    def update_11(self):
-        i = 0
-        for uuid, watch in self.data['watching'].items():
-            if not watch.get('date_created'):
-                watch['date_created'] = i
-            i+=1
-        return
-
-    # Create tag objects and their references from existing tag text
-    def update_12(self):
-        i = 0
-        for uuid, watch in self.data['watching'].items():
-            # Split out and convert old tag string
-            tag = watch.get('tag')
-            if tag:
-                tag_uuids = []
-                for t in tag.split(','):
-                    tag_uuids.append(self.add_tag(name=t))
-
-                self.data['watching'][uuid]['tags'] = tag_uuids
-

changedetectionio/templates/_common_fields.jinja

@@ -17,15 +17,14 @@
                                 <li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
                                 <li><code>tgram://</code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
                                 <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
-                                  <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
                             </div>
                             <div class="notifications-wrapper">
-                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" >Send test notification</a>
+                              <a id="send-test-notification" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Send test notification</a>
                             {% if emailprefix %}
-                              <a id="add-email-helper" class="pure-button button-secondary button-xsmall" >Add email <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='email.svg')}}" alt="Add an email address"> </a>
+                              <a id="add-email-helper" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Add email</a>
                             {% endif %}
-                              <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" >Notification debug logs</a>
+                              <a href="{{url_for('notification_logs')}}" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Notification debug logs</a>
                             </div>
                         </div>
                         <div id="notification-customisation" class="pure-control-group">
@@ -56,66 +55,48 @@
                                     </thead>
                                     <tbody>
                                     <tr>
-                                        <td><code>{{ '{{base_url}}' }}</code></td>
+                                        <td><code>{{ '{{ base_url }}' }}</code></td>
                                         <td>The URL of the changedetection.io instance you are running.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{watch_url}}' }}</code></td>
+                                        <td><code>{{ '{{ watch_url }}' }}</code></td>
                                         <td>The URL being watched.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{watch_uuid}}' }}</code></td>
+                                        <td><code>{{ '{{ watch_uuid }}' }}</code></td>
                                         <td>The UUID of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{watch_title}}' }}</code></td>
+                                        <td><code>{{ '{{ watch_title }}' }}</code></td>
                                         <td>The title of the watch.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{watch_tag}}' }}</code></td>
+                                        <td><code>{{ '{{ watch_tag }}' }}</code></td>
                                         <td>The watch label / tag</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{preview_url}}' }}</code></td>
+                                        <td><code>{{ '{{ preview_url }}' }}</code></td>
                                         <td>The URL of the preview page generated by changedetection.io.</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{diff_url}}' }}</code></td>
-                                        <td>The URL of the diff output for the watch.</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff}}' }}</code></td>
-                                        <td>The diff output - only changes, additions, and removals</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff_added}}' }}</code></td>
-                                        <td>The diff output - only changes and additions</td>
-                                    </tr>
-									<tr>
-                                        <td><code>{{ '{{diff_removed}}' }}</code></td>
-                                        <td>The diff output - only changes and removals</td>
+                                        <td><code>{{ '{{ diff_url }}' }}</code></td>
+                                        <td>The diff output - differences only</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{diff_full}}' }}</code></td>
+                                        <td><code>{{ '{{ diff_full }}' }}</code></td>
                                         <td>The diff output - full difference output</td>
                                     </tr>
                                     <tr>
-                                        <td><code>{{ '{{current_snapshot}}' }}</code></td>
+                                        <td><code>{{ '{{ current_snapshot }}' }}</code></td>
                                         <td>The current snapshot value, useful when combined with JSON or CSS filters
                                         </td>
                                     </tr>
-                                    <tr>
-                                        <td><code>{{ '{{triggered_text}}' }}</code></td>
-                                        <td>Text that tripped the trigger from filters</td>
-                                    </tr>
                                     </tbody>
                                 </table>
                                 <div class="pure-form-message-inline">
                                     <br>
-                                    URLs generated by changedetection.io (such as <code>{{ '{{diff_url}}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br>
+                                    URLs generated by changedetection.io (such as <code>{{ '{{ diff_url }}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
                                     Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
-									<br>
-									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
                                 </div>
                             </div>
                         </div>

changedetectionio/templates/_pagination.jinja

@@ -0,0 +1,7 @@
+{% macro pagination(sorted_watches, total_per_page, current_page) %}
+  {{ sorted_watches|length }}
+
+  {% for row in sorted_watches|batch(total_per_page, ' ') %}
+    {{ loop.index}}
+  {% endfor %}
+{% endmacro %}

changedetectionio/templates/base.html

@@ -2,35 +2,35 @@
 <html lang="en" data-darkmode="{{ get_darkmode_state() }}">
 
   <head>
-    <meta charset="utf-8" >
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" >
-    <meta name="description" content="Self hosted website change detection." >
+    <meta charset="utf-8"/>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+    <meta name="description" content="Self hosted website change detection."/>
     <title>Change Detection{{extra_title}}</title>
-    <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag %}- {{active_tag}}{% endif %}" href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}" >
-    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}" >
-    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}" >
+    <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag %}- {{active_tag}}{% endif %}" href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"/>
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}"/>
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}"/>
     {% if extra_stylesheets %}
       {% for m in extra_stylesheets %}
-        <link rel="stylesheet" href="{{ m }}?ver=1000" >
+        <link rel="stylesheet" href="{{ m }}?ver=1000"/>
       {% endfor %}
     {% endif %}
 
-    <link rel="apple-touch-icon" sizes="180x180" href="{{url_for('static_content', group='favicons', filename='apple-touch-icon.png')}}">
-    <link rel="icon" type="image/png" sizes="32x32" href="{{url_for('static_content', group='favicons', filename='favicon-32x32.png')}}">
-    <link rel="icon" type="image/png" sizes="16x16" href="{{url_for('static_content', group='favicons', filename='favicon-16x16.png')}}">
-    <link rel="manifest" href="{{url_for('static_content', group='favicons', filename='site.webmanifest')}}">
-    <link rel="mask-icon" href="{{url_for('static_content', group='favicons', filename='safari-pinned-tab.svg')}}" color="#5bbad5">
-    <link rel="shortcut icon" href="{{url_for('static_content', group='favicons', filename='favicon.ico')}}">
-    <meta name="msapplication-TileColor" content="#da532c">
-    <meta name="msapplication-config" content="favicons/browserconfig.xml">
-    <meta name="theme-color" content="#ffffff">
+    <link rel="apple-touch-icon" sizes="180x180" href="{{url_for('static_content', group='favicons', filename='apple-touch-icon.png')}}"/>
+    <link rel="icon" type="image/png" sizes="32x32" href="{{url_for('static_content', group='favicons', filename='favicon-32x32.png')}}"/>
+    <link rel="icon" type="image/png" sizes="16x16" href="{{url_for('static_content', group='favicons', filename='favicon-16x16.png')}}"/>
+    <link rel="manifest" href="{{url_for('static_content', group='favicons', filename='site.webmanifest')}}"/>
+    <link rel="mask-icon" href="{{url_for('static_content', group='favicons', filename='safari-pinned-tab.svg')}}" color="#5bbad5"/>
+    <link rel="shortcut icon" href="{{url_for('static_content', group='favicons', filename='favicon.ico')}}"/>
+    <meta name="msapplication-TileColor" content="#da532c"/>
+    <meta name="msapplication-config" content="favicons/browserconfig.xml"/>
+    <meta name="theme-color" content="#ffffff"/>
 
     <style>
       body::before {
         background-image: url({{url_for('static_content', group='images', filename='gradient-border.png') }});
       }
     </style>
-    <script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
+    <script type="text/javascript" src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
   </head>
 
   <body>
@@ -58,9 +58,6 @@
           {% if current_user.is_authenticated or not has_password %}
             {% if not
             current_diff_url %}
-              <li class="pure-menu-item">
-                <a href="{{ url_for('tags.tags_overview_page')}}" class="pure-menu-link">GROUPS</a>
-              </li>
               <li class="pure-menu-item">
                 <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
               </li>
@@ -85,18 +82,11 @@
               <a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a>
             </li>
           {% endif %}
-          <li class="pure-menu-item pure-form" id="search-menu-item">
-            <!-- We use GET here so it offers people a chance to set bookmarks etc -->
-            <form name="searchForm" action="" method="GET">
-              <input id="search-q" class="" name="q" placeholder="URL or Title {% if active_tag %}in '{{ active_tag }}'{% endif %}" required="" type="text" value="">
-              <input name="tags" type="hidden" value="{% if active_tag %}{{active_tag}}{% endif %}">
-              <button class="toggle-button " id="toggle-search" type="button" title="Search, or Use Alt+S Key" >
-                {% include "svgs/search-icon.svg" %}
-              </button>
-            </form>
-          </li>
           <li class="pure-menu-item">
-            <button class="toggle-button" id ="toggle-light-mode" type="button" title="Toggle Light/Dark Mode">
+            {% if dark_mode %}
+            {% set darkClass = 'dark' %}
+            {% endif %}
+            <button class="toggle-theme {{darkClass}}" type="button" title="Toggle Light/Dark Mode">
               <span class="visually-hidden">Toggle light/dark mode</span>
               <span class="icon-light">
                 {% include "svgs/light-mode-toggle-icon.svg" %}
@@ -116,7 +106,7 @@
     </div>
     {% if hosted_sticky %}
       <div class="sticky-tab" id="hosted-sticky">
-        <a href="https://changedetection.io/?ref={{guid}}">Let us host your instance!</a>
+        <a href="https://lemonade.changedetection.io/start?ref={{guid}}">Let us host your instance!</a>
       </div>
     {% endif %}
     {% if left_sticky %}
@@ -147,13 +137,16 @@
           <li class="message">
             Share this link:
             <span id="share-link">{{ session['share-link'] }}</span>
-            <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='copy.svg')}}" >
+            <img style="height: 1em; display: inline-block" src="{{url_for('static_content', group='images', filename='copy.svg')}}"/>
           </li>
         </ul>
       {% endif %}
       {% block content %}{% endblock %}
     </section>
-    <script src="{{url_for('static_content', group='js', filename='toggle-theme.js')}}" defer></script>
+    <script
+      type="text/javascript"
+      src="{{url_for('static_content', group='js', filename='toggle-theme.js')}}"
+      defer></script>
   </body>
 
 </html>

changedetectionio/templates/clear_all_history.html

@@ -6,7 +6,7 @@
       action="{{url_for('clear_all_history')}}"
       method="POST"
     >
-      <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
+      <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
       <fieldset>
         <div class="pure-control-group">
           This will remove version history (snapshots) for ALL watches, but keep

changedetectionio/templates/diff.html

@@ -7,7 +7,7 @@
     const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
     {% endif %}
 </script>
-<script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
 <div id="settings">
     <h1>Differences</h1>
@@ -15,15 +15,15 @@
         <fieldset>
 
             <label for="diffWords" class="pure-checkbox">
-                <input type="radio" name="diff_type" id="diffWords" value="diffWords"> Words</label>
+                <input type="radio" name="diff_type" id="diffWords" value="diffWords"/> Words</label>
             <label for="diffLines" class="pure-checkbox">
-                <input type="radio" name="diff_type" id="diffLines" value="diffLines" checked=""> Lines</label>
+                <input type="radio" name="diff_type" id="diffLines" value="diffLines" checked=""/> Lines</label>
 
             <label for="diffChars" class="pure-checkbox">
-                <input type="radio" name="diff_type" id="diffChars" value="diffChars"> Chars</label>
+                <input type="radio" name="diff_type" id="diffChars" value="diffChars"/> Chars</label>
             <!-- @todo - when mimetype is JSON, select this by default? -->
             <label for="diffJson" class="pure-checkbox">
-                <input type="radio" name="diff_type" id="diffJson" value="diffJson" > JSON</label>
+                <input type="radio" name="diff_type" id="diffJson" value="diffJson" /> JSON</label>
 
             {% if versions|length >= 1 %}
             <label for="diff-version">Compare newest (<span id="current-v-date"></span>) with</label>
@@ -43,7 +43,7 @@
     <span>
         <!-- https://github.com/kpdecker/jsdiff/issues/389 ? -->
         <label for="ignoreWhitespace" class="pure-checkbox" id="label-diff-ignorewhitespace">
-            <input type="checkbox" id="ignoreWhitespace" name="ignoreWhitespace" > Ignore Whitespace</label>
+            <input type="checkbox" id="ignoreWhitespace" name="ignoreWhitespace"/> Ignore Whitespace</label>
     </span>
 </div>
 
@@ -51,7 +51,7 @@
     <a onclick="next_diff();">Jump</a>
 </div>
 
-<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <div class="tabs">
     <ul>
         {% if last_error_text %}<li class="tab" id="error-text-tab"><a href="#error-text">Error Text</a></li> {% endif %}
@@ -72,16 +72,12 @@
 
     <div class="tab-pane-inner" id="error-screenshot">
         <div class="snapshot-age error">{{watch_a.snapshot_error_screenshot_ctime|format_seconds_ago}} seconds ago</div>
-        <img id="error-screenshot-img"  style="max-width: 80%" alt="Current error-ing screenshot from most recent request" >
+        <img id="error-screenshot-img"  style="max-width: 80%" alt="Current error-ing screenshot from most recent request"/>
     </div>
 
      <div class="tab-pane-inner" id="text">
-         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.</div>
-
-         {% if password_enabled_and_share_is_off %}
-           <div class="tip">Pro-tip: You can enable <strong>"share access when password is enabled"</strong> from settings</div>
-         {% endif %}
-
+         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.
+         </div>
          <div class="snapshot-age">{{watch_a.snapshot_text_ctime|format_timestamp_timeago}}</div>
 
          <table>
@@ -105,7 +101,7 @@
          {% if is_html_webdriver %}
            {% if screenshot %}
             <div class="snapshot-age">{{watch_a.snapshot_screenshot_ctime|format_timestamp_timeago}}</div>
-            <img style="max-width: 80%" id="screenshot-img" alt="Current screenshot from most recent request" >
+            <img style="max-width: 80%" id="screenshot-img" alt="Current screenshot from most recent request"/>
            {% else %}
               No screenshot available just yet! Try rechecking the page.
            {% endif %}
@@ -117,19 +113,19 @@
         <form id="extract-data-form" class="pure-form pure-form-stacked edit-form"
               action="{{ url_for('diff_history_page', uuid=uuid) }}#extract"
               method="POST">
-            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
 
             <p>This tool will extract text data from all of the watch history.</p>
 
             <div class="pure-control-group">
                 {{ render_field(extract_form.extract_regex) }}
                 <span class="pure-form-message-inline">
-                    A <strong>RegEx</strong> is a pattern that identifies exactly which part inside of the text that you want to extract.<br>
+                    A <strong>RegEx</strong> is a pattern that identifies exactly which part inside of the text that you want to extract.<br/>
 
                     <p>
-                        For example, to extract only the numbers from text &dash;<br>
-                        <strong>Raw text</strong>: <code>Temperature <span style="color: red">5.5</span>°C in Sydney</code><br>
-                        <strong>RegEx to extract:</strong> <code>Temperature <span style="color: red">([0-9\.]+)</span></code><br>
+                        For example, to extract only the numbers from text &dash;</br>
+                        <strong>Raw text</strong>: <code>Temperature <span style="color: red">5.5</span>°C in Sydney</code></br>
+                        <strong>RegEx to extract:</strong> <code>Temperature <span style="color: red">([0-9\.]+)</span></code><br/>
                     </p>
                     <p>
                         <a href="https://RegExr.com/">Be sure to test your RegEx here.</a>
@@ -149,9 +145,9 @@
 <script>
     const newest_version_timestamp = {{newest_version_timestamp}};
 </script>
-<script src="{{url_for('static_content', group='js', filename='diff.min.js')}}"></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff.min.js')}}"></script>
 
-<script src="{{url_for('static_content', group='js', filename='diff-render.js')}}"></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff-render.js')}}"></script>
 
 
-{% endblock %}
+{% endblock %}

changedetectionio/templates/edit.html

@@ -2,7 +2,7 @@
 {% block content %}
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
-<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script>
     const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
     const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
@@ -14,17 +14,15 @@
 {% endif %}
 
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
-    const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
-
 </script>
 
-<script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='visual-selector.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='limit.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='visual-selector.js')}}" defer></script>
 {% if playwright_enabled %}
-<script src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
 {% endif %}
 
 <div class="edit-form monospaced-textarea">
@@ -36,15 +34,8 @@
             {% if playwright_enabled %}
             <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
             {% endif %}
-
-            {% if watch['processor'] == 'text_json_diff' %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
             <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
-            {% endif %}
-
-            {% if watch['processor'] == 'restock_diff' %}
-            <li class="tab"><a href="#restock">Restock Detection</a></li>
-            {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
         </ul>
     </div>
@@ -52,30 +43,20 @@
     <div class="box-wrap inner">
         <form class="pure-form pure-form-stacked"
               action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save')) }}" method="POST">
-             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
 
             <div class="tab-pane-inner" id="general">
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
-                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span><br>
-                        <span class="pure-form-message-inline">You can use variables in the URL, perfect for inserting the current date and other logic, <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a></span><br>
-                        <span class="pure-form-message-inline">
-                        {% if watch['processor'] == 'text_json_diff' %}
-                            Current mode: <strong>Webpage Text/HTML, JSON and PDF changes.</strong><br>
-                          <a href="{{url_for('edit_page', uuid=uuid)}}?switch_processor=restock_diff" class="pure-button button-xsmall">Switch to re-stock detection mode.</a>
-                        {% else %}
-                        Current mode: <strong>Re-stock detection.</strong><br>
-                          <a href="{{url_for('edit_page', uuid=uuid)}}?switch_processor=text_json_diff" class="pure-button button-xsmall">Switch to Webpage Text/HTML, JSON and PDF changes mode.</a>
-                        {% endif %}
-                        </span>
-
+                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span><br/>
+                        <span class="pure-form-message-inline">You can use variables in the URL, perfect for inserting the current date and other logic, <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a></span><br/>
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.title, class="m-d") }}
                     </div>
                     <div class="pure-control-group">
-                        {{ render_field(form.tags) }}
+                        {{ render_field(form.tag) }}
                         <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
                     </div>
                     <div class="pure-control-group">
@@ -125,10 +106,10 @@
                         {{ render_field(form.webdriver_delay) }}
                         <div class="pure-form-message-inline">
                             <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
-                            <br>
+                            <br/>
                             This will wait <i>n</i> seconds before extracting the text.
                             {% if using_global_webdriver_wait %}
-                            <br><strong>Using the current global default settings</strong>
+                            <br/><strong>Using the current global default settings</strong>
                             {% endif %}
                         </div>
                     </div>
@@ -152,17 +133,6 @@
 {{ render_field(form.headers, rows=5, placeholder="Example
 Cookie: foobar
 User-Agent: wonderbra 1.0") }}
-
-                        <div class="pure-form-message-inline">
-                            {% if has_extra_headers_file %}
-                                <strong>Alert! Extra headers file found and will be added to this watch!</strong>
-                            {% else %}
-                                Headers can be also read from a file in your data-directory <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Adding-headers-from-an-external-file">Read more here</a>
-                            {% endif %}
-                            <br>
-                            (Not supported by Selenium browser)
-                        </div>
-
                     </div>
                     <div class="pure-control-group" id="request-body">
                                         {{ render_field(form.body, rows=5, placeholder="Example
@@ -176,7 +146,7 @@ User-Agent: wonderbra 1.0") }}
             </div>
             {% if playwright_enabled %}
             <div class="tab-pane-inner" id="browser-steps">
-                <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
+                <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}">
                 <fieldset>
                     <div class="pure-control-group">
                         <!--
@@ -199,12 +169,11 @@ User-Agent: wonderbra 1.0") }}
                                     <span class="loader" >
                                         <span id="browsersteps-click-start">
                                             <h2 >Click here to Start</h2>
-                                            <svg style="height: 3.5rem;" version="1.1" viewBox="0 0 32 32"  xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="Layer_1"/><g id="play_x5F_alt"><path d="M16,0C7.164,0,0,7.164,0,16s7.164,16,16,16s16-7.164,16-16S24.836,0,16,0z M10,24V8l16.008,8L10,24z" style="fill: var(--color-grey-400);"/></g></svg><br>
-                                            Please allow 10-15 seconds for the browser to connect.<br>
+                                            Please allow 10-15 seconds for the browser to connect.
                                         </span>
                                         <div class="spinner"  style="display: none;"></div>
                                     </span>
-                                    <img class="noselect" id="browsersteps-img" src="" style="max-width: 100%; width: 100%;" >
+                                    <img  class="noselect" id="browsersteps-img" src="" style="max-width: 100%; width: 100%;" />
                                     <canvas  class="noselect" id="browsersteps-selector-canvas" style="max-width: 100%; width: 100%;"></canvas>
                                 </div>
                             </div>
@@ -234,7 +203,7 @@ User-Agent: wonderbra 1.0") }}
                     <div class="field-group" id="notification-field-group">
                         {% if has_default_notification_urls %}
                         <div class="inline-warning">
-                            <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!" >
+                            <img class="inline-warning-icon" src="{{url_for('static_content', group='images', filename='notice.svg')}}" alt="Look out!" title="Lookout!"/>
                             There are <a href="{{ url_for('settings_page')}}#notifications">system-wide notification URLs enabled</a>, this form will override notification settings for this watch only &dash; an empty Notification URL list here will still send notifications.
                         </div>
                         {% endif %}
@@ -245,10 +214,9 @@ User-Agent: wonderbra 1.0") }}
                 </fieldset>
             </div>
 
-            {% if watch['processor'] == 'text_json_diff' %}
             <div class="tab-pane-inner" id="filters-and-triggers">
                     <div class="pure-control-group">
-                            <strong>Pro-tips:</strong><br>
+                            <strong>Pro-tips:</strong><br/>
                             <ul>
                                 <li>
                                     Use the preview page to see your filters and triggers highlighted.
@@ -258,6 +226,12 @@ User-Agent: wonderbra 1.0") }}
                                 </li>
                             </ul>
                     </div>
+                    <fieldset>
+                        <div class="pure-control-group">
+                            {{ render_checkbox_field(form.check_unique_lines) }}
+                            <span class="pure-form-message-inline">Good for websites that just move the content around, and you want to know when NEW content is added, compares new lines against all history for this watch.</span>
+                        </div>
+                    </fieldset>
                     <div class="pure-control-group">
                         {% set field = render_field(form.include_filters,
                             rows=5,
@@ -267,9 +241,9 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                         %}
                         {{ field }}
                         {% if '/text()' in  field %}
-                          <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br>
+                          <span class="pure-form-message-inline"><strong>Note!: //text() function does not work where the &lt;element&gt; contains &lt;![CDATA[]]&gt;</strong></span><br/>
                         {% endif %}
-                        <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br>
+                        <span class="pure-form-message-inline">One rule per line, <i>any</i> rules that matches will be used.<br/>
 
                     <ul>
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
@@ -292,42 +266,40 @@ xpath://body/div/span[contains(@class, 'example-class')]",
                             </li>
                     </ul>
                     Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! <a
-                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br>
+                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
                 </span>
                     </div>
-                <fieldset class="pure-control-group">
-                    {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
+                    <div class="pure-control-group">
+                      {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
 footer
 nav
 .stockticker") }}
-                    <span class="pure-form-message-inline">
+                      <span class="pure-form-message-inline">
                         <ul>
                           <li> Remove HTML element(s) by CSS selector before text conversion. </li>
                           <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
                         </ul>
                       </span>
-                </fieldset>
-                <div class="text-filtering">
-                <fieldset class="pure-group" id="text-filtering-type-options">
-                    <h3>Text filtering</h3>
-                        Limit trigger/ignore/block/extract to;<br>
-                        {{ render_checkbox_field(form.filter_text_added) }}
-                        {{ render_checkbox_field(form.filter_text_replaced) }}
-                        {{ render_checkbox_field(form.filter_text_removed) }}
-                    <span class="pure-form-message-inline">Note: Depending on the length and similarity of the text on each line, the algorithm may consider an <strong>addition</strong> instead of <strong>replacement</strong> for example.</span>
-                    <span class="pure-form-message-inline">So it's always better to select <strong>Added</strong>+<strong>Replaced</strong> when you're interested in new content.</span><br>
-                    <span class="pure-form-message-inline">When content is merely moved in a list, it will also trigger an <strong>addition</strong>, consider enabling <code><strong>Only trigger when unique lines appear</strong></code></span>
-                </fieldset>
+                    </div>
+                <fieldset class="pure-group">
+                    {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
+/some.regex\d{2}/ for case-INsensitive regex
+                    ") }}
+                    <span class="pure-form-message-inline">
+                        <ul>
+                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
+                            <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li>
+                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
+                            <li>Use the preview/show current tab to see ignores</li>
+                        </ul>
+                </span>
 
-                <fieldset class="pure-control-group">
-                    {{ render_checkbox_field(form.check_unique_lines) }}
-                    <span class="pure-form-message-inline">Good for websites that just move the content around, and you want to know when NEW content is added, compares new lines against all history for this watch.</span>
-                </fieldset>
+            </fieldset>
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line
 /some.regex\d{2}/ for case-INsensitive regex
-") }}
+                    ") }}
                         <span class="pure-form-message-inline">
                     <ul>
                         <li>Text to wait for before triggering a change/notification, all text and regex are tested <i>case-insensitive</i>.</li>
@@ -338,21 +310,6 @@ nav
                         </span>
                     </div>
                 </fieldset>
-                <fieldset class="pure-group">
-                    {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
-/some.regex\d{2}/ for case-INsensitive regex
-") }}
-                    <span class="pure-form-message-inline">
-                        <ul>
-                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
-                            <li>Regular Expression support, wrap the entire line in forward slash <code>/regex/</code></li>
-                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
-                            <li>Use the preview/show current tab to see ignores</li>
-                        </ul>
-                </span>
-
-                </fieldset>
-
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.text_should_not_be_present, rows=5, placeholder="For example: Out of stock
@@ -377,7 +334,7 @@ Unavailable") }}
                         <li>Extracts text in the final output (line by line) after other filters using regular expressions;
                             <ul>
                                 <li>Regular expression &dash; example <code>/reports.+?2022/i</code></li>
-                                <li>Use <code>//(?aiLmsux))</code> type flags (more <a href="https://docs.python.org/3/library/re.html#index-15">information here</a>)<br></li>
+                                <li>Use <code>//(?aiLmsux))</code> type flags (more <a href="https://docs.python.org/3/library/re.html#index-15">information here</a>)<br/></li>
                                 <li>Keyword example &dash; example <code>Out of stock</code></li>
                                 <li>Use groups to extract just that text &dash; example <code>/reports.+?(\d+)/i</code> returns a list of years only</li>
                             </ul>
@@ -387,30 +344,16 @@ Unavailable") }}
                         </span>
                     </div>
                 </fieldset>
-                </div>
             </div>
-            {% endif %}
 
-            {% if watch['processor'] == 'restock_diff' %}
-            <div class="tab-pane-inner" id="restock">
-                    <fieldset>
-                        <div class="pure-control-group">
-                            {{ render_checkbox_field(form.in_stock_only) }}
-                            <span class="pure-form-message-inline">Only trigger notifications when page changes from <strong>out of stock</strong> to <strong>back in stock</strong></span>
-                        </div>
-                    </fieldset>
-            </div>
-            {% endif %}
-
-            {% if watch['processor'] == 'text_json_diff' %}
             <div class="tab-pane-inner visual-selector-ui" id="visualselector">
-                <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
+                <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}">
 
                 <fieldset>
                     <div class="pure-control-group">
                         {% if visualselector_enabled %}
                             <span class="pure-form-message-inline">
-                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection &dash; after the <i>Browser Steps</i> has completed.<br><br>
+                                The Visual Selector tool lets you select the <i>text</i> elements that will be used for the change detection &dash; after the <i>Browser Steps</i> has completed.<br/><br/>
                             </span>
 
                             <div id="selector-header">
@@ -421,7 +364,7 @@ Unavailable") }}
                                 <!-- request the screenshot and get the element offset info ready -->
                                 <!-- use img src ready load to know everything is ready to map out -->
                                 <!-- @todo: maybe something interesting like a field to select 'elements that contain text... and their parents n' -->
-                                <img id="selector-background" >
+                                <img id="selector-background" />
                                 <canvas id="selector-canvas"></canvas>
                             </div>
                             <div id="selector-current-xpath" style="overflow-x: hidden"><strong>Currently:</strong>&nbsp;<span class="text">Loading...</span></div>
@@ -435,7 +378,6 @@ Unavailable") }}
                     </div>
                 </fieldset>
             </div>
-            {% endif %}
 
             <div id="actions">
                 <div class="pure-control-group">

changedetectionio/templates/import.html

@@ -1,7 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.jinja' import render_field %}
-<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <div class="edit-form monospaced-textarea">
 
     <div class="tabs collapsable">
@@ -13,8 +12,9 @@
 
     <div class="box-wrap inner">
         <form class="pure-form pure-form-aligned" action="{{url_for('import_page')}}" method="POST">
-            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
             <div class="tab-pane-inner" id="url-list">
+                <fieldset class="pure-group">
                     <legend>
                         Enter one URL per line, and optionally add tags for each URL after a space, delineated by comma
                         (,):
@@ -23,7 +23,7 @@
                         <br>
                         URLs which do not pass validation will stay in the textarea.
                     </legend>
-                {{ render_field(form.processor, class="processor") }}
+
 
                     <textarea name="urls" class="pure-input-1-2" placeholder="https://"
                               style="width: 100%;
@@ -31,24 +31,22 @@
                                 white-space: pre;
                                 overflow-wrap: normal;
                                 overflow-x: scroll;" rows="25">{{ import_url_list_remaining }}</textarea>
+                </fieldset>
 
-<div id="quick-watch-processor-type">
-
-                    </div>
 
             </div>
 
             <div class="tab-pane-inner" id="distill-io">
 
 
-
+                <fieldset class="pure-group">
                     <legend>
-                        Copy and Paste your Distill.io watch 'export' file, this should be a JSON file.<br>
+                        Copy and Paste your Distill.io watch 'export' file, this should be a JSON file.</br>
                         This is <i>experimental</i>, supported fields are <code>name</code>, <code>uri</code>, <code>tags</code>, <code>config:selections</code>, the rest (including <code>schedule</code>) are ignored.
-                        <br>
+                        <br/>
                         <p>
-                        How to export? <a href="https://distill.io/docs/web-monitor/how-export-and-import-monitors/">https://distill.io/docs/web-monitor/how-export-and-import-monitors/</a><br>
-                        Be sure to set your default fetcher to Chrome if required.<br>
+                        How to export? <a href="https://distill.io/docs/web-monitor/how-export-and-import-monitors/">https://distill.io/docs/web-monitor/how-export-and-import-monitors/</a><br/>
+                        Be sure to set your default fetcher to Chrome if required.</br>
                         </p>
                     </legend>
 
@@ -77,7 +75,7 @@
     ]
 }
 " rows="25">{{ original_distill_json }}</textarea>
-
+                </fieldset>
             </div>
             <button type="submit" class="pure-button pure-input-1-2 pure-button-primary">Import</button>
         </form>

changedetectionio/templates/login.html

@@ -4,13 +4,13 @@
 <div class="login-form">
  <div class="inner">
     <form class="pure-form pure-form-stacked" action="{{url_for('login')}}" method="POST">
-        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <div class="pure-control-group">
                 <label for="password">Password</label>
                 <input type="password" id="password" required="" name="password" value=""
                        size="15" autofocus />
-                <input type="hidden" id="email" name="email" value="defaultuser@changedetection.io" >
+                <input type="hidden" id="email" name="email" value="defaultuser@changedetection.io" />
             </div>
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Login</button>

changedetectionio/templates/preview.html

@@ -7,9 +7,9 @@
     const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
     {% endif %}
 </script>
-<script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
-<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <div class="tabs">
     <ul>
         {% if last_error_text %}<li class="tab" id="error-text-tab"><a href="#error-text">Error Text</a></li> {% endif %}
@@ -31,7 +31,7 @@
 
     <div class="tab-pane-inner" id="error-screenshot">
         <div class="snapshot-age error">{{watch.snapshot_error_screenshot_ctime|format_seconds_ago}} seconds ago</div>
-        <img id="error-screenshot-img"  style="max-width: 80%" alt="Current erroring screenshot from most recent request" >
+        <img id="error-screenshot-img"  style="max-width: 80%" alt="Current erroring screenshot from most recent request"/>
     </div>
 
     <div class="tab-pane-inner" id="text">
@@ -54,11 +54,11 @@
          <div class="tip">
              For now, Differences are performed on text, not graphically, only the latest screenshot is available.
          </div>
-         <br>
+         </br>
          {% if is_html_webdriver %}
            {% if screenshot %}
              <div class="snapshot-age">{{watch.snapshot_screenshot_ctime|format_timestamp_timeago}}</div>
-             <img style="max-width: 80%" id="screenshot-img" alt="Current screenshot from most recent request" >
+             <img style="max-width: 80%" id="screenshot-img" alt="Current screenshot from most recent request"/>
            {% else %}
               No screenshot available just yet! Try rechecking the page.
            {% endif %}
@@ -67,4 +67,4 @@
          {% endif %}
      </div>
 </div>
-{% endblock %}
+{% endblock %}

changedetectionio/templates/settings.html

@@ -9,10 +9,10 @@
     const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
 {% endif %}
 </script>
-<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 
-<script src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='global-settings.js')}}" defer></script>
 <div class="edit-form">
     <div class="tabs collapsable">
         <ul>
@@ -21,12 +21,11 @@
             <li class="tab"><a href="#fetching">Fetching</a></li>
             <li class="tab"><a href="#filters">Global Filters</a></li>
             <li class="tab"><a href="#api">API</a></li>
-            <li class="tab"><a href="#proxies">CAPTCHA &amp; Proxies</a></li>
         </ul>
     </div>
     <div class="box-wrap inner">
         <form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
-            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
             <div class="tab-pane-inner" id="general">
                 <fieldset>
                     <div class="pure-control-group">
@@ -40,7 +39,7 @@
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class="filter_failure_notification_threshold_attempts") }}
                         <span class="pure-form-message-inline">After this many consecutive times that the CSS/xPath filter is missing, send a notification
-                            <br>
+                            <br/>
                         Set to <strong>0</strong> to disable
                         </span>
                     </div>
@@ -57,23 +56,14 @@
                         {% endif %}
                     </div>
 
-                    <div class="pure-control-group">
-                        {{ render_checkbox_field(form.application.form.shared_diff_access, class="shared_diff_access") }}
-                        <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page)
-                        </span>
-                    </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
                         class="m-d") }}
                         <span class="pure-form-message-inline">
-                            Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notifications and RSS links.<br>Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"),
+                            Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"),
                             <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
                         </span>
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.application.form.pager_size) }}
-                        <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
-                    </div>
 
                     <div class="pure-control-group">
                         {{ render_checkbox_field(form.application.form.extract_title_as_title) }}
@@ -109,13 +99,13 @@
                         <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
                         <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                     </span>
-                    <br>
+                    <br/>
                     Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using BrightData Proxies, find out more here.</a>
                 </div>
                 <fieldset class="pure-group" id="webdriver-override-options">
                     <div class="pure-form-message-inline">
                         <strong>If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here.</strong>
-                        <br>
+                        <br/>
                         This will wait <i>n</i> seconds before extracting the text.
                     </div>
                     <div class="pure-control-group">
@@ -128,14 +118,14 @@
 
                     <fieldset class="pure-group">
                     {{ render_checkbox_field(form.application.form.ignore_whitespace) }}
-                    <span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br>
+                    <span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br/>
                     <i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc.
                     </span>
                     </fieldset>
                 <fieldset class="pure-group">
                     {{ render_checkbox_field(form.application.form.render_anchor_tag_content) }}
                     <span class="pure-form-message-inline">Render anchor tag content, default disabled, when enabled renders links as <code>(link text)[https://somesite.com]</code>
-                        <br>
+                        <br/>
                     <i>Note:</i> Changing this could affect the content of your existing watches, possibly trigger alerts etc.
                     </span>
                     </fieldset>
@@ -155,7 +145,7 @@ nav
                     {{ render_field(form.application.form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
 /some.regex\d{2}/ for case-INsensitive regex
                     ") }}
-                    <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br>
+                    <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br/>
                     <span class="pure-form-message-inline">
                         <ul>
                             <li>Note: This is applied globally in addition to the per-watch rules.</li>
@@ -174,35 +164,20 @@ nav
 
                 <div class="pure-control-group">
                     {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}
-                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header</div><br>
-                    <div class="pure-form-message-inline"><br>API Key <span id="api-key">{{api_key}}</span>
+                    <div class="pure-form-message-inline">Restrict API access limit by using <code>x-api-key</code> header</div><br/>
+                    <div class="pure-form-message-inline"><br/>API Key <span id="api-key">{{api_key}}</span>
                         <span style="display:none;" id="api-key-copy" >copy</span>
                     </div>
                 </div>
             </div>
-            <div class="tab-pane-inner" id="proxies">
 
-                <p><strong>Tip</strong>: You can connect to websites using <a href="https://brightdata.grsm.io/n0r16zf7eivq">BrightData</a> proxies, their service <strong>WebUnlocker</strong> will solve most CAPTCHAs, whilst their <strong>Residential Proxies</strong> may help to avoid CAPTCHA altogether. </p>
-                <p>It may be easier to try <strong>WebUnlocker</strong> first, WebUnlocker also supports country selection.</p>
-                <p>
-                    When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the example URL below<br>
-                    The Proxy URL with BrightData should start with <code>http://brd-customer...</code>
-                </p>
-
-                <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p>
-
-
-                <div class="pure-control-group">
-                    {{ render_field(form.requests.form.extra_proxies) }}
-                    <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span>
-                </div>
-            </div>
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}
                     <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
                     <a href="{{url_for('clear_all_history')}}" class="pure-button button-small button-cancel">Clear Snapshot History</a>
                 </div>
+
             </div>
         </form>
     </div>

changedetectionio/templates/svgs/search-icon.svg

@@ -1 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?><svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 122.879 119.799" enable-background="new 0 0 122.879 119.799" xml:space="preserve"><g><path d="M49.988,0h0.016v0.007C63.803,0.011,76.298,5.608,85.34,14.652c9.027,9.031,14.619,21.515,14.628,35.303h0.007v0.033v0.04 h-0.007c-0.005,5.557-0.917,10.905-2.594,15.892c-0.281,0.837-0.575,1.641-0.877,2.409v0.007c-1.446,3.66-3.315,7.12-5.547,10.307 l29.082,26.139l0.018,0.016l0.157,0.146l0.011,0.011c1.642,1.563,2.536,3.656,2.649,5.78c0.11,2.1-0.543,4.248-1.979,5.971 l-0.011,0.016l-0.175,0.203l-0.035,0.035l-0.146,0.16l-0.016,0.021c-1.565,1.642-3.654,2.534-5.78,2.646 c-2.097,0.111-4.247-0.54-5.971-1.978l-0.015-0.011l-0.204-0.175l-0.029-0.024L78.761,90.865c-0.88,0.62-1.778,1.209-2.687,1.765 c-1.233,0.755-2.51,1.466-3.813,2.115c-6.699,3.342-14.269,5.222-22.272,5.222v0.007h-0.016v-0.007 c-13.799-0.004-26.296-5.601-35.338-14.645C5.605,76.291,0.016,63.805,0.007,50.021H0v-0.033v-0.016h0.007 c0.004-13.799,5.601-26.296,14.645-35.338C23.683,5.608,36.167,0.016,49.955,0.007V0H49.988L49.988,0z M50.004,11.21v0.007h-0.016 h-0.033V11.21c-10.686,0.007-20.372,4.35-27.384,11.359C15.56,29.578,11.213,39.274,11.21,49.973h0.007v0.016v0.033H11.21 c0.007,10.686,4.347,20.367,11.359,27.381c7.009,7.012,16.705,11.359,27.403,11.361v-0.007h0.016h0.033v0.007 c10.686-0.007,20.368-4.348,27.382-11.359c7.011-7.009,11.358-16.702,11.36-27.4h-0.006v-0.016v-0.033h0.006 c-0.006-10.686-4.35-20.372-11.358-27.384C70.396,15.56,60.703,11.213,50.004,11.21L50.004,11.21z"/></g></svg>

changedetectionio/templates/watch-overview.html

@@ -1,155 +1,112 @@
 {% extends 'base.html' %}
 {% block content %}
 {% from '_helpers.jinja' import render_simple_field, render_field %}
-<script src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
-<script src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
+{% from '_pagination.jinja' import pagination %}
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 
 <div class="box">
 
     <form class="pure-form" action="{{ url_for('form_quick_watch_add') }}" method="POST" id="new-watch-form">
-        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <legend>Add a new change detection watch</legend>
             <div id="watch-add-wrapper-zone">
                 <div>
                     {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
-                    {{ render_simple_field(form.tags, value=tags[active_tag].title if active_tag else '', placeholder="watch label / tag") }}
+                    {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch label / tag") }}
                 </div>
                 <div>
                     {{ render_simple_field(form.watch_submit_button, title="Watch this URL!" ) }}
                     {{ render_simple_field(form.edit_and_watch_submit_button, title="Edit first then Watch") }}
                 </div>
             </div>
-            <div id="quick-watch-processor-type">
-                {{ render_simple_field(form.processor, title="Edit first then Watch") }}
-            </div>
-
         </fieldset>
-        <span style="color:#eee; font-size: 80%;"><img alt="Create a shareable link" style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread-white.svg')}}" > Tip: You can also add 'shared' watches. <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Sharing-a-Watch">More info</a></span>
+        <span style="color:#eee; font-size: 80%;"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread-white.svg')}}" /> Tip: You can also add 'shared' watches. <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Sharing-a-Watch">More info</a></a></span>
     </form>
 
     <form class="pure-form" action="{{ url_for('form_watch_list_checkbox_operations') }}" method="POST" id="watch-list-form">
-    <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" >
-    <input type="hidden" id="op_extradata" name="op_extradata" value="" >
+    <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
     <div id="checkbox-operations">
-        <button class="pure-button button-secondary button-xsmall"  name="op" value="pause">Pause</button>
-        <button class="pure-button button-secondary button-xsmall"  name="op" value="unpause">UnPause</button>
-        <button class="pure-button button-secondary button-xsmall"  name="op" value="mute">Mute</button>
-        <button class="pure-button button-secondary button-xsmall"  name="op" value="unmute">UnMute</button>
-        <button class="pure-button button-secondary button-xsmall" name="op" value="recheck">Recheck</button>
-        <button class="pure-button button-secondary button-xsmall" name="op" value="assign-tag" id="checkbox-assign-tag">Tag</button>
-        <button class="pure-button button-secondary button-xsmall" name="op" value="mark-viewed">Mark viewed</button>
-        <button class="pure-button button-secondary button-xsmall" name="op" value="notification-default">Use default notification</button>
-        <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="clear-history">Clear/reset history</button>
-        <button class="pure-button button-secondary button-xsmall" style="background: #dd4242;" name="op" value="delete">Delete</button>
+        <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="pause">Pause</button>
+        <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="unpause">UnPause</button>
+        <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="mute">Mute</button>
+        <button class="pure-button button-secondary button-xsmall" style="font-size: 70%"  name="op" value="unmute">UnMute</button>
+        <button class="pure-button button-secondary button-xsmall" style="font-size: 70%" name="op" value="notification-default">Use default notification</button>
+        <button class="pure-button button-secondary button-xsmall" style="background: #dd4242; font-size: 70%" name="op" value="delete">Delete</button>
     </div>
-    {% if watches|length >= pagination.per_page %}
-        {{ pagination.info }}
-    {% endif %}
-    {% if search_q %}<div id="search-result-info">Searching "<strong><i>{{search_q}}</i></strong>"</div>{% endif %}
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
-        {% for uuid, tag in tags.items() %}
+        {% for tag in tags %}
             {% if tag != "" %}
-                <a href="{{url_for('index', tag=uuid) }}" class="pure-button button-tag {{'active' if active_tag == uuid }}">{{ tag.title }}</a>
+                <a href="{{url_for('index', tag=tag) }}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
             {% endif %}
         {% endfor %}
     </div>
 
-    {% set sort_order = sort_order or 'asc' %}
-    {% set sort_attribute = sort_attribute or 'last_changed'  %}
+    {% set sort_order = request.args.get('order', 'asc') == 'asc' %}
+    {% set sort_attribute = request.args.get('sort', 'last_changed')   %}
     {% set pagination_page = request.args.get('page', 0) %}
 
     <div id="watch-table-wrapper">
-
         <table class="pure-table pure-table-striped watch-table">
             <thead>
             <tr>
-                {% set link_order = "desc" if sort_order  == 'asc' else "asc" %}
-                {% set arrow_span = "" %}
-                <th><input style="vertical-align: middle" type="checkbox" id="check-all" > <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}"  href="{{url_for('index', sort='date_created', order=link_order, tag=active_tag)}}"># <span class='arrow {{link_order}}'></span></a></th>
+                <th><input style="vertical-align: middle" type="checkbox" id="check-all"/> #</th>
                 <th></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order, tag=active_tag)}}">Website <span class='arrow {{link_order}}'></span></a></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order, tag=active_tag)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
-                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order, tag=active_tag)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
+                {% set link_order = "desc" if sort_order else "asc" %}
+                {% set arrow_span = "" %}
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order)}}">Website <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
+                <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
                 <th></th>
             </tr>
             </thead>
             <tbody>
-            {% if not watches|length %}
-            <tr>
-                <td colspan="6">No website watches configured, please add a URL in the box above, or <a href="{{ url_for('import_page')}}" >import a list</a>.</td>
-            </tr>
-            {% endif %}
-            {% for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) %}
+
+            {% set sorted_watches = watches|sort(attribute=sort_attribute, reverse=sort_order) %}
+            {% for watch in sorted_watches %}
+
+            {# WIP for pagination, disabled for now
+              {% if not ( loop.index >= 3 and loop.index <=4) %}{% continue %}{% endif %} -->
+             #}
             <tr id="{{ watch.uuid }}"
-                class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} processor-{{ watch['processor'] }}
+                class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
                 {% if watch.last_notification_error is defined and watch.last_notification_error != False %}error{% endif %}
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
                 {% if watch.newest_history_key| int > watch.last_viewed and watch.history_n>=2 %}unviewed{% endif %}
                 {% if watch.uuid in queued_uuids %}queued{% endif %}">
-                <td class="inline checkbox-uuid" ><input name="uuids"  type="checkbox" value="{{ watch.uuid}} " > <span>{{ loop.index+pagination.skip }}</span></td>
+                <td class="inline checkbox-uuid" ><input name="uuids"  type="checkbox" value="{{ watch.uuid}} "/> <span>{{ loop.index }}</span></td>
                 <td class="inline watch-controls">
                     {% if not watch.paused %}
-                    <a class="state-off" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause" ></a>
+                    <a class="state-off" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause checks" title="Pause checks" class="icon icon-pause"/></a>
                     {% else %}
-                    <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause" ></a>
+                    <a class="state-on" href="{{url_for('index', op='pause', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='play.svg')}}" alt="UnPause checks" title="UnPause checks" class="icon icon-unpause"/></a>
                     {% endif %}
-                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute" ></a>
+                    <a class="link-mute state-{{'on' if watch.notification_muted else 'off'}}" href="{{url_for('index', op='mute', uuid=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='bell-off.svg')}}" alt="Mute notifications" title="Mute notifications" class="icon icon-mute"/></a>
                 </td>
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}"></a>
-                    <a class="link-spread" href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="status-icon icon icon-spread" title="Create a link to share watch config with others" ></a>
+                    <a class="link-spread" href="{{url_for('form_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" class="icon icon-spread" /></a>
 
-                    {% if watch.get_fetch_backend == "html_webdriver"
-                         or (  watch.get_fetch_backend == "system" and system_default_fetcher == 'html_webdriver'  )
-                    %}
-                    <img class="status-icon" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" title="Using a chrome browser" >
-                    {% endif %}
+                    {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
 
-                    {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" >{% endif %}
                     {% if watch.last_error is defined and watch.last_error != False %}
-                    <div class="fetch-error">{{ watch.last_error }}
-
-                        {% if '403' in watch.last_error %}
-                            {% if has_proxies %}
-                                <a href="{{ url_for('settings_page', uuid=watch.uuid) }}#proxies">Try other proxies/location</a>&nbsp;
-                            {% endif %}
-                            <a href="{{ url_for('settings_page', uuid=watch.uuid) }}#proxies">Try adding external proxies/locations</a>
-                        
-                        {% endif %}
-                    </div>
+                    <div class="fetch-error">{{ watch.last_error }}</div>
                     {% endif %}
                     {% if watch.last_notification_error is defined and watch.last_notification_error != False %}
                     <div class="fetch-error notification-error"><a href="{{url_for('notification_logs')}}">{{ watch.last_notification_error }}</a></div>
                     {% endif %}
-
-                    {% if watch['processor'] == 'text_json_diff'  %}
-                        {% if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data']  %}
-                        <div class="ldjson-price-track-offer">Embedded price data detected, follow only price data? <a href="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">Yes</a> <a href="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="">No</a></div>
-                        {% endif %}
-                        {% if watch['track_ldjson_price_data'] == 'accepted' %}
-                        <span class="tracking-ldjson-price-data" title="Automatically following embedded price information"><img src="{{url_for('static_content', group='images', filename='price-tag-icon.svg')}}"  class="status-icon price-follow-tag-icon" > Price</span>
-                        {% endif %}
+                    {% if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data']  %}
+                    <div class="ldjson-price-track-offer">Embedded price data detected, follow only price data? <a href="{{url_for('price_data_follower.accept', uuid=watch.uuid)}}" class="pure-button button-xsmall">Yes</a> <a href="{{url_for('price_data_follower.reject', uuid=watch.uuid)}}" class="">No</a></div>
                     {% endif %}
-
-                    {% if watch['processor'] == 'restock_diff'  %}
-                    <span class="restock-label {{'in-stock' if watch['in_stock'] else 'not-in-stock' }}" title="detecting restock conditions">
-                        <!-- maybe some object watch['processor'][restock_diff] or.. -->
-                        {% if watch['last_checked'] %}
-                            {% if watch['in_stock'] %} In stock {% else %} Not in stock {% endif %}
-                        {% else %}
-                            Not yet checked
-                        {% endif %}
-                    </span>
+                    {% if watch['track_ldjson_price_data'] %}
+                    <span class="tracking-ldjson-price-data">Price</span>
+                    {% endif %}
+                    {% if not active_tag %}
+                    <span class="watch-tag-list">{{ watch.tag}}</span>
                     {% endif %}
-
-
-                    {% for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() %}
-                      <span class="watch-tag-list">{{ watch_tag.title }}</span>
-                    {% endfor %}
-
                 </td>
                 <td class="last-checked">{{watch|format_last_checked_time|safe}}</td>
                 <td class="last-changed">{% if watch.history_n >=2 and watch.last_changed >0 %}
@@ -182,13 +139,16 @@
             {% endif %}
             <li>
                <a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
-                all {% if active_tag%} in "{{tags[active_tag].title}}"{%endif%}</a>
+                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
             </li>
             <li>
                 <a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>
             </li>
         </ul>
-        {{ pagination.links }}
+        {# WIP for pagination, disabled for now
+         {{ pagination(sorted_watches,3, pagination_page) }}
+         #}
+
     </div>
     </form>
 </div>

changedetectionio/tests/conftest.py

@@ -14,16 +14,13 @@ global app
 
 def cleanup(datastore_path):
     # Unlink test output files
-    files = [
-        'count.txt',
-        'endpoint-content.txt'
-        'headers.txt',
-        'headers-testtag.txt',
-        'notification.txt',
-        'secret.txt',
-        'url-watches.json',
-        'output.txt',
-    ]
+    files = ['output.txt',
+             'url-watches.json',
+             'secret.txt',
+             'notification.txt',
+             'count.txt',
+             'endpoint-content.txt'
+                 ]
     for file in files:
         try:
             os.unlink("{}/{}".format(datastore_path, file))

changedetectionio/tests/proxy_list/proxies.json-example

@@ -1,10 +1,10 @@
 {
   "proxy-one": {
-    "label": "Proxy One",
-    "url": "http://squid-one:3128"
+    "label": "One",
+    "url": "http://127.0.0.1:3128"
   },
   "proxy-two": {
-    "label": "Proxy Two",
-    "url": "http://squid-two:3128"
+    "label": "two",
+    "url": "http://127.0.0.1:3129"
   }
 }

changedetectionio/tests/proxy_list/squid-auth.conf

@@ -1,48 +0,0 @@
-acl localnet src 0.0.0.1-0.255.255.255  # RFC 1122 "this" network (LAN)
-acl localnet src 10.0.0.0/8             # RFC 1918 local private network (LAN)
-acl localnet src 100.64.0.0/10          # RFC 6598 shared address space (CGN)
-acl localnet src 169.254.0.0/16         # RFC 3927 link-local (directly plugged) machines
-acl localnet src 172.16.0.0/12          # RFC 1918 local private network (LAN)
-acl localnet src 192.168.0.0/16         # RFC 1918 local private network (LAN)
-acl localnet src fc00::/7               # RFC 4193 local private network range
-acl localnet src fe80::/10              # RFC 4291 link-local (directly plugged) machines
-acl localnet src 159.65.224.174
-acl SSL_ports port 443
-acl Safe_ports port 80          # http
-acl Safe_ports port 21          # ftp
-acl Safe_ports port 443         # https
-acl Safe_ports port 70          # gopher
-acl Safe_ports port 210         # wais
-acl Safe_ports port 1025-65535  # unregistered ports
-acl Safe_ports port 280         # http-mgmt
-acl Safe_ports port 488         # gss-http
-acl Safe_ports port 591         # filemaker
-acl Safe_ports port 777         # multiling http
-acl CONNECT method CONNECT
-
-http_access deny !Safe_ports
-http_access deny CONNECT !SSL_ports
-#http_access allow localhost manager
-http_access deny manager
-#http_access allow localhost
-#http_access allow localnet
-
-auth_param basic program /usr/lib/squid3/basic_ncsa_auth /etc/squid3/passwords
-auth_param basic realm proxy
-acl authenticated proxy_auth REQUIRED
-http_access allow authenticated
-http_access deny all
-
-
-http_port 3128
-coredump_dir /var/spool/squid
-refresh_pattern ^ftp:           1440    20%     10080
-refresh_pattern ^gopher:        1440    0%      1440
-refresh_pattern -i (/cgi-bin/|\?) 0     0%      0
-refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims
-refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims
-refresh_pattern \/InRelease$ 0 0% 0 refresh-ims
-refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims
-refresh_pattern .               0       20%     4320
-logfile_rotate 0
-

changedetectionio/tests/proxy_list/squid-passwords.txt

@@ -1 +0,0 @@
-test:$apr1$xvhFolTA$E/kz5/Rw1ewcyaSUdwqZs.

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -28,7 +28,7 @@ def test_preferred_proxy(client, live_server):
                 "fetch_backend": "html_requests",
                 "headers": "",
                 "proxy": "proxy-two",
-                "tags": "",
+                "tag": "",
                 "url": url,
               },
         follow_redirects=True

changedetectionio/tests/proxy_list/test_select_custom_proxy.py

@@ -1,50 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks
-
-# just make a request, we will grep in the docker logs to see it actually got called
-def test_select_custom(client, live_server):
-    live_server_setup(live_server)
-
-    # Goto settings, add our custom one
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-ignore_whitespace": "y",
-            "application-fetch_backend": "html_requests",
-            "requests-extra_proxies-0-proxy_name": "custom-test-proxy",
-            # test:awesome is set in tests/proxy_list/squid-passwords.txt
-            "requests-extra_proxies-0-proxy_url": "http://test:awesome@squid-custom:3128",
-        },
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    res = client.post(
-        url_for("import_page"),
-        # Because a URL wont show in squid/proxy logs due it being SSLed
-        # Use plain HTTP or a specific domain-name here
-        data={"urls": "https://changedetection.io/CHANGELOG.txt"},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("index"))
-    assert b'Proxy Authentication Required' not in res.data
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    # We should see something via proxy
-    assert b'<div class=""> - 0.' in res.data
-
-    #
-    # Now we should see the request in the container logs for "squid-squid-custom" because it will be the only default
-

changedetectionio/tests/restock/__init__.py

@@ -1,2 +0,0 @@
-"""Tests for the app."""
-

changedetectionio/tests/restock/conftest.py

@@ -1,3 +0,0 @@
-#!/usr/bin/python3
-
-from .. import conftest

changedetectionio/tests/restock/test_restock.py

@@ -1,106 +0,0 @@
-#!/usr/bin/python3
-import os
-import time
-from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
-from changedetectionio.notification import (
-    default_notification_body,
-    default_notification_format,
-    default_notification_title,
-    valid_notification_formats,
-)
-
-
-def set_original_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text<br>
-     <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
-     <div>price: $10.99</div>
-     <div id="sametext">Out of stock</div>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-
-
-def set_back_in_stock_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text<br>
-     <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
-     <div>price: $10.99</div>
-     <div id="sametext">Available!</div>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-# Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
-def test_restock_detection(client, live_server):
-
-    set_original_response()
-    #assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
-
-    time.sleep(1)
-    live_server_setup(live_server)
-    #####################
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http://localhost', 'http://changedet').replace('http', 'json')
-
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title "+default_notification_title,
-              "application-notification_body": "fallback-body "+default_notification_body,
-              "application-notification_format": default_notification_format,
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_webdriver"},
-        follow_redirects=True
-    )
-    # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url
-    test_url = url_for('test_endpoint', _external=True).replace('http://localhost', 'http://changedet')
-
-
-    client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": '', 'processor': 'restock_diff'},
-        follow_redirects=True
-    )
-
-    # Is it correctly show as NOT in stock?
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'not-in-stock' in res.data
-
-    # Is it correctly shown as in stock
-    set_back_in_stock_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'not-in-stock' not in res.data
-
-    # We should have a notification
-    time.sleep(2)
-    assert os.path.isfile("test-datastore/notification.txt")
-    os.unlink("test-datastore/notification.txt")
-
-    # Default behaviour is to only fire notification when it goes OUT OF STOCK -> IN STOCK
-    # So here there should be no file, because we go IN STOCK -> OUT OF STOCK
-    set_original_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    assert not os.path.isfile("test-datastore/notification.txt")

changedetectionio/tests/test_access_control.py

@@ -1,34 +1,18 @@
-from . util import live_server_setup, extract_UUID_from_client
 from flask import url_for
-import time
+from . util import live_server_setup
 
-def test_check_access_control(app, client, live_server):
+def test_check_access_control(app, client):
     # Still doesnt work, but this is closer.
-    live_server_setup(live_server)
 
     with app.test_client(use_cookies=True) as c:
         # Check we don't have any password protection enabled yet.
         res = c.get(url_for("settings_page"))
         assert b"Remove password" not in res.data
 
-        # add something that we can hit via diff page later
-        res = c.post(
-            url_for("import_page"),
-            data={"urls": url_for('test_random_content_endpoint', _external=True)},
-            follow_redirects=True
-        )
-
-        assert b"1 Imported" in res.data
-        time.sleep(2)
-        res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-        assert b'1 watches queued for rechecking.' in res.data
-        time.sleep(2)
-
-        # Enable password check and diff page access bypass
+        # Enable password check.
         res = c.post(
             url_for("settings_page"),
             data={"application-password": "foobar",
-                  "application-shared_diff_access": "True",
                   "requests-time_between_check-minutes": 180,
                   'application-fetch_backend': "html_requests"},
             follow_redirects=True
@@ -38,24 +22,9 @@ def test_check_access_control(app, client, live_server):
 
         # Check we hit the login
         res = c.get(url_for("index"), follow_redirects=True)
-        # Should be logged out
+
         assert b"Login" in res.data
 
-        # The diff page should return something valid when logged out
-        res = client.get(url_for("diff_history_page", uuid="first"))
-        assert b'Random content' in res.data
-
-        # Check wrong password does not let us in
-        res = c.post(
-            url_for("login"),
-            data={"password": "WRONG PASSWORD"},
-            follow_redirects=True
-        )
-
-        assert b"LOG OUT" not in res.data
-        assert b"Incorrect password" in res.data
-
-
         # Menu should not be available yet
         #        assert b"SETTINGS" not in res.data
         #        assert b"BACKUP" not in res.data
@@ -140,25 +109,3 @@ def test_check_access_control(app, client, live_server):
 
         assert b"Password protection enabled" not in res.data
 
-        # Now checking the diff access
-        # Enable password check and diff page access bypass
-        res = c.post(
-            url_for("settings_page"),
-            data={"application-password": "foobar",
-                  # Should be disabled
-#                  "application-shared_diff_access": "True",
-                  "requests-time_between_check-minutes": 180,
-                  'application-fetch_backend': "html_requests"},
-            follow_redirects=True
-        )
-
-        assert b"Password protection enabled." in res.data
-
-        # Check we hit the login
-        res = c.get(url_for("index"), follow_redirects=True)
-        # Should be logged out
-        assert b"Login" in res.data
-
-        # The diff page should return something valid when logged out
-        res = client.get(url_for("diff_history_page", uuid="first"))
-        assert b'Random content' not in res.data

changedetectionio/tests/test_add_replace_remove_filter.py

@@ -1,175 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
-from changedetectionio import html_tools
-
-
-def set_original(excluding=None, add_line=None):
-    test_return_data = """<html>
-     <body>
-     <p>Some initial text</p>
-     <p>So let's see what happens.</p>
-     <p>and a new line!</p>
-     <p>The golden line</p>
-     <p>A BREAK TO MAKE THE TOP LINE STAY AS "REMOVED" OR IT WILL GET COUNTED AS "CHANGED INTO"</p>
-     <p>Something irrelevant</p>          
-     </body>
-     </html>
-    """
-
-    if add_line:
-        c=test_return_data.splitlines()
-        c.insert(5, add_line)
-        test_return_data = "\n".join(c)
-
-    if excluding:
-        output = ""
-        for i in test_return_data.splitlines():
-            if not excluding in i:
-                output += f"{i}\n"
-
-        test_return_data = output
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-def test_setup(client, live_server):
-    live_server_setup(live_server)
-
-def test_check_removed_line_contains_trigger(client, live_server):
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
-    set_original()
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    # Give the thread time to pick it up
-    wait_for_all_checks(client)
-
-    # Goto the edit page, add our ignore text
-    # Add our URL to the import page
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"trigger_text": 'The golden line',
-              "url": test_url,
-              'fetch_backend': "html_requests",
-              'filter_text_removed': 'y'},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-    set_original(excluding='Something irrelevant')
-
-    # A line thats not the trigger should not trigger anything
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'unviewed' not in res.data
-
-    # The trigger line is REMOVED,  this should trigger
-    set_original(excluding='The golden line')
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'unviewed' in res.data
-
-
-    # Now add it back, and we should not get a trigger
-    client.get(url_for("mark_all_viewed"), follow_redirects=True)
-    set_original(excluding=None)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'unviewed' not in res.data
-
-    # Remove it again, and we should get a trigger
-    set_original(excluding='The golden line')
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'unviewed' in res.data
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-
-def test_check_add_line_contains_trigger(client, live_server):
-    #live_server_setup(live_server)
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
-    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://') + "?xxx={{ watch_url }}"
-
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
-              "application-notification_body": 'triggered text was -{{triggered_text}}-',
-              # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-              "application-notification_urls": test_notification_url,
-              "application-minutes_between_check": 180,
-              "application-fetch_backend": "html_requests"
-              },
-        follow_redirects=True
-    )
-    assert b'Settings updated' in res.data
-
-    set_original()
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    # Give the thread time to pick it up
-    wait_for_all_checks(client)
-    # Goto the edit page, add our ignore text
-    # Add our URL to the import page
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"trigger_text": 'Oh yes please',
-              "url": test_url,
-              'fetch_backend': "html_requests",
-              'filter_text_removed': '',
-              'filter_text_added': 'y'},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-    set_original(excluding='Something irrelevant')
-
-    # A line thats not the trigger should not trigger anything
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'unviewed' not in res.data
-
-    # The trigger line is ADDED,  this should trigger
-    set_original(add_line='<p>Oh yes please</p>')
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    res = client.get(url_for("index"))
-    assert b'unviewed' in res.data
-
-    # Takes a moment for apprise to fire
-    time.sleep(3)
-    with open("test-datastore/notification.txt", 'r') as f:
-        response= f.read()
-        assert '-Oh yes please-' in response
-
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_api.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, extract_api_key_from_UI, wait_for_all_checks
+from .util import live_server_setup, extract_api_key_from_UI
 
 import json
 import uuid
@@ -11,10 +11,10 @@ import uuid
 def set_original_response():
     test_return_data = """<html>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
      <div id="sametext">Some text thats the same</div>
      <div id="changetext">Some text that will change</div>
      </body>
@@ -29,10 +29,10 @@ def set_original_response():
 def set_modified_response():
     test_return_data = """<html>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>which has this one new line</p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
      <div id="sametext">Some text thats the same</div>
      <div id="changetext">Some text that changes</div>
      </body>
@@ -53,16 +53,14 @@ def is_valid_uuid(val):
         return False
 
 
-def test_setup(client, live_server):
-    live_server_setup(live_server)
-
 def test_api_simple(client, live_server):
-    #live_server_setup(live_server)
+    live_server_setup(live_server)
 
     api_key = extract_api_key_from_UI(client)
 
     # Create a watch
     set_original_response()
+    watch_uuid = None
 
     # Validate bad URL
     test_url = url_for('test_endpoint', _external=True,
@@ -82,48 +80,39 @@ def test_api_simple(client, live_server):
         headers={'content-type': 'application/json', 'x-api-key': api_key},
         follow_redirects=True
     )
-
-    assert is_valid_uuid(res.json.get('uuid'))
-    watch_uuid = res.json.get('uuid')
+    s = json.loads(res.data)
+    assert is_valid_uuid(s['uuid'])
+    watch_uuid = s['uuid']
     assert res.status_code == 201
 
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Verify its in the list and that recheck worked
     res = client.get(
-        url_for("createwatch", tag="OnE"),
+        url_for("createwatch"),
         headers={'x-api-key': api_key}
     )
-    assert watch_uuid in res.json.keys()
-    before_recheck_info = res.json[watch_uuid]
+    assert watch_uuid in json.loads(res.data).keys()
+    before_recheck_info = json.loads(res.data)[watch_uuid]
     assert before_recheck_info['last_checked'] != 0
     #705 `last_changed` should be zero on the first check
     assert before_recheck_info['last_changed'] == 0
     assert before_recheck_info['title'] == 'My test URL'
 
-    # Check the limit by tag doesnt return anything when nothing found
-    res = client.get(
-        url_for("createwatch", tag="Something else entirely"),
-        headers={'x-api-key': api_key}
-    )
-    assert len(res.json) == 0
-
-    wait_for_all_checks(client)
-
     set_modified_response()
     # Trigger recheck of all ?recheck_all=1
     client.get(
         url_for("createwatch", recheck_all='1'),
         headers={'x-api-key': api_key},
     )
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Did the recheck fire?
     res = client.get(
         url_for("createwatch"),
         headers={'x-api-key': api_key},
     )
-    after_recheck_info = res.json[watch_uuid]
+    after_recheck_info = json.loads(res.data)[watch_uuid]
     assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']
     assert after_recheck_info['last_changed'] != 0
 
@@ -132,11 +121,12 @@ def test_api_simple(client, live_server):
         url_for("watchhistory", uuid=watch_uuid),
         headers={'x-api-key': api_key},
     )
-    assert len(res.json) == 2, "Should have two history entries (the original and the changed)"
+    history = json.loads(res.data)
+    assert len(history) == 2, "Should have two history entries (the original and the changed)"
 
     # Fetch a snapshot by timestamp, check the right one was found
     res = client.get(
-        url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(res.json.keys())[-1]),
+        url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(history.keys())[-1]),
         headers={'x-api-key': api_key},
     )
     assert b'which has this one new line' in res.data
@@ -153,7 +143,7 @@ def test_api_simple(client, live_server):
         url_for("watch", uuid=watch_uuid),
         headers={'x-api-key': api_key}
     )
-    watch = res.json
+    watch = json.loads(res.data)
     # @todo how to handle None/default global values?
     assert watch['history_n'] == 2, "Found replacement history section, which is in its own API"
 
@@ -162,46 +152,10 @@ def test_api_simple(client, live_server):
         url_for("systeminfo"),
         headers={'x-api-key': api_key},
     )
-    assert res.json.get('watch_count') == 1
-    assert res.json.get('uptime') > 0.5
+    info = json.loads(res.data)
+    assert info.get('watch_count') == 1
+    assert info.get('uptime') > 0.5
 
-    ######################################################
-    # Mute and Pause, check it worked
-    res = client.get(
-        url_for("watch", uuid=watch_uuid, paused='paused'),
-        headers={'x-api-key': api_key}
-    )
-    assert b'OK' in res.data
-    res = client.get(
-        url_for("watch", uuid=watch_uuid,  muted='muted'),
-        headers={'x-api-key': api_key}
-    )
-    assert b'OK' in res.data
-    res = client.get(
-        url_for("watch", uuid=watch_uuid),
-        headers={'x-api-key': api_key}
-    )
-    assert res.json.get('paused') == True
-    assert res.json.get('notification_muted') == True
-
-    # Now unpause, unmute
-    res = client.get(
-        url_for("watch", uuid=watch_uuid,  muted='unmuted'),
-        headers={'x-api-key': api_key}
-    )
-    assert b'OK' in res.data
-    res = client.get(
-        url_for("watch", uuid=watch_uuid, paused='unpaused'),
-        headers={'x-api-key': api_key}
-    )
-    assert b'OK' in res.data
-    res = client.get(
-        url_for("watch", uuid=watch_uuid),
-        headers={'x-api-key': api_key}
-    )
-    assert res.json.get('paused') == 0
-    assert res.json.get('notification_muted') == 0
-    ######################################################
 
     # Finally delete the watch
     res = client.delete(
@@ -215,7 +169,9 @@ def test_api_simple(client, live_server):
         url_for("createwatch"),
         headers={'x-api-key': api_key}
     )
-    assert len(res.json) == 0, "Watch list should be empty"
+    watch_list = json.loads(res.data)
+    assert len(watch_list) == 0, "Watch list should be empty"
+
 
 def test_access_denied(client, live_server):
     # `config_api_token_enabled` Should be On by default
@@ -247,99 +203,3 @@ def test_access_denied(client, live_server):
         url_for("createwatch")
     )
     assert res.status_code == 200
-
-    # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-fetch_backend": "html_requests",
-            "application-api_access_token_enabled": "y"
-        },
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
-
-def test_api_watch_PUT_update(client, live_server):
-
-    #live_server_setup(live_server)
-    api_key = extract_api_key_from_UI(client)
-
-    # Create a watch
-    set_original_response()
-    test_url = url_for('test_endpoint', _external=True,
-                       headers={'x-api-key': api_key}, )
-
-    # Create new
-    res = client.post(
-        url_for("createwatch"),
-        data=json.dumps({"url": test_url, 'tag': "One, Two", "title": "My test URL", 'headers': {'cookie': 'yum'} }),
-        headers={'content-type': 'application/json', 'x-api-key': api_key},
-        follow_redirects=True
-    )
-
-    assert res.status_code == 201
-
-
-    # Get a listing, it will be the first one
-    res = client.get(
-        url_for("createwatch"),
-        headers={'x-api-key': api_key}
-    )
-
-    watch_uuid = list(res.json.keys())[0]
-
-    # Check in the edit page just to be sure
-    res = client.get(
-        url_for("edit_page", uuid=watch_uuid),
-    )
-    assert b"cookie: yum" in res.data, "'cookie: yum' found in 'headers' section"
-    assert b"One" in res.data, "Tag 'One' was found"
-    assert b"Two" in res.data, "Tag 'Two' was found"
-
-    # HTTP PUT ( UPDATE an existing watch )
-    res = client.put(
-        url_for("watch", uuid=watch_uuid),
-        headers={'x-api-key': api_key, 'content-type': 'application/json'},
-        data=json.dumps({"title": "new title", 'time_between_check': {'minutes': 552}, 'headers': {'cookie': 'all eaten'}}),
-    )
-    assert res.status_code == 200, "HTTP PUT update was sent OK"
-
-    # HTTP GET single watch, title should be updated
-    res = client.get(
-        url_for("watch", uuid=watch_uuid),
-        headers={'x-api-key': api_key}
-    )
-    assert res.json.get('title') == 'new title'
-
-    # Check in the edit page just to be sure
-    res = client.get(
-        url_for("edit_page", uuid=watch_uuid),
-    )
-    assert b"new title" in res.data, "new title found in edit page"
-    assert b"552" in res.data, "552 minutes found in edit page"
-    assert b"One" in res.data, "Tag 'One' was found"
-    assert b"Two" in res.data, "Tag 'Two' was found"
-    assert b"cookie: all eaten" in res.data, "'cookie: all eaten' found in 'headers' section"
-
-    ######################################################
-
-    # HTTP PUT try a field that doenst exist
-
-    # HTTP PUT an update
-    res = client.put(
-        url_for("watch", uuid=watch_uuid),
-        headers={'x-api-key': api_key, 'content-type': 'application/json'},
-        data=json.dumps({"title": "new title", "some other field": "uh oh"}),
-    )
-
-    assert res.status_code == 400, "Should get error 400 when we give a field that doesnt exist"
-    # Message will come from `flask_expects_json`
-    assert b'Additional properties are not allowed' in res.data
-
-    # Cleanup everything
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_auth.py

@@ -24,7 +24,7 @@ def test_basic_auth(client, live_server):
     # Check form validation
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": "", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -4,13 +4,16 @@ import time
 from flask import url_for
 from .util import live_server_setup, extract_UUID_from_client, extract_api_key_from_UI
 
+from ..html_tools import *
+
+
 def set_response_with_ldjson():
     test_return_data = """<html>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
      <div class="sametext">Some text thats the same</div>
      <div class="changetext">Some text that will change</div>
      <script type="application/ld+json">
@@ -58,22 +61,6 @@ def set_response_with_ldjson():
         f.write(test_return_data)
     return None
 
-def set_response_without_ldjson():
-    test_return_data = """<html>
-       <body>
-     Some initial text<br>
-     <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
-     <div class="sametext">Some text thats the same</div>
-     <div class="changetext">Some text that will change</div>     
-     </body>
-     </html>
-"""
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
 
 # actually only really used by the distll.io importer, but could be handy too
 def test_check_ldjson_price_autodetect(client, live_server):
@@ -119,28 +106,7 @@ def test_check_ldjson_price_autodetect(client, live_server):
         headers={'x-api-key': api_key},
     )
 
-    # Should see this (dont know where the whitespace came from)
-    assert b'"highPrice": 8099900' in res.data
-    # And not this cause its not the ld-json
-    assert b"So let's see what happens" not in res.data
+    # Should just see the price in the API reply
+    assert res.data == b'8097000'
 
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-
-    ##########################################################################################
-    # And we shouldnt see the offer
-    set_response_without_ldjson()
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    time.sleep(3)
-    res = client.get(url_for("index"))
-    assert b'ldjson-price-track-offer' not in res.data
-    
-    ##########################################################################################
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_backend.py

@@ -3,7 +3,7 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 
 sleep_time_for_fetch_thread = 3
 
@@ -11,7 +11,7 @@ sleep_time_for_fetch_thread = 3
 # Basic test to check inscriptus is not adding return line chars, basically works etc
 def test_inscriptus():
     from inscriptis import get_text
-    html_content = "<html><body>test!<br>ok man</body></html>"
+    html_content = "<html><body>test!<br/>ok man</body></html>"
     stripped_text_from_html = get_text(html_content)
     assert stripped_text_from_html == 'test!\nok man'
 
@@ -67,7 +67,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Force recheck
     res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    assert b'1 watches queued for rechecking.' in res.data
+    assert b'1 watches are queued for rechecking.' in res.data
 
     wait_for_all_checks(client)
 
@@ -76,13 +76,12 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'unviewed' in res.data
 
     # #75, and it should be in the RSS feed
-    rss_token = extract_rss_token_from_UI(client)
-    res = client.get(url_for("rss", token=rss_token, _external=True))
+    res = client.get(url_for("rss"))
     expected_url = url_for('test_endpoint', _external=True)
     assert b'<rss' in res.data
 
     # re #16 should have the diff in here too
-    assert b'(into) which has this one new line' in res.data
+    assert b'(into   ) which has this one new line' in res.data
     assert b'CDATA' in res.data
 
     assert expected_url.encode('utf-8') in res.data

changedetectionio/tests/test_block_while_text_present.py

@@ -2,16 +2,16 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from . util import live_server_setup
 from changedetectionio import html_tools
 
 def set_original_ignore_response():
     test_return_data = """<html>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
      </body>
      </html>
 
@@ -24,10 +24,10 @@ def set_original_ignore_response():
 def set_modified_original_ignore_response():
     test_return_data = """<html>
        <body>
-     Some NEW nice initial text<br>
+     Some NEW nice initial text</br>
      <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
      <p>new ignore stuff</p>
      <p>out of stock</p>
      <p>blah</p>
@@ -44,11 +44,11 @@ def set_modified_original_ignore_response():
 def set_modified_response_minus_block_text():
     test_return_data = """<html>
        <body>
-     Some NEW nice initial text<br>
+     Some NEW nice initial text</br>
      <p>Which is across multiple lines</p>
      <p>now on sale $2/p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
      <p>new ignore stuff</p>
      <p>blah</p>
      </body>
@@ -61,7 +61,7 @@ def set_modified_response_minus_block_text():
 
 
 def test_check_block_changedetection_text_NOT_present(client, live_server):
-
+    sleep_time_for_fetch_thread = 3
     live_server_setup(live_server)
     # Use a mix of case in ZzZ to prove it works case-insensitive.
     ignore_text = "out of stoCk\r\nfoobar"
@@ -81,22 +81,19 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"text_should_not_be_present": ignore_text,
-              "url": test_url,
-              'fetch_backend': "html_requests"
-              },
+        data={"text_should_not_be_present": ignore_text, "url": test_url, 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     # Check it saved
     res = client.get(
         url_for("edit_page", uuid="first"),
@@ -107,7 +104,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -120,7 +117,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
@@ -131,7 +128,8 @@ def test_check_block_changedetection_text_NOT_present(client, live_server):
     # Now we set a change where the text is gone, it should now trigger
     set_modified_response_minus_block_text()
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
+
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 

changedetectionio/tests/test_css_selector.py

@@ -12,10 +12,10 @@ def test_setup(live_server):
 def set_original_response():
     test_return_data = """<html>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
      <div id="sametext">Some text thats the same</div>
      <div id="changetext">Some text that will change</div>
      </body>
@@ -29,10 +29,10 @@ def set_original_response():
 def set_modified_response():
     test_return_data = """<html>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>which has this one new line</p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
      <div id="sametext">Some text thats the same</div>
      <div id="changetext">Some text that changes</div>
      </body>
@@ -96,7 +96,7 @@ def test_check_markup_include_filters_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"include_filters": include_filters, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
+        data={"include_filters": include_filters, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -157,7 +157,7 @@ def test_check_multiple_filters(client, live_server):
         url_for("edit_page", uuid="first"),
         data={"include_filters": include_filters,
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               'fetch_backend': "html_requests"},
         follow_redirects=True

changedetectionio/tests/test_element_removal.py

@@ -25,10 +25,10 @@ def set_original_response():
     </ul>
     </nav>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
     <div id="changetext">Some text that will change</div>
      </body>
     <footer>
@@ -54,10 +54,10 @@ def set_modified_response():
     </ul>
     </nav>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
     <div id="changetext">Some text that changes</div>
      </body>
     <footer>
@@ -71,6 +71,7 @@ def set_modified_response():
 
 
 def test_element_removal_output():
+    from changedetectionio import fetch_site_status
     from inscriptis import get_text
 
     # Check text with sub-parts renders correctly
@@ -84,7 +85,7 @@ def test_element_removal_output():
     </ul>
     </nav>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>across multiple lines</p>
      <div id="changetext">Some text that changes</div>
      </body>
@@ -129,7 +130,7 @@ def test_element_removal_full(client, live_server):
         data={
             "subtractive_selectors": subtractive_selectors_data,
             "url": test_url,
-            "tags": "",
+            "tag": "",
             "headers": "",
             "fetch_backend": "html_requests",
         },

changedetectionio/tests/test_errorhandling.py

@@ -59,8 +59,6 @@ def test_http_error_handler(client, live_server):
     _runner_test_http_errors(client, live_server, 404, 'Page not found')
     _runner_test_http_errors(client, live_server, 500, '(Internal server Error) received')
     _runner_test_http_errors(client, live_server, 400, 'Error - Request returned a HTTP error code 400')
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
 
 # Just to be sure error text is properly handled
 def test_DNS_errors(client, live_server):
@@ -83,48 +81,4 @@ def test_DNS_errors(client, live_server):
     assert found_name_resolution_error
     # Should always record that we tried
     assert bytes("just now".encode('utf-8')) in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
 
-# Re 1513
-def test_low_level_errors_clear_correctly(client, live_server):
-    #live_server_setup(live_server)
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write("<html><body><div id=here>Hello world</div></body></html>")
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": "https://dfkjasdkfjaidjfsdajfksdajfksdjfDOESNTEXIST.com"},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    time.sleep(2)
-
-    # We should see the DNS error
-    res = client.get(url_for("index"))
-    found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
-    assert found_name_resolution_error
-
-    # Update with what should work
-    client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-            "url": test_url,
-            "fetch_backend": "html_requests"},
-        follow_redirects=True
-    )
-
-    # Now the error should be gone
-    time.sleep(2)
-    res = client.get(url_for("index"))
-    found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
-    assert not found_name_resolution_error
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_extract_regex.py

@@ -10,10 +10,10 @@ from ..html_tools import *
 def set_original_response():
     test_return_data = """<html>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
      <div id="sametext">Some text thats the same</div>
      <div class="changetext">Some text that will change</div>     
      </body>
@@ -28,12 +28,12 @@ def set_original_response():
 def set_modified_response():
     test_return_data = """<html>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>which has this one new line</p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
      <div id="sametext">Some text thats the same</div>
-     <div class="changetext">Some text that did change ( 1000 online <br> 80 guests<br>  2000 online )</div>
+     <div class="changetext">Some text that did change ( 1000 online <br/> 80 guests<br/>  2000 online )</div>
      <div class="changetext">SomeCase insensitive 3456</div>
      </body>
      </html>
@@ -49,8 +49,8 @@ def set_multiline_response():
     test_return_data = """<html>
        <body>
      
-     <p>Something <br>
-        across 6 billion multiple<br>
+     <p>Something <br/>
+        across 6 billion multiple<br/>
         lines
      </p>
      
@@ -91,7 +91,7 @@ def test_check_filter_multiline(client, live_server):
         data={"include_filters": '',
               'extract_text': '/something.+?6 billion.+?lines/si',
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               'fetch_backend': "html_requests"
               },
@@ -146,7 +146,7 @@ def test_check_filter_and_regex_extract(client, live_server):
         data={"include_filters": include_filters,
               'extract_text': '\d+ online\r\n\d+ guests\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i',
               "url": test_url,
-              "tags": "",
+              "tag": "",
               "headers": "",
               'fetch_backend': "html_requests"
               },

changedetectionio/tests/test_filter_exist_changes.py

@@ -11,10 +11,10 @@ from changedetectionio.model import App
 def set_response_without_filter():
     test_return_data = """<html>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
      <div id="nope-doesnt-exist">Some text thats the same</div>     
      </body>
      </html>
@@ -28,10 +28,10 @@ def set_response_without_filter():
 def set_response_with_filter():
     test_return_data = """<html>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
      <div class="ticket-available">Ticket now on sale!</div>     
      </body>
      </html>
@@ -56,7 +56,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'cinema'},
+        data={"url": test_url, "tag": 'cinema'},
         follow_redirects=True
     )
     assert b"Watch added" in res.data
@@ -89,7 +89,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     notification_form_data.update({
         "url": test_url,
-        "tags": "my tag",
+        "tag": "my tag",
         "title": "my title",
         "headers": "",
         "include_filters": '.ticket-available',
@@ -117,3 +117,18 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
 
     assert 'Ticket now on sale' in notification
     os.unlink("test-datastore/notification.txt")
+
+
+    # Test that if it gets removed, then re-added, we get a notification
+    # Remove the target and re-add it, we should get a new notification
+    set_response_without_filter()
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(3)
+    assert not os.path.isfile("test-datastore/notification.txt")
+
+    set_response_with_filter()
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    time.sleep(3)
+    assert os.path.isfile("test-datastore/notification.txt")
+
+# Also test that the filter was updated after the first one was requested

changedetectionio/tests/test_filter_failure_notification.py

@@ -1,17 +1,18 @@
 import os
 import time
+import re
 from flask import url_for
-from .util import set_original_response, live_server_setup, extract_UUID_from_client, wait_for_all_checks
+from .util import set_original_response, live_server_setup
 from changedetectionio.model import App
 
 
 def set_response_with_filter():
     test_return_data = """<html>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
      <div id="nope-doesnt-exist">Some text thats the same</div>     
      </body>
      </html>
@@ -37,14 +38,14 @@ def run_filter_test(client, content_filter):
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": ''},
+        data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
 
     assert b"Watch added" in res.data
 
     # Give the thread time to pick up the first version
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -71,8 +72,8 @@ def run_filter_test(client, content_filter):
 
     notification_form_data.update({
         "url": test_url,
-        "tags": "my tag",
-        "title": "my title 123",
+        "tag": "my tag",
+        "title": "my title",
         "headers": "",
         "filter_failure_notification_send": 'y',
         "include_filters": content_filter,
@@ -85,57 +86,41 @@ def run_filter_test(client, content_filter):
     )
 
     assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Now the notification should not exist, because we didnt reach the threshold
     assert not os.path.isfile("test-datastore/notification.txt")
 
-    # -2 because we would have checked twice above (on adding and on edit)
-    for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT-2):
+    for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT):
         res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-        wait_for_all_checks(client)
-        assert not os.path.isfile("test-datastore/notification.txt"), f"test-datastore/notification.txt should not exist - Attempt {i}"
+        time.sleep(3)
 
     # We should see something in the frontend
     assert b'Warning, no filters were found' in res.data
 
-    # One more check should trigger it (see -2 above)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
     # Now it should exist and contain our "filter not found" alert
     assert os.path.isfile("test-datastore/notification.txt")
-
+    notification = False
     with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
-
     assert 'CSS/xPath filter was not present in the page' in notification
     assert content_filter.replace('"', '\\"') in notification
 
-    # Remove it and prove that it doesn't trigger when not expected
-    # It should register a change, but no 'filter not found'
+    # Remove it and prove that it doesnt trigger when not expected
     os.unlink("test-datastore/notification.txt")
     set_response_with_filter()
 
-    # Try several times, it should NOT have 'filter not found'
     for i in range(0, App._FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT):
         client.get(url_for("form_watch_checknow"), follow_redirects=True)
-        wait_for_all_checks(client)
+        time.sleep(3)
 
     # It should have sent a notification, but..
     assert os.path.isfile("test-datastore/notification.txt")
-    # but it should not contain the info about a failed filter (because there was none in this case)
+    # but it should not contain the info about the failed filter
     with open("test-datastore/notification.txt", 'r') as f:
         notification = f.read()
     assert not 'CSS/xPath filter was not present in the page' in notification
 
-    # Re #1247 - All tokens got replaced correctly in the notification
-    res = client.get(url_for("index"))
-    uuid = extract_UUID_from_client(client)
-    # UUID is correct, but notification contains tag uuid as UUIID wtf
-    assert uuid in notification
-
     # cleanup for the next
     client.get(
         url_for("form_delete", uuid="all"),
@@ -149,7 +134,7 @@ def test_setup(live_server):
 
 def test_check_include_filters_failure_notification(client, live_server):
     set_original_response()
-    wait_for_all_checks(client)
+    time.sleep(1)
     run_filter_test(client, '#nope-doesnt-exist')
 
 def test_check_xpath_filter_failure_notification(client, live_server):
@@ -157,4 +142,4 @@ def test_check_xpath_filter_failure_notification(client, live_server):
     time.sleep(1)
     run_filter_test(client, '//*[@id="nope-doesnt-exist"]')
 
-# Test that notification is never sent
+# Test that notification is never sent

changedetectionio/tests/test_group.py

@@ -1,322 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name, extract_UUID_from_client
-import os
-
-
-def test_setup(client, live_server):
-    live_server_setup(live_server)
-
-def set_original_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text<br>
-     <p id="only-this">Should be only this</p>
-     <br>
-     <p id="not-this">And never this</p>     
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-def set_modified_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text<br>
-     <p id="only-this">Should be REALLY only this</p>
-     <br>
-     <p id="not-this">And never this</p>     
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-def test_setup_group_tag(client, live_server):
-    #live_server_setup(live_server)
-    set_original_response()
-
-    # Add a tag with some config, import a tag and it should roughly work
-    res = client.post(
-        url_for("tags.form_tag_add"),
-        data={"name": "test-tag"},
-        follow_redirects=True
-    )
-    assert b"Tag added" in res.data
-    assert b"test-tag" in res.data
-
-    res = client.post(
-        url_for("tags.form_tag_edit_submit", uuid="first"),
-        data={"name": "test-tag",
-              "include_filters": '#only-this',
-              "subtractive_selectors": '#not-this'},
-        follow_redirects=True
-    )
-    assert b"Updated" in res.data
-    tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
-    res = client.get(
-        url_for("tags.form_tag_edit", uuid="first")
-    )
-    assert b"#only-this" in res.data
-    assert b"#not-this" in res.data
-
-    # Tag should be setup and ready, now add a watch
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url + "?first-imported=1 test-tag, extra-import-tag"},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    res = client.get(url_for("index"))
-    assert b'import-tag' in res.data
-    assert b'extra-import-tag' in res.data
-
-    res = client.get(
-        url_for("tags.tags_overview_page"),
-        follow_redirects=True
-    )
-    assert b'import-tag' in res.data
-    assert b'extra-import-tag' in res.data
-
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("index"))
-    assert b'Warning, no filters were found' not in res.data
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    assert b'Should be only this' in res.data
-    assert b'And never this' not in res.data
-
-
-    # RSS Group tag filter
-    # An extra one that should be excluded
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url + "?should-be-excluded=1 some-tag"},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-    set_modified_response()
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    rss_token = extract_rss_token_from_UI(client)
-    res = client.get(
-        url_for("rss", token=rss_token, tag="extra-import-tag", _external=True),
-        follow_redirects=True
-    )
-    assert b"should-be-excluded" not in res.data
-    assert res.status_code == 200
-    assert b"first-imported=1" in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_tag_import_singular(client, live_server):
-    #live_server_setup(live_server)
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url + " test-tag, test-tag\r\n"+ test_url + "?x=1 test-tag, test-tag\r\n"},
-        follow_redirects=True
-    )
-    assert b"2 Imported" in res.data
-
-    res = client.get(
-        url_for("tags.tags_overview_page"),
-        follow_redirects=True
-    )
-    # Should be only 1 tag because they both had the same
-    assert res.data.count(b'test-tag') == 1
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_tag_add_in_ui(client, live_server):
-    #live_server_setup(live_server)
-#
-    res = client.post(
-        url_for("tags.form_tag_add"),
-        data={"name": "new-test-tag"},
-        follow_redirects=True
-    )
-    assert b"Tag added" in res.data
-    assert b"new-test-tag" in res.data
-
-    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
-    assert b'All tags deleted' in res.data
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_group_tag_notification(client, live_server):
-    #live_server_setup(live_server)
-    set_original_response()
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'test-tag, other-tag'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
-    notification_form_data = {"notification_urls": notification_url,
-                              "notification_title": "New GROUP TAG ChangeDetection.io Notification - {{watch_url}}",
-                              "notification_body": "BASE URL: {{base_url}}\n"
-                                                   "Watch URL: {{watch_url}}\n"
-                                                   "Watch UUID: {{watch_uuid}}\n"
-                                                   "Watch title: {{watch_title}}\n"
-                                                   "Watch tag: {{watch_tag}}\n"
-                                                   "Preview: {{preview_url}}\n"
-                                                   "Diff URL: {{diff_url}}\n"
-                                                   "Snapshot: {{current_snapshot}}\n"
-                                                   "Diff: {{diff}}\n"
-                                                   "Diff Added: {{diff_added}}\n"
-                                                   "Diff Removed: {{diff_removed}}\n"
-                                                   "Diff Full: {{diff_full}}\n"
-                                                   ":-)",
-                              "notification_screenshot": True,
-                              "notification_format": "Text",
-                              "title": "test-tag"}
-
-    res = client.post(
-        url_for("tags.form_tag_edit_submit", uuid=get_UUID_for_tag_name(client, name="test-tag")),
-        data=notification_form_data,
-        follow_redirects=True
-    )
-    assert b"Updated" in res.data
-
-    wait_for_all_checks(client)
-
-    set_modified_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(3)
-
-    assert os.path.isfile("test-datastore/notification.txt")
-
-    # Verify what was sent as a notification, this file should exist
-    with open("test-datastore/notification.txt", "r") as f:
-        notification_submission = f.read()
-    os.unlink("test-datastore/notification.txt")
-
-    # Did we see the URL that had a change, in the notification?
-    # Diff was correctly executed
-    assert test_url in notification_submission
-    assert ':-)' in notification_submission
-    assert "Diff Full: Some initial text" in notification_submission
-    assert "New GROUP TAG ChangeDetection.io" in notification_submission
-    assert "test-tag" in notification_submission
-    assert "other-tag" in notification_submission
-
-    #@todo Test that multiple notifications fired
-    #@todo Test that each of multiple notifications with different settings
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_limit_tag_ui(client, live_server):
-    #live_server_setup(live_server)
-
-    test_url = url_for('test_endpoint', _external=True)
-    urls=[]
-
-    for i in range(20):
-        urls.append(test_url+"?x="+str(i)+" test-tag")
-
-    for i in range(20):
-        urls.append(test_url+"?non-grouped="+str(i))
-
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": "\r\n".join(urls)},
-        follow_redirects=True
-    )
-
-    assert b"40 Imported" in res.data
-
-    res = client.get(url_for("index"))
-    assert b'test-tag' in res.data
-
-    # All should be here
-    assert res.data.count(b'processor-text_json_diff') == 40
-
-    tag_uuid = get_UUID_for_tag_name(client, name="test-tag")
-
-    res = client.get(url_for("index", tag=tag_uuid))
-
-    # Just a subset should be here
-    assert b'test-tag' in res.data
-    assert res.data.count(b'processor-text_json_diff') == 20
-    assert b"object at" not in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
-    assert b'All tags deleted' in res.data
-def test_clone_tag_on_import(client, live_server):
-    #live_server_setup(live_server)
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url + " test-tag, another-tag\r\n"},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-
-    res = client.get(url_for("index"))
-    assert b'test-tag' in res.data
-    assert b'another-tag' in res.data
-
-    watch_uuid = extract_UUID_from_client(client)
-    res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
-
-    assert b'Cloned' in res.data
-    # 2 times plus the top link to tag
-    assert res.data.count(b'test-tag') == 3
-    assert res.data.count(b'another-tag') == 3
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_clone_tag_on_quickwatchform_add(client, live_server):
-    #live_server_setup(live_server)
-
-    test_url = url_for('test_endpoint', _external=True)
-
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": ' test-tag, another-tag      '},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    res = client.get(url_for("index"))
-    assert b'test-tag' in res.data
-    assert b'another-tag' in res.data
-
-    watch_uuid = extract_UUID_from_client(client)
-    res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
-
-    assert b'Cloned' in res.data
-    # 2 times plus the top link to tag
-    assert res.data.count(b'test-tag') == 3
-    assert res.data.count(b'another-tag') == 3
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
-    assert b'All tags deleted' in res.data

changedetectionio/tests/test_html_to_text.py

@@ -6,11 +6,11 @@ from ..html_tools import html_to_text
 def test_html_to_text_func():
     test_html = """<html>
        <body>
-     Some initial text<br>
+     Some initial text</br>
      <p>Which is across multiple lines</p>
      <a href="/first_link"> More Text </a>
-     <br>
-     So let's see what happens.  <br>
+     </br>
+     So let's see what happens.  </br>
      <a href="second_link.com"> Even More Text </a>
      </body>
      </html>
@@ -21,7 +21,7 @@ def test_html_to_text_func():
 
     no_links_text = \
         "Some initial text\n\nWhich is across multiple " \
-        "lines\n\nMore Text\nSo let's see what happens.\nEven More Text"
+        "lines\n\nMore Text So let's see what happens. Even More Text"
 
     # check that no links are in the extracted text
     assert text_content == no_links_text
@@ -31,7 +31,7 @@ def test_html_to_text_func():
 
     links_text = \
         "Some initial text\n\nWhich is across multiple lines\n\n[ More Text " \
-        "](/first_link)\nSo let's see what happens.\n[ Even More Text ]" \
+        "](/first_link) So let's see what happens. [ Even More Text ]" \
         "(second_link.com)"
 
     # check that links are present in the extracted text