tweak distill importer

.dockerignore

@@ -1,18 +1,2 @@
 .git
 .github
-changedetectionio/processors/__pycache__
-changedetectionio/api/__pycache__
-changedetectionio/model/__pycache__
-changedetectionio/blueprint/price_data_follower/__pycache__
-changedetectionio/blueprint/tags/__pycache__
-changedetectionio/blueprint/__pycache__
-changedetectionio/blueprint/browser_steps/__pycache__
-changedetectionio/fetchers/__pycache__
-changedetectionio/tests/visualselector/__pycache__
-changedetectionio/tests/restock/__pycache__
-changedetectionio/tests/__pycache__
-changedetectionio/tests/fetchers/__pycache__
-changedetectionio/tests/unit/__pycache__
-changedetectionio/tests/proxy_list/__pycache__
-changedetectionio/__pycache__
-

.github/test/Dockerfile-alpine

@@ -2,7 +2,7 @@
 # Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)
 # Some packages wont install via pypi because they dont have a wheel available under this architecture.
 
-FROM ghcr.io/linuxserver/baseimage-alpine:3.18
+FROM ghcr.io/linuxserver/baseimage-alpine:3.16
 ENV PYTHONUNBUFFERED=1
 
 COPY requirements.txt /requirements.txt
@@ -26,6 +26,6 @@ RUN \
     py3-pip && \
   echo "**** pip3 install test of changedetection.io ****" && \
   pip3 install -U pip wheel setuptools && \
-  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.18/ -r /requirements.txt && \
+  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.16/ -r /requirements.txt && \
   apk del --purge \
     build-dependencies

.github/workflows/codeql-analysis.yml

@@ -30,11 +30,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v2
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v1
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -45,7 +45,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v1
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -59,4 +59,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v1

.github/workflows/containers.yml

@@ -39,11 +39,11 @@ jobs:
     # Or if we are in a tagged release scenario.
     if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''
     steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v4
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
         with:
-          python-version: 3.11
+          python-version: 3.9
 
       - name: Install dependencies
         run: |
@@ -58,27 +58,27 @@ jobs:
           echo ${{ github.ref }} > changedetectionio/tag.txt
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v1
         with:
           image: tonistiigi/binfmt:latest
           platforms: all
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Login to Docker Hub Container Registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v1
         with:
           install: true
           version: latest
@@ -88,17 +88,16 @@ jobs:
       - name: Build and push :dev
         id: docker_build
         if: ${{ github.ref }} == "refs/heads/master"
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v2
         with:
           context: ./
           file: ./Dockerfile
           push: true
           tags: |
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
 # Looks like this was disabled
 #          provenance: false
 
@@ -106,7 +105,7 @@ jobs:
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v2
         with:
           context: ./
           file: ./Dockerfile
@@ -116,12 +115,19 @@ jobs:
             ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
             ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
             ghcr.io/dgtlmoon/changedetection.io:latest
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
 # Looks like this was disabled
 #          provenance: false
 
       - name: Image digest
         run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
 
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-

.github/workflows/test-container-build.yml

@@ -24,22 +24,22 @@ jobs:
   test-container-build:
     runs-on: ubuntu-latest
     steps:
-        - uses: actions/checkout@v4
-        - name: Set up Python 3.11
-          uses: actions/setup-python@v4
+        - uses: actions/checkout@v2
+        - name: Set up Python 3.9
+          uses: actions/setup-python@v2
           with:
-            python-version: 3.11
+            python-version: 3.9
 
         # Just test that the build works, some libraries won't compile on ARM/rPi etc
         - name: Set up QEMU
-          uses: docker/setup-qemu-action@v3
+          uses: docker/setup-qemu-action@v1
           with:
             image: tonistiigi/binfmt:latest
             platforms: all
 
         - name: Set up Docker Buildx
           id: buildx
-          uses: docker/setup-buildx-action@v3
+          uses: docker/setup-buildx-action@v1
           with:
             install: true
             version: latest
@@ -49,7 +49,7 @@ jobs:
         # Check we can still build under alpine/musl
         - name: Test that the docker containers can build (musl via alpine check)
           id: docker_build_musl
-          uses: docker/build-push-action@v5
+          uses: docker/build-push-action@v2
           with:
             context: ./
             file: ./.github/test/Dockerfile-alpine
@@ -57,12 +57,12 @@ jobs:
 
         - name: Test that the docker containers can build
           id: docker_build
-          uses: docker/build-push-action@v5
+          uses: docker/build-push-action@v2
           # https://github.com/docker/build-push-action#customizing
           with:
             context: ./
             file: ./Dockerfile
-            platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
+            platforms: linux/arm/v7,linux/arm/v6,linux/amd64,linux/arm64,
             cache-from: type=local,src=/tmp/.buildx-cache
             cache-to: type=local,dest=/tmp/.buildx-cache
 

.github/workflows/test-only.yml

@@ -7,13 +7,13 @@ jobs:
   test-application:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
 
       # Mainly just for link/flake8
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v2
         with:
-          python-version: '3.11'
+          python-version: '3.10'
 
       - name: Lint with flake8
         run: |
@@ -29,23 +29,17 @@ jobs:
           docker network create changedet-network
 
           # Selenium+browserless
-          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
-          docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.60-chrome-stable
+          docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome-debug:3.141.59
+          docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.53-chrome-stable
 
       - name: Build changedetection.io container for testing
         run: |         
           # Build a changedetection.io container and start testing inside
           docker build . -t test-changedetectionio
-          # Debug info
-          docker run test-changedetectionio  bash -c 'pip list'
-
-      - name: Spin up ancillary SMTP+Echo message test server
-        run: |
-          # Debug SMTP server/echo message back server
-          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py' 
 
       - name: Test built container with pytest
         run: |
+          
           # Unit tests
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           
@@ -64,16 +58,11 @@ jobs:
           # Settings headers playwright tests - Call back in from Browserless, check headers
           docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
           docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'          
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
           
           # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it
           docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 
-      - name: Test SMTP notification mime types
-        run: |
-          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
-          docker run --rm  --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
-
       - name: Test with puppeteer fetcher and disk cache
         run: |
           docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
@@ -83,7 +72,6 @@ jobs:
         run: |
           cd changedetectionio
           ./run_proxy_tests.sh
-          # And again with PLAYWRIGHT_DRIVER_URL=..
           cd ..
 
       - name: Test changedetection.io container starts+runs basically without error
@@ -99,4 +87,4 @@ jobs:
 
 #export WEBDRIVER_URL=http://localhost:4444/wd/hub
 #pytest tests/fetchers/test_content.py
-#pytest tests/test_errorhandling.py
+#pytest tests/test_errorhandling.py

.github/workflows/test-pip-build.yml

@@ -11,12 +11,12 @@ jobs:
   test-pip-build-basics:
     runs-on: ubuntu-latest
     steps:
-        - uses: actions/checkout@v4
+        - uses: actions/checkout@v2
 
-        - name: Set up Python 3.11
-          uses: actions/setup-python@v4
+        - name: Set up Python 3.9
+          uses: actions/setup-python@v2
           with:
-            python-version: 3.11
+            python-version: 3.9
 
 
         - name: Test that the basic pip built package runs without error

Dockerfile

@@ -1,5 +1,5 @@
 # pip dependencies install stage
-FROM python:3.11-slim-bookworm as builder
+FROM python:3.10-slim-bullseye as builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -25,13 +25,14 @@ RUN pip install --target=/dependencies -r /requirements.txt
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN pip install --target=/dependencies playwright~=1.39 \
+RUN pip install --target=/dependencies playwright~=1.27.1 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.11-slim-bookworm
+FROM python:3.10-slim-bullseye
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl1.1 \
     libxslt1.1 \
     # For pdftohtml
     poppler-utils \

MANIFEST.in

@@ -13,7 +13,3 @@ include changedetection.py
 global-exclude *.pyc
 global-exclude node_modules
 global-exclude venv
-
-global-exclude test-datastore
-global-exclude changedetection.io*dist-info
-global-exclude changedetectionio/tests/proxy_socks5/test-datastore

README-pip.md

@@ -2,44 +2,19 @@
 
 Live your data-life pro-actively, track website content changes and receive notifications via Discord, Email, Slack, Telegram and 70+ more
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring, list of websites with changes"  title="Self-hosted web page change monitoring, list of websites with changes"  />](https://changedetection.io)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://changedetection.io)
 
 
 [**Don't have time? Let us host it for you! try our extremely affordable subscription use our proxies and support!**](https://changedetection.io) 
 
 
-### Target specific parts of the webpage using the Visual Selector tool.
-
-Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
-
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Select parts and elements of a web page to monitor for changes"  title="Select parts and elements of a web page to monitor for changes" />](https://changedetection.io?src=pip)
-
-### Easily see what changed, examine by word, line, or individual character.
-
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://changedetection.io?src=pip)
-
-
-### Perform interactive browser steps
-
-Fill in text boxes, click buttons and more, setup your changedetection scenario. 
-
-Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
-
-[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more"  title="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more" />](https://changedetection.io?src=pip)
-
-After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
-Requires Playwright to be enabled.
-
-
-### Example use cases
+#### Example use cases
 
 - Products and services have a change in pricing
 - _Out of stock notification_ and _Back In stock notification_
-- Monitor and track PDF file changes, know when a PDF file has text changes.
 - Governmental department updates (changes are often only on their websites)
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
-- Discogs restock alerts and monitoring
 - Realestate listing changes
 - Know when your favourite whiskey is on sale, or other special deals are announced before anyone else
 - COVID related news from government websites
@@ -52,34 +27,18 @@ Requires Playwright to be enabled.
 - Create RSS feeds based on changes in web content
 - Monitor HTML source code for unexpected changes, strengthen your PCI compliance
 - You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
-- Get notified when certain keywords appear in Twitter search results
-- Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
-- Get alerts when new job positions are open on Bamboo HR and other job platforms
-- Website defacement monitoring
-- Pokémon Card Restock Tracker / Pokémon TCG Tracker
-- RegTech - stay ahead of regulatory changes, regulatory compliance
 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 
 #### Key Features
 
 - Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
-- Target elements with xPath(1.0) and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
+- Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
 - Switch between fast non-JS and Chrome JS based "fetchers"
-- Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)
 - Easily specify how often a site should be checked
 - Execute JS before extracting text (Good for logging in, see examples in the UI!)
 - Override Request Headers, Specify `POST` or `GET` and other methods
 - Use the "Visual Selector" to help target specific elements
-- Configurable [proxy per watch](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration)
-- Send a screenshot with the notification when a change is detected in the web page
-
-We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
-
-[Oxylabs](https://oxylabs.go2cloud.org/SH2d) is also an excellent proxy provider and well worth using, they offer Residental, ISP, Rotating and many other proxy types to suit your project. 
-
-Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
-
 
 
 ```bash
@@ -95,5 +54,5 @@ $ changedetection.io -d /path/to/empty/data/dir -p 5000
 
 Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
 
-See https://changedetection.io for more information.
+See https://github.com/dgtlmoon/changedetection.io for more information.
 

README.md

@@ -5,7 +5,7 @@
 _Live your data-life pro-actively._ 
 
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web site page change monitoring"  title="Self-hosted web site page change monitoring"  />](https://changedetection.io?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://changedetection.io?src=github)
 
 [![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)
 
@@ -22,7 +22,7 @@ _Live your data-life pro-actively._
 
 Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Select parts and elements of a web page to monitor for changes"  title="Select parts and elements of a web page to monitor for changes" />](https://changedetection.io?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://changedetection.io?src=github)
 
 ### Easily see what changed, examine by word, line, or individual character.
 
@@ -35,7 +35,7 @@ Fill in text boxes, click buttons and more, setup your changedetection scenario.
 
 Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
 
-[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more"  title="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more" />](https://changedetection.io?src=github)
+[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Website change detection with interactive browser steps, login, cookies etc" />](https://changedetection.io?src=github)
 
 After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
 Requires Playwright to be enabled.
@@ -67,14 +67,13 @@ Requires Playwright to be enabled.
 - Get alerts when new job positions are open on Bamboo HR and other job platforms
 - Website defacement monitoring
 - Pokémon Card Restock Tracker / Pokémon TCG Tracker
-- RegTech - stay ahead of regulatory changes, regulatory compliance
 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 
 #### Key Features
 
 - Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
-- Target elements with xPath(1.0) and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
+- Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
 - Switch between fast non-JS and Chrome JS based "fetchers"
 - Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)
 - Easily specify how often a site should be checked
@@ -86,8 +85,6 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W
 
 We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
 
-[Oxylabs](https://oxylabs.go2cloud.org/SH2d) is also an excellent proxy provider and well worth using, they offer Residental, ISP, Rotating and many other proxy types to suit your project. 
-
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
 ## Installation
@@ -147,7 +144,7 @@ See the wiki for more information https://github.com/dgtlmoon/changedetection.io
 
 ## Filters
 
-XPath(1.0), JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. 
+XPath, JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. 
 (We support LXML `re:test`, `re:match` and `re:replace`.)
 
 ## Notifications
@@ -186,7 +183,7 @@ This will re-parse the JSON and apply formatting to the text, making it super ea
 
 ### JSONPath or jq?
 
-For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more specific information on jq.
+For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more specifc information on jq.
 
 One big advantage of `jq` is that you can use logic in your JSON filter, such as filters to only show items that have a value greater than/less than etc.
 
@@ -226,19 +223,12 @@ The application also supports notifying you that it can follow this information
 
 ## Proxy Configuration
 
-See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [Bright Data proxy services where possible](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support) and [Oxylabs](https://oxylabs.go2cloud.org/SH2d) proxy services.
+See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [BrightData proxy services where possible]( https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support)
 
 ## Raspberry Pi support?
 
 Raspberry Pi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! See the wiki for [details](https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver)
 
-## Import support
-
-Easily [import your list of websites to watch for changes in Excel .xslx file format](https://changedetection.io/tutorial/how-import-your-website-change-detection-lists-excel), or paste in lists of website URLs as plaintext. 
-
-Excel import is recommended - that way you can better organise tags/groups of websites and other features.
-
-
 ## API Support
 
 Supports managing the website watch list [via our API](https://changedetection.io/docs/api_v1/index.html)

changedetectionio/__init__.py

@@ -38,9 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.45.7.3'
-
-from changedetectionio.store import BASE_URL_NOT_SET_TEXT
+__version__ = '0.43'
 
 datastore = None
 
@@ -105,10 +103,6 @@ def get_darkmode_state():
     css_dark_mode = request.cookies.get('css_dark_mode', 'false')
     return 'true' if css_dark_mode and strtobool(css_dark_mode) else 'false'
 
-@app.template_global()
-def get_css_version():
-    return __version__
-
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
 # running or something similar.
 @app.template_filter('format_last_checked_time')
@@ -190,6 +184,7 @@ class User(flask_login.UserMixin):
 
     pass
 
+
 def login_optionally_required(func):
     @wraps(func)
     def decorated_view(*args, **kwargs):
@@ -202,6 +197,7 @@ def login_optionally_required(func):
         # Permitted
         elif request.endpoint == 'diff_history_page' and datastore.data['settings']['application'].get('shared_diff_access'):
             return func(*args, **kwargs)
+
         elif request.method in flask_login.config.EXEMPT_METHODS:
             return func(*args, **kwargs)
         elif app.config.get('LOGIN_DISABLED'):
@@ -359,11 +355,11 @@ def changedetection_app(config=None, datastore_o=None):
 
                 # Include a link to the diff page, they will have to login here to see if password protection is enabled.
                 # Description is the page you watch, link takes you to the diff JS UI page
-                # Dict val base_url will get overriden with the env var if it is set.
-                ext_base_url = datastore.data['settings']['application'].get('active_base_url')
+                base_url = datastore.data['settings']['application']['base_url']
+                if base_url == '':
+                    base_url = "<base-url-env-var-not-set>"
 
-                # Because we are called via whatever web server, flask should figure out the right path (
-                diff_link = {'href': url_for('diff_history_page', uuid=watch['uuid'], _external=True)}
+                diff_link = {'href': "{}{}".format(base_url, url_for('diff_history_page', uuid=watch['uuid']))}
 
                 fe.link(link=diff_link)
 
@@ -420,18 +416,11 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
-        with_errors = request.args.get('with_errors') == "1"
-        errored_count = 0
         search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
         for uuid, watch in datastore.data['watching'].items():
-            if with_errors and not watch.get('last_error'):
-                continue
-
             if limit_tag and not limit_tag in watch['tags']:
                     continue
-            if watch.get('last_error'):
-                errored_count += 1
-                
+
             if search_q:
                 if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():
                     sorted_watches.append(watch)
@@ -453,7 +442,6 @@ def changedetection_app(config=None, datastore_o=None):
                                  active_tag=limit_tag,
                                  app_rss_token=datastore.data['settings']['application']['rss_access_token'],
                                  datastore=datastore,
-                                 errored_count=errored_count,
                                  form=form,
                                  guid=datastore.data['app_guid'],
                                  has_proxies=datastore.proxy_list,
@@ -634,6 +622,7 @@ def changedetection_app(config=None, datastore_o=None):
 
             if request.args.get('unpause_on_save'):
                 extra_update_obj['paused'] = False
+
             # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
             # Assume we use the default value, unless something relevant is different, then use the form value
             # values could be None, 0 etc.
@@ -719,11 +708,12 @@ def changedetection_app(config=None, datastore_o=None):
 
             # Only works reliably with Playwright
             visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver
+
             output = render_template("edit.html",
                                      available_processors=processors.available_processors(),
                                      browser_steps_config=browser_step_ui_config,
+                                     current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
-                                     extra_title=f" - Edit - {watch.label}",
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
                                      has_empty_checktime=using_default_check_time,
@@ -812,6 +802,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         output = render_template("settings.html",
                                  form=form,
+                                 current_base_url = datastore.data['settings']['application']['base_url'],
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
                                  api_key=datastore.data['settings']['application'].get('api_access_token'),
                                  emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
@@ -826,7 +817,6 @@ def changedetection_app(config=None, datastore_o=None):
         from . import forms
 
         if request.method == 'POST':
-
             from .importer import import_url_list, import_distill_io_json
 
             # URL List import
@@ -850,32 +840,11 @@ def changedetection_app(config=None, datastore_o=None):
                 for uuid in d_importer.new_uuids:
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
 
-            # XLSX importer
-            if request.files and request.files.get('xlsx_file'):
-                file = request.files['xlsx_file']
-                from .importer import import_xlsx_wachete, import_xlsx_custom
-
-                if request.values.get('file_mapping') == 'wachete':
-                    w_importer = import_xlsx_wachete()
-                    w_importer.run(data=file, flash=flash, datastore=datastore)
-                else:
-                    w_importer = import_xlsx_custom()
-                    # Building mapping of col # to col # type
-                    map = {}
-                    for i in range(10):
-                        c = request.values.get(f"custom_xlsx[col_{i}]")
-                        v = request.values.get(f"custom_xlsx[col_type_{i}]")
-                        if c and v:
-                            map[int(c)] = v
-
-                    w_importer.import_profile = map
-                    w_importer.run(data=file, flash=flash, datastore=datastore)
-
-                for uuid in w_importer.new_uuids:
-                    update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
 
+        form = forms.importForm(formdata=request.form if request.method == 'POST' else None,
+#                               data=default,
+                               )
         # Could be some remaining, or we could be on GET
-        form = forms.importForm(formdata=request.form if request.method == 'POST' else None)
         output = render_template("import.html",
                                  form=form,
                                  import_url_list_remaining="\n".join(remaining_urls),
@@ -889,10 +858,7 @@ def changedetection_app(config=None, datastore_o=None):
     def mark_all_viewed():
 
         # Save the current newest history as the most recently viewed
-        with_errors = request.args.get('with_errors') == "1"
         for watch_uuid, watch in datastore.data['watching'].items():
-            if with_errors and not watch.get('last_error'):
-                continue
             datastore.set_last_viewed(watch_uuid, int(time.time()))
 
         return redirect(url_for('index'))
@@ -948,29 +914,21 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Read as binary and force decode as UTF-8
         # Windows may fail decode in python if we just use 'r' mode (chardet decode exception)
-        from_version = request.args.get('from_version')
-        from_version_index = -2 # second newest
-        if from_version and from_version in dates:
-            from_version_index = dates.index(from_version)
-        else:
-            from_version = dates[from_version_index]
+        try:
+            newest_version_file_contents = watch.get_history_snapshot(dates[-1])
+        except Exception as e:
+            newest_version_file_contents = "Unable to read {}.\n".format(dates[-1])
+
+        previous_version = request.args.get('previous_version')
+        previous_timestamp = dates[-2]
+        if previous_version:
+            previous_timestamp = previous_version
 
         try:
-            from_version_file_contents = watch.get_history_snapshot(dates[from_version_index])
+            previous_version_file_contents = watch.get_history_snapshot(previous_timestamp)
         except Exception as e:
-            from_version_file_contents = "Unable to read to-version at index{}.\n".format(dates[from_version_index])
+            previous_version_file_contents = "Unable to read {}.\n".format(previous_timestamp)
 
-        to_version = request.args.get('to_version')
-        to_version_index = -1
-        if to_version and to_version in dates:
-            to_version_index = dates.index(to_version)
-        else:
-            to_version = dates[to_version_index]
-
-        try:
-            to_version_file_contents = watch.get_history_snapshot(dates[to_version_index])
-        except Exception as e:
-            to_version_file_contents = "Unable to read to-version at index{}.\n".format(dates[to_version_index])
 
         screenshot_url = watch.get_screenshot()
 
@@ -986,24 +944,22 @@ def changedetection_app(config=None, datastore_o=None):
 
         output = render_template("diff.html",
                                  current_diff_url=watch['url'],
-                                 from_version=str(from_version),
-                                 to_version=str(to_version),
+                                 current_previous_version=str(previous_version),
                                  extra_stylesheets=extra_stylesheets,
-                                 extra_title=f" - Diff - {watch.label}",
+                                 extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
                                  extract_form=extract_form,
                                  is_html_webdriver=is_html_webdriver,
                                  last_error=watch['last_error'],
                                  last_error_screenshot=watch.get_error_snapshot(),
                                  last_error_text=watch.get_error_text(),
                                  left_sticky=True,
-                                 newest=to_version_file_contents,
+                                 newest=newest_version_file_contents,
                                  newest_version_timestamp=dates[-1],
                                  password_enabled_and_share_is_off=password_enabled_and_share_is_off,
-                                 from_version_file_contents=from_version_file_contents,
-                                 to_version_file_contents=to_version_file_contents,
+                                 previous=previous_version_file_contents,
                                  screenshot=screenshot_url,
                                  uuid=uuid,
-                                 versions=dates, # All except current/last
+                                 versions=dates[:-1], # All except current/last
                                  watch_a=watch
                                  )
 
@@ -1212,7 +1168,8 @@ def changedetection_app(config=None, datastore_o=None):
             # These files should be in our subdirectory
             try:
                 # set nocache, set content-type
-                response = make_response(send_from_directory(os.path.join(datastore_o.datastore_path, filename), "elements.json"))
+                watch_dir = datastore_o.datastore_path + "/" + filename
+                response = make_response(send_from_directory(filename="elements.json", directory=watch_dir, path=watch_dir + "/elements.json"))
                 response.headers['Content-type'] = 'application/json'
                 response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
                 response.headers['Pragma'] = 'no-cache'
@@ -1300,8 +1257,6 @@ def changedetection_app(config=None, datastore_o=None):
         # Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True})))
         tag = request.args.get('tag')
         uuid = request.args.get('uuid')
-        with_errors = request.args.get('with_errors') == "1"
-
         i = 0
 
         running_uuids = []
@@ -1313,12 +1268,10 @@ def changedetection_app(config=None, datastore_o=None):
                 update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
             i = 1
 
-        elif tag:
+        elif tag != None:
             # Items that have this current tag
             for watch_uuid, watch in datastore.data['watching'].items():
-                if tag in watch.get('tags', {}):
-                    if with_errors and not watch.get('last_error'):
-                        continue
+                if (tag != None and tag in watch.get('tags', {})):
                     if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                         update_q.put(
                             queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False})
@@ -1329,11 +1282,8 @@ def changedetection_app(config=None, datastore_o=None):
             # No tag, no uuid, add everything.
             for watch_uuid, watch in datastore.data['watching'].items():
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                    if with_errors and not watch.get('last_error'):
-                        continue
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
                     i += 1
-
         flash("{} watches queued for rechecking.".format(i))
         return redirect(url_for('index', tag=tag))
 
@@ -1480,28 +1430,6 @@ def changedetection_app(config=None, datastore_o=None):
         # paste in etc
         return redirect(url_for('index'))
 
-    @app.route("/highlight_submit_ignore_url", methods=['POST'])
-    @login_optionally_required
-    def highlight_submit_ignore_url():
-        import re
-        mode = request.form.get('mode')
-        selection = request.form.get('selection')
-
-        uuid = request.args.get('uuid','')
-        if datastore.data["watching"].get(uuid):
-            if mode == 'exact':
-                for l in selection.splitlines():
-                    datastore.data["watching"][uuid]['ignore_text'].append(l.strip())
-            elif mode == 'digit-regex':
-                for l in selection.splitlines():
-                    # Replace any series of numbers with a regex
-                    s = re.escape(l.strip())
-                    s = re.sub(r'[0-9]+', r'\\d+', s)
-                    datastore.data["watching"][uuid]['ignore_text'].append('/' + s + '/')
-
-        return f"<a href={url_for('preview_page', uuid=uuid)}>Click to preview</a>"
-
-
     import changedetectionio.blueprint.browser_steps as browser_steps
     app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
 
@@ -1511,10 +1439,6 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.blueprint.tags as tags
     app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags')
 
-    import changedetectionio.blueprint.check_proxies as check_proxies
-    app.register_blueprint(check_proxies.construct_blueprint(datastore=datastore), url_prefix='/check_proxy')
-
-
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
     threading.Thread(target=notification_runner).start()

changedetectionio/api/api_v1.py

@@ -1,6 +1,3 @@
-import os
-from distutils.util import strtobool
-
 from flask_expects_json import expects_json
 from changedetectionio import queuedWatchMetaData
 from flask_restful import abort, Resource
@@ -36,7 +33,7 @@ class Watch(Resource):
     @auth.check_token
     def get(self, uuid):
         """
-        @api {get} /api/v1/watch/:uuid Single watch - get data, recheck, pause, mute.
+        @api {get} /api/v1/watch/:uuid Get a single watch data
         @apiDescription Retrieve watch information and set muted/paused status
         @apiExample {curl} Example usage:
             curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
@@ -212,9 +209,7 @@ class CreateWatch(Resource):
         json_data = request.get_json()
         url = json_data['url'].strip()
 
-        # If hosts that only contain alphanumerics are allowed ("localhost" for example)
-        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
-        if not validators.url(url, simple_host=allow_simplehost):
+        if not validators.url(json_data['url'].strip()):
             return "Invalid or unsupported URL", 400
 
         if json_data.get('proxy'):

changedetectionio/blueprint/browser_steps/__init__.py

@@ -23,10 +23,8 @@
 
 from distutils.util import strtobool
 from flask import Blueprint, request, make_response
-import logging
 import os
-import re
-
+import logging
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio import login_optionally_required
 
@@ -46,7 +44,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
 
 
         # We keep the playwright session open for many minutes
-        keepalive_seconds = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
+        seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
 
         browsersteps_start_session = {'start_time': time.time()}
 
@@ -58,18 +56,16 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             # Start the Playwright context, which is actually a nodejs sub-process and communicates over STDIN/STDOUT pipes
             io_interface_context = io_interface_context.start()
 
-        keepalive_ms = ((keepalive_seconds + 3) * 1000)
-        base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '')
-        a = "?" if not '?' in base_url else '&'
-        base_url += a + f"timeout={keepalive_ms}"
 
+        # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
+        keepalive = "&timeout={}".format(((seconds_keepalive + 3) * 1000))
         try:
-            browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(base_url)
+            browsersteps_start_session['browser'] = io_interface_context.chromium.connect_over_cdp(
+                os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
         except Exception as e:
             if 'ECONNREFUSED' in str(e):
                 return make_response('Unable to start the Playwright Browser session, is it running?', 401)
             else:
-                # Other errors, bad URL syntax, bad reply etc
                 return make_response(str(e), 401)
 
         proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
@@ -122,31 +118,6 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         print("Starting connection with playwright - done")
         return {'browsersteps_session_id': browsersteps_session_id}
 
-    @login_optionally_required
-    @browser_steps_blueprint.route("/browsersteps_image", methods=['GET'])
-    def browser_steps_fetch_screenshot_image():
-        from flask import (
-            make_response,
-            request,
-            send_from_directory,
-        )
-        uuid = request.args.get('uuid')
-        step_n = int(request.args.get('step_n'))
-
-        watch = datastore.data['watching'].get(uuid)
-        filename = f"step_before-{step_n}.jpeg" if request.args.get('type', '') == 'before' else f"step_{step_n}.jpeg"
-
-        if step_n and watch and os.path.isfile(os.path.join(watch.watch_data_dir, filename)):
-            response = make_response(send_from_directory(directory=watch.watch_data_dir, path=filename))
-            response.headers['Content-type'] = 'image/jpeg'
-            response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
-            response.headers['Pragma'] = 'no-cache'
-            response.headers['Expires'] = 0
-            return response
-
-        else:
-            return make_response('Unable to fetch image, is the URL correct? does the watch exist? does the step_type-n.jpeg exist?', 401)
-
     # A request for an action was received
     @login_optionally_required
     @browser_steps_blueprint.route("/browsersteps_update", methods=['POST'])

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -77,13 +77,13 @@ class steppable_browser_interface():
     def action_goto_url(self, selector=None, value=None):
         # self.page.set_viewport_size({"width": 1280, "height": 5000})
         now = time.time()
-        response = self.page.goto(value, timeout=0, wait_until='load')
-        # Should be the same as the puppeteer_fetch.js methods, means, load with no timeout set (skip timeout)
-        #and also wait for seconds ?
-        #await page.waitForTimeout(1000);
-        #await page.waitForTimeout(extra_wait_ms);
+        response = self.page.goto(value, timeout=0, wait_until='commit')
+
+        # Wait_until = commit
+        # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
+        # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
+        # This seemed to solve nearly all 'TimeoutErrors'
         print("Time to goto URL ", time.time() - now)
-        return response
 
     def action_click_element_containing_text(self, selector=None, value=''):
         if not len(value.strip()):
@@ -99,8 +99,7 @@ class steppable_browser_interface():
         self.page.fill(selector, value, timeout=10 * 1000)
 
     def action_execute_js(self, selector, value):
-        response = self.page.evaluate(value)
-        return response
+        self.page.evaluate(value)
 
     def action_click_element(self, selector, value):
         print("Clicking element")
@@ -139,13 +138,13 @@ class steppable_browser_interface():
     def action_wait_for_text(self, selector, value):
         import json
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=30000)
+        self.page.wait_for_function(f'document.querySelector("body").innerText.includes({v});', timeout=90000)
 
     def action_wait_for_text_in_element(self, selector, value):
         import json
         s = json.dumps(selector)
         v = json.dumps(value)
-        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=30000)
+        self.page.wait_for_function(f'document.querySelector({s}).innerText.includes({v});', timeout=90000)
 
     # @todo - in the future make some popout interface to capture what needs to be set
     # https://playwright.dev/python/docs/api/class-keyboard

changedetectionio/blueprint/check_proxies/__init__.py

@@ -1,118 +0,0 @@
-from concurrent.futures import ThreadPoolExecutor
-
-from functools import wraps
-
-from flask import Blueprint
-from flask_login import login_required
-
-from changedetectionio.processors import text_json_diff
-from changedetectionio.store import ChangeDetectionStore
-
-
-STATUS_CHECKING = 0
-STATUS_FAILED = 1
-STATUS_OK = 2
-THREADPOOL_MAX_WORKERS = 3
-_DEFAULT_POOL = ThreadPoolExecutor(max_workers=THREADPOOL_MAX_WORKERS)
-
-
-# Maybe use fetch-time if its >5 to show some expected load time?
-def threadpool(f, executor=None):
-    @wraps(f)
-    def wrap(*args, **kwargs):
-        return (executor or _DEFAULT_POOL).submit(f, *args, **kwargs)
-
-    return wrap
-
-
-def construct_blueprint(datastore: ChangeDetectionStore):
-    check_proxies_blueprint = Blueprint('check_proxies', __name__)
-    checks_in_progress = {}
-
-    @threadpool
-    def long_task(uuid, preferred_proxy):
-        import time
-        from changedetectionio import content_fetcher
-
-        status = {'status': '', 'length': 0, 'text': ''}
-        from jinja2 import Environment, BaseLoader
-
-        contents = ''
-        now = time.time()
-        try:
-            update_handler = text_json_diff.perform_site_check(datastore=datastore)
-            changed_detected, update_obj, contents = update_handler.run(uuid, preferred_proxy=preferred_proxy, skip_when_checksum_same=False)
-        # title, size is len contents not len xfer
-        except content_fetcher.Non200ErrorCodeReceived as e:
-            if e.status_code == 404:
-                status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but 404 (page not found)"})
-            elif e.status_code == 403 or e.status_code == 401:
-                status.update({'status': 'ERROR', 'length': len(contents), 'text': f"{e.status_code} - Access denied"})
-            else:
-                status.update({'status': 'ERROR', 'length': len(contents), 'text': f"Status code: {e.status_code}"})
-        except text_json_diff.FilterNotFoundInResponse:
-            status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but CSS/xPath filter not found (page changed layout?)"})
-        except content_fetcher.EmptyReply as e:
-            if e.status_code == 403 or e.status_code == 401:
-                status.update({'status': 'ERROR OTHER', 'length': len(contents), 'text': f"Got empty reply with code {e.status_code} - Access denied"})
-            else:
-                status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': f"Empty reply with code {e.status_code}, needs chrome?"})
-        except content_fetcher.ReplyWithContentButNoText as e:
-            txt = f"Got reply but with no content - Status code {e.status_code} - It's possible that the filters were found, but contained no usable text (or contained only an image)."
-            status.update({'status': 'ERROR', 'text': txt})
-        except Exception as e:
-            status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': 'Error: '+type(e).__name__+str(e)})
-        else:
-            status.update({'status': 'OK', 'length': len(contents), 'text': ''})
-
-        if status.get('text'):
-            status['text'] = Environment(loader=BaseLoader()).from_string('{{text|e}}').render({'text': status['text']})
-
-        status['time'] = "{:.2f}s".format(time.time() - now)
-
-        return status
-
-    def _recalc_check_status(uuid):
-
-        results = {}
-        for k, v in checks_in_progress.get(uuid, {}).items():
-            try:
-                r_1 = v.result(timeout=0.05)
-            except Exception as e:
-                # If timeout error?
-                results[k] = {'status': 'RUNNING'}
-
-            else:
-                results[k] = r_1
-
-        return results
-
-    @login_required
-    @check_proxies_blueprint.route("/<string:uuid>/status", methods=['GET'])
-    def get_recheck_status(uuid):
-        results = _recalc_check_status(uuid=uuid)
-        return results
-
-    @login_required
-    @check_proxies_blueprint.route("/<string:uuid>/start", methods=['GET'])
-    def start_check(uuid):
-
-        if not datastore.proxy_list:
-            return
-
-        if checks_in_progress.get(uuid):
-            state = _recalc_check_status(uuid=uuid)
-            for proxy_key, v in state.items():
-                if v.get('status') == 'RUNNING':
-                    return state
-        else:
-            checks_in_progress[uuid] = {}
-
-        for k, v in datastore.proxy_list.items():
-            if not checks_in_progress[uuid].get(k):
-                checks_in_progress[uuid][k] = long_task(uuid=uuid, preferred_proxy=k)
-
-        results = _recalc_check_status(uuid=uuid)
-        return results
-
-    return check_proxies_blueprint

changedetectionio/blueprint/tags/templates/edit-tag.html

@@ -2,10 +2,6 @@
 {% block content %}
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
-<script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
-</script>
-
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script>
 

changedetectionio/content_fetcher.py

@@ -1,15 +1,12 @@
-from abc import abstractmethod
-from distutils.util import strtobool
-from urllib.parse import urlparse
-import chardet
 import hashlib
+from abc import abstractmethod
+import chardet
 import json
 import logging
 import os
 import requests
 import sys
 import time
-import urllib.parse
 
 visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary'
 
@@ -80,13 +77,11 @@ class ScreenshotUnavailable(Exception):
 
 
 class ReplyWithContentButNoText(Exception):
-    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content=''):
+    def __init__(self, status_code, url, screenshot=None):
         # Set this so we can use it in other parts of the app
         self.status_code = status_code
         self.url = url
         self.screenshot = screenshot
-        self.has_filters = has_filters
-        self.html_content = html_content
         return
 
 
@@ -159,16 +154,6 @@ class Fetcher():
         """
         return {k.lower(): v for k, v in self.headers.items()}
 
-    def browser_steps_get_valid_steps(self):
-        if self.browser_steps is not None and len(self.browser_steps):
-            valid_steps = filter(
-                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-                self.browser_steps)
-
-            return valid_steps
-
-        return None
-
     def iterate_browser_steps(self):
         from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
         from playwright._impl._api_types import TimeoutError
@@ -180,7 +165,10 @@ class Fetcher():
         if self.browser_steps is not None and len(self.browser_steps):
             interface = steppable_browser_interface()
             interface.page = self.page
-            valid_steps = self.browser_steps_get_valid_steps()
+
+            valid_steps = filter(
+                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
+                self.browser_steps)
 
             for step in valid_steps:
                 step_n += 1
@@ -213,8 +201,7 @@ class Fetcher():
             dest = os.path.join(self.browser_steps_screenshot_path, 'step_*.jpeg')
             files = glob.glob(dest)
             for f in files:
-                if os.path.isfile(f):
-                    os.unlink(f)
+                os.unlink(f)
 
 
 #   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
@@ -276,6 +263,7 @@ class base_html_playwright(Fetcher):
 
         if self.proxy:
             # Playwright needs separate username and password values
+            from urllib.parse import urlparse
             parsed = urlparse(self.proxy.get('server'))
             if parsed.username:
                 self.proxy['username'] = parsed.username
@@ -330,11 +318,13 @@ class base_html_playwright(Fetcher):
 
         # Append proxy connect string
         if self.proxy:
+            import urllib.parse
             # Remove username/password if it exists in the URL or you will receive "ERR_NO_SUPPORTED_PROXIES" error
             # Actual authentication handled by Puppeteer/node
             o = urlparse(self.proxy.get('server'))
             proxy_url = urllib.parse.quote(o._replace(netloc="{}:{}".format(o.hostname, o.port)).geturl())
-            browserless_function_url = f"{browserless_function_url}&--proxy-server={proxy_url}"
+            browserless_function_url = f"{browserless_function_url}&--proxy-server={proxy_url}&dumpio=true"
+
 
         try:
             amp = '&' if '?' in browserless_function_url else '?'
@@ -352,9 +342,9 @@ class base_html_playwright(Fetcher):
                         'req_headers': request_headers,
                         'screenshot_quality': int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)),
                         'url': url,
-                        'user_agent': {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None),
-                        'proxy_username': self.proxy.get('username', '') if self.proxy else False,
-                        'proxy_password': self.proxy.get('password', '') if self.proxy and self.proxy.get('username') else False,
+                        'user_agent': request_headers.get('User-Agent', 'Mozilla/5.0'),
+                        'proxy_username': self.proxy.get('username','') if self.proxy else False,
+                        'proxy_password': self.proxy.get('password', '') if self.proxy else False,
                         'no_cache_list': [
                             'twitter',
                             '.pdf'
@@ -423,8 +413,8 @@ class base_html_playwright(Fetcher):
                 lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
                 self.browser_steps))
 
-        if not has_browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
-            if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')):
+        if not has_browser_steps:
+            if os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
                 # Temporary backup solution until we rewrite the playwright code
                 return self.run_fetch_browserless_puppeteer(
                     url,
@@ -441,7 +431,6 @@ class base_html_playwright(Fetcher):
 
         self.delete_browser_steps_screenshots()
         response = None
-
         with sync_playwright() as p:
             browser_type = getattr(p, self.browser_type)
 
@@ -450,13 +439,10 @@ class base_html_playwright(Fetcher):
             # 60,000 connection timeout only
             browser = browser_type.connect_over_cdp(self.command_executor, timeout=60000)
 
-            # SOCKS5 with authentication is not supported (yet)
-            # https://github.com/microsoft/playwright/issues/10567
-
             # Set user agent to prevent Cloudflare from blocking the browser
             # Use the default one configured in the App.py model that's passed from fetch_site_status.py
             context = browser.new_context(
-                user_agent={k.lower(): v for k, v in request_headers.items()}.get('user-agent', None),
+                user_agent=request_headers.get('User-Agent', 'Mozilla/5.0'),
                 proxy=self.proxy,
                 # This is needed to enable JavaScript execution on GitHub and others
                 bypass_csp=True,
@@ -470,26 +456,40 @@ class base_html_playwright(Fetcher):
             if len(request_headers):
                 context.set_extra_http_headers(request_headers)
 
-            # Listen for all console events and handle errors
-            self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
+                self.page.set_default_navigation_timeout(90000)
+                self.page.set_default_timeout(90000)
 
-            # Re-use as much code from browser steps as possible so its the same
-            from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface
-            browsersteps_interface = steppable_browser_interface()
-            browsersteps_interface.page = self.page
+                # Listen for all console events and handle errors
+                self.page.on("console", lambda msg: print(f"Playwright console: Watch URL: {url} {msg.type}: {msg.text} {msg.args}"))
 
-            response = browsersteps_interface.action_goto_url(value=url)
-            self.headers = response.all_headers()
+            # Goto page
+            try:
+                # Wait_until = commit
+                # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
+                # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
+                # This seemed to solve nearly all 'TimeoutErrors'
+                response = self.page.goto(url, wait_until='commit')
+            except playwright._impl._api_types.Error as e:
+                # Retry once - https://github.com/browserless/chrome/issues/2485
+                # Sometimes errors related to invalid cert's and other can be random
+                print("Content Fetcher > retrying request got error - ", str(e))
+                time.sleep(1)
+                response = self.page.goto(url, wait_until='commit')
 
-            if response is None:
+            except Exception as e:
+                print("Content Fetcher > Other exception when page.goto", str(e))
                 context.close()
                 browser.close()
-                print("Content Fetcher > Response object was none")
-                raise EmptyReply(url=url, status_code=None)
+                raise PageUnloadable(url=url, status_code=None, message=str(e))
 
+            # Execute any browser steps
             try:
+                extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
+                self.page.wait_for_timeout(extra_wait * 1000)
+
                 if self.webdriver_js_execute_code is not None and len(self.webdriver_js_execute_code):
-                    browsersteps_interface.action_execute_js(value=self.webdriver_js_execute_code, selector=None)
+                    self.page.evaluate(self.webdriver_js_execute_code)
+
             except playwright._impl._api_types.TimeoutError as e:
                 context.close()
                 browser.close()
@@ -501,26 +501,28 @@ class base_html_playwright(Fetcher):
                 browser.close()
                 raise PageUnloadable(url=url, status_code=None, message=str(e))
 
+            if response is None:
+                context.close()
+                browser.close()
+                print("Content Fetcher > Response object was none")
+                raise EmptyReply(url=url, status_code=None)
+
+            # Run Browser Steps here
+            self.iterate_browser_steps()
+
             extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
-            self.page.wait_for_timeout(extra_wait * 1000)
-
+            time.sleep(extra_wait)
 
+            self.content = self.page.content()
             self.status_code = response.status
-
-            if self.status_code != 200 and not ignore_status_codes:
-                raise Non200ErrorCodeReceived(url=url, status_code=self.status_code)
-
             if len(self.page.content().strip()) == 0:
                 context.close()
                 browser.close()
                 print("Content Fetcher > Content was empty")
                 raise EmptyReply(url=url, status_code=response.status)
 
-            # Run Browser Steps here
-            if self.browser_steps_get_valid_steps():
-                self.iterate_browser_steps()
-                
-            self.page.wait_for_timeout(extra_wait * 1000)
+            self.status_code = response.status
+            self.headers = response.all_headers()
 
             # So we can find an element on the page where its selector was entered manually (maybe not xPath etc)
             if current_include_filters is not None:
@@ -532,7 +534,6 @@ class base_html_playwright(Fetcher):
                 "async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
             self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
 
-            self.content = self.page.content()
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
             # JPEG is better here because the screenshots can be very very large
@@ -547,7 +548,7 @@ class base_html_playwright(Fetcher):
             except Exception as e:
                 context.close()
                 browser.close()
-                raise ScreenshotUnavailable(url=url, status_code=response.status_code)
+                raise ScreenshotUnavailable(url=url, status_code=None)
 
             context.close()
             browser.close()
@@ -606,17 +607,15 @@ class base_html_webdriver(Fetcher):
             is_binary=False):
 
         from selenium import webdriver
-        from selenium.webdriver.chrome.options import Options as ChromeOptions
+        from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
         from selenium.common.exceptions import WebDriverException
         # request_body, request_method unused for now, until some magic in the future happens.
 
-        options = ChromeOptions()
-        if self.proxy:
-            options.proxy = self.proxy
-
+        # check env for WEBDRIVER_URL
         self.driver = webdriver.Remote(
             command_executor=self.command_executor,
-            options=options)
+            desired_capabilities=DesiredCapabilities.CHROME,
+            proxy=self.proxy)
 
         try:
             self.driver.get(url)
@@ -648,11 +647,11 @@ class base_html_webdriver(Fetcher):
     # Does the connection to the webdriver work? run a test connection.
     def is_ready(self):
         from selenium import webdriver
-        from selenium.webdriver.chrome.options import Options as ChromeOptions
+        from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 
         self.driver = webdriver.Remote(
             command_executor=self.command_executor,
-            options=ChromeOptions())
+            desired_capabilities=DesiredCapabilities.CHROME)
 
         # driver.quit() seems to cause better exceptions
         self.quit()
@@ -684,17 +683,13 @@ class html_requests(Fetcher):
             is_binary=False):
 
         # Make requests use a more modern looking user-agent
-        if not {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None):
+        if not 'User-Agent' in request_headers:
             request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT",
                                                       'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36')
 
         proxies = {}
 
         # Allows override the proxy on a per-request basis
-
-        # https://requests.readthedocs.io/en/latest/user/advanced/#socks
-        # Should also work with `socks5://user:pass@host:port` type syntax.
-
         if self.proxy_override:
             proxies = {'http': self.proxy_override, 'https': self.proxy_override, 'ftp': self.proxy_override}
         else:

changedetectionio/diff.py

@@ -35,19 +35,15 @@ def customSequenceMatcher(before, after, include_equal=False, include_removed=Tr
 
 # only_differences - only return info about the differences, no context
 # line_feed_sep could be "<br>" or "<li>" or "\n" etc
-def render_diff(previous_version_file_contents, newest_version_file_contents, include_equal=False, include_removed=True, include_added=True, include_replaced=True, line_feed_sep="\n", include_change_type_prefix=True, patch_format=False):
+def render_diff(previous_version_file_contents, newest_version_file_contents, include_equal=False, include_removed=True, include_added=True, include_replaced=True, line_feed_sep="\n", include_change_type_prefix=True):
 
     newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
 
     if previous_version_file_contents:
-        previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
+            previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
     else:
         previous_version_file_contents = ""
 
-    if patch_format:
-        patch = difflib.unified_diff(previous_version_file_contents, newest_version_file_contents)
-        return line_feed_sep.join(patch)
-
     rendered_diff = customSequenceMatcher(before=previous_version_file_contents,
                                           after=newest_version_file_contents,
                                           include_equal=include_equal,
@@ -58,5 +54,4 @@ def render_diff(previous_version_file_contents, newest_version_file_contents, in
 
     # Recursively join lists
     f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
-    p= f(rendered_diff)
-    return p
+    return f(rendered_diff)

changedetectionio/forms.py

@@ -1,6 +1,5 @@
 import os
 import re
-from distutils.util import strtobool
 
 from wtforms import (
     BooleanField,
@@ -15,20 +14,14 @@ from wtforms import (
     validators,
     widgets
 )
-from flask_wtf.file import FileField, FileAllowed
 from wtforms.fields import FieldList
-
 from wtforms.validators import ValidationError
 
-from validators.url import url as url_validator
-
-
 # default
 # each select <option data-enabled="enabled-0-0"
 from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
 
-from changedetectionio import content_fetcher, html_tools
-
+from changedetectionio import content_fetcher
 from changedetectionio.notification import (
     valid_notification_formats,
 )
@@ -46,7 +39,7 @@ valid_method = {
 }
 
 default_method = 'GET'
-allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+
 
 class StringListField(StringField):
     widget = widgets.TextArea()
@@ -235,19 +228,16 @@ class ValidateJinja2Template(object):
     def __call__(self, form, field):
         from changedetectionio import notification
 
-        from jinja2 import Environment, BaseLoader, TemplateSyntaxError, UndefinedError
+        from jinja2 import Environment, BaseLoader, TemplateSyntaxError
         from jinja2.meta import find_undeclared_variables
 
 
         try:
             jinja2_env = Environment(loader=BaseLoader)
             jinja2_env.globals.update(notification.valid_tokens)
-
             rendered = jinja2_env.from_string(field.data).render()
         except TemplateSyntaxError as e:
             raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
-        except UndefinedError as e:
-            raise ValidationError(f"A variable or function is not defined: {e}") from e
 
         ast = jinja2_env.parse(field.data)
         undefined = ", ".join(find_undeclared_variables(ast))
@@ -266,23 +256,18 @@ class validateURL(object):
         self.message = message
 
     def __call__(self, form, field):
-        # This should raise a ValidationError() or not
-        validate_url(field.data)
+        import validators
 
-def validate_url(test_url):
-    # If hosts that only contain alphanumerics are allowed ("localhost" for example)
-    try:
-        url_validator(test_url, simple_host=allow_simplehost)
-    except validators.ValidationError:
-        #@todo check for xss
-        message = f"'{test_url}' is not a valid URL."
-        # This should be wtforms.validators.
-        raise ValidationError(message)
+        try:
+            validators.url(field.data.strip())
+        except validators.ValidationFailure:
+            message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
+            raise ValidationError(message)
+
+        from .model.Watch import is_safe_url
+        if not is_safe_url(field.data):
+            raise ValidationError('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX')
 
-    from .model.Watch import is_safe_url
-    if not is_safe_url(test_url):
-        # This should be wtforms.validators.
-        raise ValidationError('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX or incorrect URL format')
 
 class ValidateListRegex(object):
     """
@@ -294,10 +279,11 @@ class ValidateListRegex(object):
     def __call__(self, form, field):
 
         for line in field.data:
-            if re.search(html_tools.PERL_STYLE_REGEX, line, re.IGNORECASE):
+            if line[0] == '/' and line[-1] == '/':
+                # Because internally we dont wrap in /
+                line = line.strip('/')
                 try:
-                    regex = html_tools.perl_style_slash_enclosed_regex_to_options(line)
-                    re.compile(regex)
+                    re.compile(line)
                 except re.error:
                     message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
                     raise ValidationError(message % (line))
@@ -407,9 +393,6 @@ class importForm(Form):
     from . import processors
     processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff")
     urls = TextAreaField('URLs')
-    xlsx_file = FileField('Upload .xlsx file', validators=[FileAllowed(['xlsx'], 'Must be .xlsx file!')])
-    file_mapping = SelectField('File mapping', [validators.DataRequired()], choices={('wachete', 'Wachete mapping'), ('custom','Custom mapping')})
-
 
 class SingleBrowserStep(Form):
 
@@ -493,7 +476,7 @@ class SingleExtraProxy(Form):
 
     # maybe better to set some <script>var..
     proxy_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
-    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "socks5:// or regular proxy http://user:pass@...:3128", "size":50})
+    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "http://user:pass@...:3128", "size":50})
     # @todo do the validation here instead
 
 # datastore.data['settings']['requests']..
@@ -517,10 +500,7 @@ class globalSettingsRequestForm(Form):
 class globalSettingsApplicationForm(commonSettingsForm):
 
     api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
-    base_url = StringField('Notification base URL override',
-                           validators=[validators.Optional()],
-                           render_kw={"placeholder": os.getenv('BASE_URL', 'Not set')}
-                           )
+    base_url = StringField('Base URL', validators=[validators.Optional()])
     empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
     fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])

changedetectionio/html_tools.py

@@ -1,44 +1,23 @@
 
 from bs4 import BeautifulSoup
 from inscriptis import get_text
+from inscriptis.model.config import ParserConfig
 from jsonpath_ng.ext import parse
 from typing import List
-from inscriptis.css_profiles import CSS_PROFILES, HtmlElement
-from inscriptis.html_properties import Display
-from inscriptis.model.config import ParserConfig
-from xml.sax.saxutils import escape as xml_escape
 import json
 import re
 
-
 # HTML added to be sure each result matching a filter (.example) gets converted to a new line by Inscriptis
 TEXT_FILTER_LIST_LINE_SUFFIX = "<br>"
 
-PERL_STYLE_REGEX = r'^/(.*?)/([a-z]*)?$'
 # 'price' , 'lowPrice', 'highPrice' are usually under here
-# All of those may or may not appear on different websites - I didnt find a way todo case-insensitive searching here
-LD_JSON_PRODUCT_OFFER_SELECTORS = ["json:$..offers", "json:$..Offers"]
+# all of those may or may not appear on different websites
+LD_JSON_PRODUCT_OFFER_SELECTOR = "json:$..offers"
 
 class JSONNotFound(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
-
-
-# Doesn't look like python supports forward slash auto enclosure in re.findall
-# So convert it to inline flag "(?i)foobar" type configuration
-def perl_style_slash_enclosed_regex_to_options(regex):
-
-    res = re.search(PERL_STYLE_REGEX, regex, re.IGNORECASE)
-
-    if res:
-        flags = res.group(2) if res.group(2) else 'i'
-        regex = f"(?{flags}){res.group(1)}"
-    else:
-        # Fall back to just ignorecase as an option
-        regex = f"(?i){regex}"
-
-    return regex
-
+        
 # Given a CSS Rule, and a blob of HTML, return the blob of HTML that matches
 def include_filters(include_filters, html_content, append_pretty_line_formatting=False):
     soup = BeautifulSoup(html_content, "html.parser")
@@ -71,15 +50,10 @@ def element_removal(selectors: List[str], html_content):
 
 
 # Return str Utf-8 of matched rules
-def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False, is_rss=False):
+def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False):
     from lxml import etree, html
 
-    parser = None
-    if is_rss:
-        # So that we can keep CDATA for cdata_in_document_to_text() to process
-        parser = etree.XMLParser(strip_cdata=False)
-
-    tree = html.fromstring(bytes(html_content, encoding='utf-8'), parser=parser)
+    tree = html.fromstring(bytes(html_content, encoding='utf-8'))
     html_block = ""
 
     r = tree.xpath(xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'})
@@ -102,6 +76,7 @@ def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False
 
     return html_block
 
+
 # Extract/find element
 def extract_element(find='title', html_content=''):
 
@@ -169,6 +144,7 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         # As a last resort, try to parse the whole <body>
+        s = []
         soup = BeautifulSoup(content, 'html.parser')
 
         if ensure_is_ldjson_info_type:
@@ -194,24 +170,13 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
         
         for json_data in bs_jsons:
             stripped_text_from_html = _parse_json(json_data, json_filter)
-
             if ensure_is_ldjson_info_type:
                 # Could sometimes be list, string or something else random
                 if isinstance(json_data, dict):
                     # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
                     # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
-                    # @type could also be a list (Product, SubType)
-                    # LD_JSON auto-extract also requires some content PLUS the ldjson to be present
-                    # 1833 - could be either str or dict, should not be anything else
-                    if json_data.get('@type') and stripped_text_from_html:
-                        try:
-                            if json_data.get('@type') == str or json_data.get('@type') == dict:
-                                types = [json_data.get('@type')] if isinstance(json_data.get('@type'), str) else json_data.get('@type')
-                                if ensure_is_ldjson_info_type.lower() in [x.lower().strip() for x in types]:
-                                    break
-                        except:
-                            continue
-
+                    if json_data.get('@type', False) and json_data.get('@type','').lower() == ensure_is_ldjson_info_type.lower() and stripped_text_from_html:
+                        break
             elif stripped_text_from_html:
                 break
 
@@ -226,56 +191,50 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 #
 # wordlist - list of regex's (str) or words (str)
 def strip_ignore_text(content, wordlist, mode="content"):
+    ignore = []
+    ignore_regex = []
+
+    # @todo check this runs case insensitive
+    for k in wordlist:
+
+        # Is it a regex?
+        if k[0] == '/':
+            ignore_regex.append(k.strip(" /"))
+        else:
+            ignore.append(k)
+
     i = 0
     output = []
-    ignore_text = []
-    ignore_regex = []
     ignored_line_numbers = []
-
-    for k in wordlist:
-        # Is it a regex?
-        res = re.search(PERL_STYLE_REGEX, k, re.IGNORECASE)
-        if res:
-            ignore_regex.append(re.compile(perl_style_slash_enclosed_regex_to_options(k)))
-        else:
-            ignore_text.append(k.strip())
-
     for line in content.splitlines():
         i += 1
         # Always ignore blank lines in this mode. (when this function gets called)
-        got_match = False
         if len(line.strip()):
-            for l in ignore_text:
-                if l.lower() in line.lower():
-                    got_match = True
+            regex_matches = False
 
-            if not got_match:
-                for r in ignore_regex:
-                    if r.search(line):
-                        got_match = True
+            # if any of these match, skip
+            for regex in ignore_regex:
+                try:
+                    if re.search(regex, line, re.IGNORECASE):
+                        regex_matches = True
+                except Exception as e:
+                    continue
 
-            if not got_match:
-                # Not ignored
+            if not regex_matches and not any(skip_text.lower() in line.lower() for skip_text in ignore):
                 output.append(line.encode('utf8'))
             else:
                 ignored_line_numbers.append(i)
 
 
+
     # Used for finding out what to highlight
     if mode == "line numbers":
         return ignored_line_numbers
 
     return "\n".encode('utf8').join(output)
 
-def cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False) -> str:
-    pattern = '<!\[CDATA\[(\s*(?:.(?<!\]\]>)\s*)*)\]\]>'
-    def repl(m):
-        text = m.group(1)
-        return xml_escape(html_to_text(html_content=text)).strip()
 
-    return re.sub(pattern, repl, html_content)
-
-def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False) -> str:
+def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
     """Converts html string to a string with just the text. If ignoring
     rendering anchor tag content is enable, anchor tag content are also
     included in the text
@@ -291,21 +250,16 @@ def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=Fals
     #  if anchor tag content flag is set to True define a config for
     #  extracting this content
     if render_anchor_tag_content:
+
         parser_config = ParserConfig(
-            annotation_rules={"a": ["hyperlink"]},
-            display_links=True
+            annotation_rules={"a": ["hyperlink"]}, display_links=True
         )
-    # otherwise set config to None/default
+
+    # otherwise set config to None
     else:
         parser_config = None
 
-    # RSS Mode - Inscriptis will treat `title` as something else.
-    # Make it as a regular block display element (//item/title)
-    # This is a bit of a hack - the real way it to use XSLT to convert it to HTML #1874
-    if is_rss:
-        html_content = re.sub(r'<title([\s>])', r'<h1\1', html_content)
-        html_content = re.sub(r'</title>', r'</h1>', html_content)
-
+    # get text and annotations via inscriptis
     text_content = get_text(html_content, config=parser_config)
 
     return text_content
@@ -313,18 +267,9 @@ def html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=Fals
 
 # Does LD+JSON exist with a @type=='product' and a .price set anywhere?
 def has_ldjson_product_info(content):
-    pricing_data = ''
-
     try:
-        if not 'application/ld+json' in content:
-            return False
-
-        for filter in LD_JSON_PRODUCT_OFFER_SELECTORS:
-            pricing_data += extract_json_as_string(content=content,
-                                                  json_filter=filter,
-                                                  ensure_is_ldjson_info_type="product")
-
-    except Exception as e:
+        pricing_data = extract_json_as_string(content=content, json_filter=LD_JSON_PRODUCT_OFFER_SELECTOR, ensure_is_ldjson_info_type="product")
+    except JSONNotFound as e:
         # Totally fine
         return False
     x=bool(pricing_data)

changedetectionio/importer.py

@@ -1,9 +1,6 @@
 from abc import ABC, abstractmethod
 import time
 import validators
-from wtforms import ValidationError
-
-from changedetectionio.forms import validate_url
 
 
 class Importer():
@@ -15,7 +12,6 @@ class Importer():
         self.new_uuids = []
         self.good = 0
         self.remaining_data = []
-        self.import_profile = None
 
     @abstractmethod
     def run(self,
@@ -136,167 +132,3 @@ class import_distill_io_json(Importer):
                     good += 1
 
         flash("{} Imported from Distill.io in {:.2f}s, {} Skipped.".format(len(self.new_uuids), time.time() - now, len(self.remaining_data)))
-
-
-class import_xlsx_wachete(Importer):
-
-    def run(self,
-            data,
-            flash,
-            datastore,
-            ):
-
-        good = 0
-        now = time.time()
-        self.new_uuids = []
-
-        from openpyxl import load_workbook
-
-        try:
-            wb = load_workbook(data)
-        except Exception as e:
-            # @todo correct except
-            flash("Unable to read export XLSX file, something wrong with the file?", 'error')
-            return
-
-        row_id = 2
-        for row in wb.active.iter_rows(min_row=row_id):
-            try:
-                extras = {}
-                data = {}
-                for cell in row:
-                    if not cell.value:
-                        continue
-                    column_title = wb.active.cell(row=1, column=cell.column).value.strip().lower()
-                    data[column_title] = cell.value
-
-                # Forced switch to webdriver/playwright/etc
-                dynamic_wachet = str(data.get('dynamic wachet', '')).strip().lower()  # Convert bool to str to cover all cases
-                # libreoffice and others can have it as =FALSE() =TRUE(), or bool(true)
-                if 'true' in dynamic_wachet or dynamic_wachet == '1':
-                    extras['fetch_backend'] = 'html_webdriver'
-                elif 'false' in dynamic_wachet or dynamic_wachet == '0':
-                    extras['fetch_backend'] = 'html_requests'
-
-                if data.get('xpath'):
-                    # @todo split by || ?
-                    extras['include_filters'] = [data.get('xpath')]
-                if data.get('name'):
-                    extras['title'] = data.get('name').strip()
-                if data.get('interval (min)'):
-                    minutes = int(data.get('interval (min)'))
-                    hours, minutes = divmod(minutes, 60)
-                    days, hours = divmod(hours, 24)
-                    weeks, days = divmod(days, 7)
-                    extras['time_between_check'] = {'weeks': weeks, 'days': days, 'hours': hours, 'minutes': minutes, 'seconds': 0}
-
-                # At minimum a URL is required.
-                if data.get('url'):
-                    try:
-                        validate_url(data.get('url'))
-                    except ValidationError as e:
-                        print(">> import URL error", data.get('url'), str(e))
-                        flash(f"Error processing row number {row_id}, URL value was incorrect, row was skipped.", 'error')
-                        # Don't bother processing anything else on this row
-                        continue
-
-                    new_uuid = datastore.add_watch(url=data['url'].strip(),
-                                                   extras=extras,
-                                                   tag=data.get('folder'),
-                                                   write_to_disk_now=False)
-                    if new_uuid:
-                        # Straight into the queue.
-                        self.new_uuids.append(new_uuid)
-                        good += 1
-            except Exception as e:
-                print(e)
-                flash(f"Error processing row number {row_id}, check all cell data types are correct, row was skipped.", 'error')
-            else:
-                row_id += 1
-
-        flash(
-            "{} imported from Wachete .xlsx in {:.2f}s".format(len(self.new_uuids), time.time() - now))
-
-
-class import_xlsx_custom(Importer):
-
-    def run(self,
-            data,
-            flash,
-            datastore,
-            ):
-
-        good = 0
-        now = time.time()
-        self.new_uuids = []
-
-        from openpyxl import load_workbook
-
-        try:
-            wb = load_workbook(data)
-        except Exception as e:
-            # @todo correct except
-            flash("Unable to read export XLSX file, something wrong with the file?", 'error')
-            return
-
-        # @todo cehck atleast 2 rows, same in other method
-        from .forms import validate_url
-        row_i = 1
-
-        try:
-            for row in wb.active.iter_rows():
-                url = None
-                tags = None
-                extras = {}
-
-                for cell in row:
-                    if not self.import_profile.get(cell.col_idx):
-                        continue
-                    if not cell.value:
-                        continue
-
-                    cell_map = self.import_profile.get(cell.col_idx)
-
-                    cell_val = str(cell.value).strip()  # could be bool
-
-                    if cell_map == 'url':
-                        url = cell.value.strip()
-                        try:
-                            validate_url(url)
-                        except ValidationError as e:
-                            print(">> Import URL error", url, str(e))
-                            flash(f"Error processing row number {row_i}, URL value was incorrect, row was skipped.", 'error')
-                            # Don't bother processing anything else on this row
-                            url = None
-                            break
-                    elif cell_map == 'tag':
-                        tags = cell.value.strip()
-                    elif cell_map == 'include_filters':
-                        # @todo validate?
-                        extras['include_filters'] = [cell.value.strip()]
-                    elif cell_map == 'interval_minutes':
-                        hours, minutes = divmod(int(cell_val), 60)
-                        days, hours = divmod(hours, 24)
-                        weeks, days = divmod(days, 7)
-                        extras['time_between_check'] = {'weeks': weeks, 'days': days, 'hours': hours, 'minutes': minutes, 'seconds': 0}
-                    else:
-                        extras[cell_map] = cell_val
-
-                # At minimum a URL is required.
-                if url:
-                    new_uuid = datastore.add_watch(url=url,
-                                                   extras=extras,
-                                                   tag=tags,
-                                                   write_to_disk_now=False)
-                    if new_uuid:
-                        # Straight into the queue.
-                        self.new_uuids.append(new_uuid)
-                        good += 1
-        except Exception as e:
-            print(e)
-            flash(f"Error processing row number {row_i}, check all cell data types are correct, row was skipped.", 'error')
-        else:
-            row_i += 1
-
-        flash(
-            "{} imported from custom .xlsx in {:.2f}s".format(len(self.new_uuids), time.time() - now))

changedetectionio/model/Watch.py

@@ -4,7 +4,6 @@ import os
 import re
 import time
 import uuid
-from pathlib import Path
 
 # Allowable protocols, protects against javascript: etc
 # file:// is further checked by ALLOW_FILE_URI
@@ -19,7 +18,6 @@ from changedetectionio.notification import (
 
 base_config = {
     'body': None,
-    'browser_steps_last_error_step': None,
     'check_unique_lines': False,  # On change-detected, compare against all history if its something new
     'check_count': 0,
     'date_created': None,
@@ -27,7 +25,6 @@ base_config = {
     'extract_text': [],  # Extract text by regex after filters
     'extract_title_as_title': False,
     'fetch_backend': 'system', # plaintext, playwright etc
-    'fetch_time': 0.0,
     'processor': 'text_json_diff', # could be restock_diff or others from .processors
     'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
     'filter_text_added': True,
@@ -170,7 +167,9 @@ class model(dict):
     @property
     def label(self):
         # Used for sorting
-        return self.get('title') if self.get('title') else self.get('url')
+        if self['title']:
+            return self['title']
+        return self['url']
 
     @property
     def last_changed(self):
@@ -492,13 +491,3 @@ class model(dict):
         filepath = os.path.join(self.watch_data_dir, 'last-fetched.br')
         with open(filepath, 'wb') as f:
             f.write(brotli.compress(contents, mode=brotli.MODE_TEXT))
-
-    @property
-    def get_browsersteps_available_screenshots(self):
-        "For knowing which screenshots are available to show the user in BrowserSteps UI"
-        available = []
-        for f in Path(self.watch_data_dir).glob('step_before-*.jpeg'):
-            step_n=re.search(r'step_before-(\d+)', f.name)
-            if step_n:
-                available.append(step_n.group(1))
-        return available

changedetectionio/notification.py

@@ -9,7 +9,6 @@ valid_tokens = {
     'diff': '',
     'diff_added': '',
     'diff_full': '',
-    'diff_patch': '',
     'diff_removed': '',
     'diff_url': '',
     'preview_url': '',
@@ -93,13 +92,7 @@ def process_notification(n_object, datastore):
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
     )
-
-    # If we arrived with 'System default' then look it up
-    if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
-        # Initially text or whatever
-        n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
-
-
+    
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
     # raise it as an exception
@@ -152,12 +145,9 @@ def process_notification(n_object, datastore):
                     # Apprise will default to HTML, so we need to override it
                     # So that whats' generated in n_body is in line with what is going to be sent.
                     # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-                    if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
+                    if not 'format=' in url and (n_format == 'text' or n_format == 'markdown'):
                         prefix = '?' if not '?' in url else '&'
-                        # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
-                        n_format = n_format.tolower()
                         url = "{}{}format={}".format(url, prefix, n_format)
-                    # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
 
                 apobj.add(url)
 
@@ -178,7 +168,7 @@ def process_notification(n_object, datastore):
                 log_value = logs.getvalue()
                 if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
                     raise Exception(log_value)
-
+                
                 sent_objs.append({'title': n_title,
                                   'body': n_body,
                                   'url' : url,
@@ -208,11 +198,15 @@ def create_notification_parameters(n_object, datastore):
         watch_tag = ''
 
     # Create URLs to customise the notification with
-    # active_base_url - set in store.py data property
-    base_url = datastore.data['settings']['application'].get('active_base_url')
+    base_url = datastore.data['settings']['application']['base_url']
 
     watch_url = n_object['watch_url']
 
+    # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services
+    #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.
+    if base_url == '':
+        base_url = "<base-url-env-var-not-set>"
+
     diff_url = "{}/diff/{}".format(base_url, uuid)
     preview_url = "{}/preview/{}".format(base_url, uuid)
 
@@ -222,12 +216,11 @@ def create_notification_parameters(n_object, datastore):
     # Valid_tokens also used as a field validator
     tokens.update(
         {
-            'base_url': base_url,
+            'base_url': base_url if base_url is not None else '',
             'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '',
             'diff': n_object.get('diff', ''),  # Null default in the case we use a test
             'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test
             'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
-            'diff_patch': n_object.get('diff_patch', ''),  # Null default in the case we use a test
             'diff_removed': n_object.get('diff_removed', ''),  # Null default in the case we use a test
             'diff_url': diff_url,
             'preview_url': preview_url,

changedetectionio/processors/__init__.py

@@ -9,7 +9,7 @@ class difference_detection_processor():
         super().__init__(*args, **kwargs)
 
     @abstractmethod
-    def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
+    def run(self, uuid, skip_when_checksum_same=True):
         update_obj = {'last_notification_error': False, 'last_error': False}
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()

changedetectionio/processors/text_json_diff.py

@@ -11,19 +11,17 @@ from changedetectionio import content_fetcher, html_tools
 from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
 from copy import deepcopy
 from . import difference_detection_processor
-from ..html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
-name = 'Webpage Text/HTML, JSON and PDF changes'
+
+name =  'Webpage Text/HTML, JSON and PDF changes'
 description = 'Detects all text changes where possible'
-json_filter_prefixes = ['json:', 'jq:']
 
 class FilterNotFoundInResponse(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
 
-
 class PDFToHTMLToolNotFound(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
@@ -39,7 +37,20 @@ class perform_site_check(difference_detection_processor):
         super().__init__(*args, **kwargs)
         self.datastore = datastore
 
-    def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
+    # Doesn't look like python supports forward slash auto enclosure in re.findall
+    # So convert it to inline flag "foobar(?i)" type configuration
+    def forward_slash_enclosed_regex_to_options(self, regex):
+        res = re.search(r'^/(.*?)/(\w+)$', regex, re.IGNORECASE)
+
+        if res:
+            regex = res.group(1)
+            regex += '(?{})'.format(res.group(2))
+        else:
+            regex += '(?{})'.format('i')
+
+        return regex
+
+    def run(self, uuid, skip_when_checksum_same=True):
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
@@ -94,11 +105,7 @@ class perform_site_check(difference_detection_processor):
             # If the klass doesnt exist, just use a default
             klass = getattr(content_fetcher, "html_requests")
 
-        if preferred_proxy:
-            proxy_id = preferred_proxy
-        else:
-            proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
-
+        proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
         proxy_url = None
         if proxy_id:
             proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')
@@ -124,8 +131,7 @@ class perform_site_check(difference_detection_processor):
         # requests for PDF's, images etc should be passwd the is_binary flag
         is_binary = watch.is_pdf
 
-        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'),
-                    is_binary=is_binary)
+        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'), is_binary=is_binary)
         fetcher.quit()
 
         self.screenshot = fetcher.screenshot
@@ -141,6 +147,7 @@ class perform_site_check(difference_detection_processor):
             if update_obj['previous_md5_before_filters'] == watch.get('previous_md5_before_filters'):
                 raise content_fetcher.checksumFromPreviousCheckWasTheSame()
 
+
         # Fetching complete, now filters
         # @todo move to class / maybe inside of fetcher abstract base?
 
@@ -153,22 +160,13 @@ class perform_site_check(difference_detection_processor):
 
         is_json = 'application/json' in fetcher.get_all_headers().get('content-type', '').lower()
         is_html = not is_json
-        is_rss = False
-
-        ctype_header = fetcher.get_all_headers().get('content-type', '').lower()
-        # Go into RSS preprocess for converting CDATA/comment to usable text
-        if any(substring in ctype_header for substring in ['application/xml', 'application/rss', 'text/xml']):
-            if '<rss' in fetcher.content[:100].lower():
-                fetcher.content = cdata_in_document_to_text(html_content=fetcher.content)
-                is_rss = True
 
         # source: support, basically treat it as plaintext
         if is_source:
             is_html = False
             is_json = False
 
-        inline_pdf = fetcher.get_all_headers().get('content-disposition', '') and '%PDF-1' in fetcher.content[:10]
-        if watch.is_pdf or 'application/pdf' in fetcher.get_all_headers().get('content-type', '').lower() or inline_pdf:
+        if watch.is_pdf or 'application/pdf' in fetcher.get_all_headers().get('content-type', '').lower():
             from shutil import which
             tool = os.getenv("PDF_TO_HTML_TOOL", "pdftohtml")
             if not which(tool):
@@ -205,7 +203,7 @@ class perform_site_check(difference_detection_processor):
 
         # Inject a virtual LD+JSON price tracker rule
         if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
-            include_filters_rule += html_tools.LD_JSON_PRODUCT_OFFER_SELECTORS
+            include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
 
         has_filter_rule = len(include_filters_rule) and len(include_filters_rule[0].strip())
         has_subtractive_selectors = len(subtractive_selectors) and len(subtractive_selectors[0].strip())
@@ -223,11 +221,14 @@ class perform_site_check(difference_detection_processor):
                 pass
 
         if has_filter_rule:
+            json_filter_prefixes = ['json:', 'jq:']
             for filter in include_filters_rule:
                 if any(prefix in filter for prefix in json_filter_prefixes):
                     stripped_text_from_html += html_tools.extract_json_as_string(content=fetcher.content, json_filter=filter)
                     is_html = False
 
+
+
         if is_html or is_source:
 
             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
@@ -251,8 +252,7 @@ class perform_site_check(difference_detection_processor):
                         if filter_rule[0] == '/' or filter_rule.startswith('xpath:'):
                             html_content += html_tools.xpath_filter(xpath_filter=filter_rule.replace('xpath:', ''),
                                                                     html_content=fetcher.content,
-                                                                    append_pretty_line_formatting=not is_source,
-                                                                    is_rss=is_rss)
+                                                                    append_pretty_line_formatting=not is_source)
                         else:
                             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
                             html_content += html_tools.include_filters(include_filters=filter_rule,
@@ -272,14 +272,14 @@ class perform_site_check(difference_detection_processor):
                     do_anchor = self.datastore.data["settings"]["application"].get("render_anchor_tag_content", False)
                     stripped_text_from_html = \
                         html_tools.html_to_text(
-                            html_content=html_content,
-                            render_anchor_tag_content=do_anchor,
-                            is_rss=is_rss # #1874 activate the <title workaround hack
+                            html_content,
+                            render_anchor_tag_content=do_anchor
                         )
 
         # Re #340 - return the content before the 'ignore text' was applied
         text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
+
         # @todo whitespace coming from missing rtrim()?
         # stripped_text_from_html could be based on their preferences, replace the processed text with only that which they want to know about.
         # Rewrite's the processing text based on only what diff result they want to see
@@ -289,13 +289,13 @@ class perform_site_check(difference_detection_processor):
             # needs to not include (added) etc or it may get used twice
             # Replace the processed text with the preferred result
             rendered_diff = diff.render_diff(previous_version_file_contents=watch.get_last_fetched_before_filters(),
-                                             newest_version_file_contents=stripped_text_from_html,
-                                             include_equal=False,  # not the same lines
-                                             include_added=watch.get('filter_text_added', True),
-                                             include_removed=watch.get('filter_text_removed', True),
-                                             include_replaced=watch.get('filter_text_replaced', True),
-                                             line_feed_sep="\n",
-                                             include_change_type_prefix=False)
+                                                       newest_version_file_contents=stripped_text_from_html,
+                                                       include_equal=False,  # not the same lines
+                                                       include_added=watch.get('filter_text_added', True),
+                                                       include_removed=watch.get('filter_text_removed', True),
+                                                       include_replaced=watch.get('filter_text_replaced', True),
+                                                       line_feed_sep="\n",
+                                                       include_change_type_prefix=False)
 
             watch.save_last_fetched_before_filters(text_content_before_ignored_filter)
 
@@ -310,12 +310,7 @@ class perform_site_check(difference_detection_processor):
         # Treat pages with no renderable text content as a change? No by default
         empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
         if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
-            raise content_fetcher.ReplyWithContentButNoText(url=url,
-                                                            status_code=fetcher.get_last_status_code(),
-                                                            screenshot=screenshot,
-                                                            has_filters=has_filter_rule,
-                                                            html_content=html_content
-                                                            )
+            raise content_fetcher.ReplyWithContentButNoText(url=url, status_code=fetcher.get_last_status_code(), screenshot=screenshot)
 
         # We rely on the actual text in the html output.. many sites have random script vars etc,
         # in the future we'll implement other mechanisms.
@@ -336,25 +331,16 @@ class perform_site_check(difference_detection_processor):
             regex_matched_output = []
             for s_re in extract_text:
                 # incase they specified something in '/.../x'
-                if re.search(PERL_STYLE_REGEX, s_re, re.IGNORECASE):
-                    regex = html_tools.perl_style_slash_enclosed_regex_to_options(s_re)
-                    result = re.findall(regex.encode('utf-8'), stripped_text_from_html)
+                regex = self.forward_slash_enclosed_regex_to_options(s_re)
+                result = re.findall(regex.encode('utf-8'), stripped_text_from_html)
 
-                    for l in result:
-                        if type(l) is tuple:
-                            # @todo - some formatter option default (between groups)
-                            regex_matched_output += list(l) + [b'\n']
-                        else:
-                            # @todo - some formatter option default (between each ungrouped result)
-                            regex_matched_output += [l] + [b'\n']
-                else:
-                    # Doesnt look like regex, just hunt for plaintext and return that which matches
-                    # `stripped_text_from_html` will be bytes, so we must encode s_re also to bytes
-                    r = re.compile(re.escape(s_re.encode('utf-8')), re.IGNORECASE)
-                    res = r.findall(stripped_text_from_html)
-                    if res:
-                        for match in res:
-                            regex_matched_output += [match] + [b'\n']
+                for l in result:
+                    if type(l) is tuple:
+                        # @todo - some formatter option default (between groups)
+                        regex_matched_output += list(l) + [b'\n']
+                    else:
+                        # @todo - some formatter option default (between each ungrouped result)
+                        regex_matched_output += [l] + [b'\n']
 
             # Now we will only show what the regex matched
             stripped_text_from_html = b''

changedetectionio/res/puppeteer_fetch.js

@@ -18,19 +18,12 @@ module.exports = async ({page, context}) => {
 
     await page.setBypassCSP(true)
     await page.setExtraHTTPHeaders(req_headers);
-
-    if (user_agent) {
-        await page.setUserAgent(user_agent);
-    }
+    await page.setUserAgent(user_agent);
     // https://ourcodeworld.com/articles/read/1106/how-to-solve-puppeteer-timeouterror-navigation-timeout-of-30000-ms-exceeded
 
     await page.setDefaultNavigationTimeout(0);
 
     if (proxy_username) {
-        // Setting Proxy-Authentication header is deprecated, and doing so can trigger header change errors from Puppeteer
-        // https://github.com/puppeteer/puppeteer/issues/676 ?
-        // https://help.brightdata.com/hc/en-us/articles/12632549957649-Proxy-Manager-How-to-Guides#h_01HAKWR4Q0AFS8RZTNYWRDFJC2
-        // https://cri.dev/posts/2020-03-30-How-to-solve-Puppeteer-Chrome-Error-ERR_INVALID_ARGUMENT/
         await page.authenticate({
             username: proxy_username,
             password: proxy_password

changedetectionio/res/stock-not-in-stock.js

@@ -5,19 +5,14 @@ function isItemInStock() {
     'agotado',
     'artikel zurzeit vergriffen',
     'as soon as stock is available',
-    'ausverkauft', // sold out
     'available for back order',
-    'back-order or out of stock',
     'backordered',
-    'benachrichtigt mich', // notify me
     'brak na stanie',
     'brak w magazynie',
     'coming soon',
     'currently have any tickets for this',
     'currently unavailable',
-    'dostępne wkrótce',
     'en rupture de stock',
-    'ist derzeit nicht auf lager',
     'item is no longer available',
     'message if back in stock',
     'nachricht bei',
@@ -42,7 +37,6 @@ function isItemInStock() {
     'unavailable tickets',
     'we do not currently have an estimate of when this product will be back in stock.',
     'zur zeit nicht an lager',
-    '已售完',
   ];
 
 

changedetectionio/run_proxy_tests.sh

@@ -2,48 +2,12 @@
 
 # exit when any command fails
 set -e
-# enable debug
-set -x
 
 # Test proxy list handling, starting two squids on different ports
 # Each squid adds a different header to the response, which is the main thing we test for.
 docker run --network changedet-network -d --name squid-one --hostname squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
 docker run --network changedet-network -d --name squid-two --hostname squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
 
-# SOCKS5 related - start simple Socks5 proxy server
-# SOCKSTEST=xyz should show in the logs of this service to confirm it fetched
-docker run --network changedet-network -d --hostname socks5proxy --name socks5proxy -p 1080:1080 -e PROXY_USER=proxy_user123 -e PROXY_PASSWORD=proxy_pass123 serjs/go-socks5-proxy
-docker run --network changedet-network -d --hostname socks5proxy-noauth -p 1081:1080 --name socks5proxy-noauth  serjs/go-socks5-proxy
-
-echo "---------------------------------- SOCKS5 -------------------"
-# SOCKS5 related - test from proxies.json
-docker run --network changedet-network \
-  -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
-  --rm \
-  -e "SOCKSTEST=proxiesjson" \
-  test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py'
-
-# SOCKS5 related - by manually entering in UI
-docker run --network changedet-network \
-  --rm \
-  -e "SOCKSTEST=manual" \
-  test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy.py'
-
-# SOCKS5 related - test from proxies.json via playwright - NOTE- PLAYWRIGHT DOESNT SUPPORT AUTHENTICATING PROXY
-docker run --network changedet-network \
-  -e "SOCKSTEST=manual-playwright" \
-  -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \
-  -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" \
-  --rm \
-  test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py'
-
-echo "socks5 server logs"
-docker logs socks5proxy
-echo "----------------------------------"
-
 # Used for configuring a custom proxy URL via the UI
 docker run --network changedet-network -d \
   --name squid-custom \
@@ -55,6 +19,7 @@ docker run --network changedet-network -d \
 
 
 ## 2nd test actually choose the preferred proxy from proxies.json
+
 docker run --network changedet-network \
   -v `pwd`/tests/proxy_list/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
   test-changedetectionio \
@@ -79,6 +44,7 @@ fi
 
 
 # Test the UI configurable proxies
+
 docker run --network changedet-network \
   test-changedetectionio \
   bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py'
@@ -92,25 +58,4 @@ then
   exit 1
 fi
 
-# Test "no-proxy" option
-docker run --network changedet-network \
-  test-changedetectionio \
-  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py'
-
-# We need to handle grep returning 1
-set +e
-# Check request was never seen in any container
-for c in $(echo "squid-one squid-two squid-custom"); do
-  echo Checking $c
-  docker logs $c &> $c.txt
-  grep noproxy $c.txt
-  if [ $? -ne 1 ]
-  then
-    echo "Saw request for noproxy in $c container"
-    cat $c.txt
-    exit 1
-  fi
-done
-
-
 docker kill squid-one squid-two squid-custom

changedetectionio/static/images/brightdata.svg

@@ -1,58 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   height="59.553207"
-   viewBox="-0.36 95.21 25.082135 59.553208"
-   width="249.99138"
-   version="1.1"
-   id="svg12"
-   sodipodi:docname="brightdata.svg"
-   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg">
-  <defs
-     id="defs16" />
-  <sodipodi:namedview
-     id="namedview14"
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1.0"
-     inkscape:pageshadow="2"
-     inkscape:pageopacity="0.0"
-     inkscape:pagecheckerboard="0"
-     showgrid="false"
-     fit-margin-top="0"
-     fit-margin-left="0"
-     fit-margin-right="0"
-     fit-margin-bottom="0"
-     inkscape:zoom="0.9464"
-     inkscape:cx="22.189349"
-     inkscape:cy="-90.870668"
-     inkscape:window-width="1920"
-     inkscape:window-height="1051"
-     inkscape:window-x="1920"
-     inkscape:window-y="0"
-     inkscape:window-maximized="1"
-     inkscape:current-layer="svg12" />
-  <path
-     d="m -34.416031,129.28 c -3.97,-2.43 -5.1,-6.09 -4.32,-10.35 0.81,-4.4 3.95,-6.75 8.04,-7.75 4.23,-1.04 8.44,-0.86 12.3,1.5 0.63,0.39 0.93,0.03 1.31,-0.29 1.5,-1.26 3.27,-1.72 5.189999,-1.83 0.79,-0.05 1.04,0.24 1.01,1.01 -0.05,1.31 -0.04,2.63 0,3.95 0.02,0.65 -0.19,0.93 -0.87,0.89 -0.889999,-0.04 -1.789999,0.03 -2.669999,-0.02 -0.82,-0.04 -1.08,0.1 -0.88,1.04 0.83,3.9 -0.06,7.37 -3.1,10.06 -2.76,2.44 -6.13,3.15 -9.72,3.04 -0.51,-0.02 -1.03,-0.02 -1.52,-0.13 -1.22,-0.25 -1.96,0.14 -2.19,1.41 -0.28,1.54 0.16,2.62 1.37,3.07 0.84,0.31 1.74,0.35 2.63,0.39 2.97,0.13 5.95,-0.18 8.91,0.21 2.93,0.39 5.69,1.16 6.85,4.25 1.269999,3.38 0.809999,6.62 -1.48,9.47 -2.73,3.39 -6.52,4.78 -10.66,5.33 -3.53,0.48 -7.04,0.27 -10.39,-1.11 -3.89,-1.6 -5.75,-4.95 -4.84,-8.72 0.51,-2.11 1.85,-3.58 3.69,-4.65 0.38,-0.22 0.93,-0.32 0.28,-0.96 -2.91,-2.83 -2.85,-6.16 0.1,-8.95 0.28,-0.26 0.6,-0.53 0.96,-0.86 z m 8.07,21.5 c 0.95,0.04 1.87,-0.13 2.78,-0.33 1.89,-0.42 3.51,-1.3 4.49,-3.06 1.82,-3.25 0.24,-6.2 -3.37,-6.58 -2.88,-0.3 -5.76,0.24 -8.63,-0.13 -0.53,-0.07 -0.75,0.34 -0.95,0.71 -1.16,2.24 -1.08,4.53 0,6.73 1.15,2.34 3.46,2.48 5.68,2.66 z m -5,-30.61 c -0.03,1.67 0.08,3.19 0.74,4.61 0.76,1.62 2.17,2.42 4.03,2.31 1.62,-0.1 2.9,-1.12 3.36,-2.84 0.66,-2.46 0.69,-4.95 0.01,-7.42 -0.49,-1.76 -1.7,-2.64 -3.56,-2.7 -2.08,-0.07 -3.37,0.7 -4.04,2.42 -0.47,1.21 -0.6,2.47 -0.54,3.62 z m 32.9399993,6.56 c 0,2.59 0.05,5.18 -0.02,7.77 -0.03,1.03 0.31,1.46 1.32,1.52 0.65,0.04 1.61,-0.09 1.82,0.57 0.26,0.81 0.11,1.76 0.06,2.65 -0.03,0.48 -0.81,0.39 -0.81,0.39 l -11.47,0.01 c 0,0 -0.95,-0.21 -0.88,-0.88 0.03,-0.29 0.04,-0.6 0,-0.89 -0.19,-1.24 0.21,-1.92 1.58,-1.9 0.99,0.01 1.28,-0.52 1.28,-1.53 -0.05,-8.75 -0.05,-17.49 0,-26.24 0.01,-1.15 -0.36,-1.62 -1.44,-1.67 -0.17,-0.01 -0.34,-0.04 -0.5,-0.07 -1.43,-0.22 -2.12,-1.57 -1.53,-2.91 0.15,-0.35 0.43,-0.36 0.72,-0.4 2.94,-0.41 5.88,-0.81 8.82000002,-1.23 0.81999998,-0.12 0.99999998,0.27 0.98999998,1.01 -0.02,3.35 0,6.71 0.02,10.06 0,0.35 -0.23,0.84 0.18,1.03 0.38,0.17 0.69,-0.25 0.99,-0.45 2.56,-1.74 5.33,-2.73 8.4900007,-2.56 3.51005,0.19 5.65005,1.95 6.35005,5.46 0.42,2.09 0.52,4.21 0.51,6.33 -0.02,3.86 0.05,7.73 -0.04,11.59 -0.02,1.12 0.37,1.5 1.39,1.6 0.61,0.05 1.55,-0.13 1.74,0.47 0.26,0.85 0.12,1.84 0.1,2.77 -0.01,0.41 -0.69,0.37 -0.69,0.37 l -11.4700504,0.01 c 0,0 -0.81,-0.29 -0.8,-0.85 0.01,-0.38 0.04,-0.77 -0.01,-1.15 -0.13,-1.01 0.32,-1.52 1.31,-1.56 1.0600004,-0.05 1.3800004,-0.55 1.3500004,-1.63 -0.14,-4.84 0.16,-9.68 -0.18,-14.51 -0.26,-3.66 -2.1100004,-4.95 -5.6700007,-3.99 -0.25,0.07 -0.49,0.15 -0.73,0.22 -2.57,0.8 -2.79,1.09 -2.79,3.71 0.01,2.3 0.01,4.59 0.01,6.88 z M -109.26603,122.56 c 0,-4.75 -0.02,-9.51 0.02,-14.26 0.01,-0.92 -0.17,-1.47 -1.19,-1.45 -0.16,0 -0.33,-0.07 -0.5,-0.1 -1.56,-0.27 -2.24,-1.47 -1.69,-2.92 0.14,-0.37 0.41,-0.38 0.7,-0.42 2.98,-0.41 5.97,-0.81 8.94,-1.24 0.85,-0.12 0.88,0.33 0.88,0.96 -0.01,3.01 -0.01,6.03 0,9.04 0,0.4 -0.18,0.96 0.27,1.16 0.36,0.16 0.66,-0.3 0.96,-0.52 4.729999,-3.51 12.459999,-2.61 14.889999,4.48 1.89,5.51 1.91,11.06 -0.96,16.28 -2.37,4.31 -6.19,6.49 -11.15,6.59 -3.379999,0.07 -6.679999,-0.3 -9.909999,-1.37 -0.93,-0.31 -1.3,-0.78 -1.28,-1.83 0.05,-4.81 0.02,-9.6 0.02,-14.4 z m 7.15,3.89 c 0,2.76 0.02,5.52 -0.01,8.28 -0.01,0.76 0.18,1.29 0.91,1.64 1.899999,0.9 4.299999,0.5 5.759999,-1.01 0.97,-1 1.56,-2.21 1.96,-3.52 1.03,-3.36 0.97,-6.78 0.61,-10.22 a 9.991,9.991 0 0 0 -0.93,-3.29 c -1.47,-3.06 -4.67,-3.85 -7.439999,-1.86 -0.6,0.43 -0.88,0.93 -0.87,1.7 0.04,2.76 0.01,5.52 0.01,8.28 z"
-     fill="#4280f6"
-     id="path2" />
-  <path
-     d="m 68.644019,137.2 c -1.62,1.46 -3.41,2.56 -5.62,2.96 -4.4,0.8 -8.7,-1.39 -10.49,-5.49 -2.31,-5.31 -2.3,-10.67 -0.1,-15.98 2.31,-5.58 8.29,-8.65 14.24,-7.46 1.71,0.34 1.9,0.18 1.9,-1.55 0,-0.68 -0.05,-1.36 0.01,-2.04 0.09,-1.02 -0.25,-1.54 -1.34,-1.43 -0.64,0.06 -1.26,-0.1 -1.88,-0.21 -1.32,-0.24 -1.6,-0.62 -1.37,-1.97 0.07,-0.41 0.25,-0.57 0.65,-0.62 2.63,-0.33 5.27,-0.66 7.9,-1.02 1.04,-0.14 1.17,0.37 1.17,1.25 -0.02,10.23 -0.02,20.45 -0.01,30.68 v 1.02 c 0.02,0.99 0.35,1.6 1.52,1.47 0.52,-0.06 1.35,-0.27 1.25,0.73 -0.08,0.8 0.58,1.93 -0.94,2.18 -1.29,0.22 -2.51,0.69 -3.86,0.65 -2.04,-0.06 -2.3,-0.23 -2.76,-2.19 -0.09,-0.3 0.06,-0.67 -0.27,-0.98 z m -0.07,-12.46 c 0,-2.8 -0.04,-5.6 0.02,-8.39 0.02,-0.9 -0.28,-1.47 -1.05,-1.81 -3.18,-1.4 -7.54,-0.8 -9.3,2.87 -0.83,1.74 -1.31,3.54 -1.49,5.46 -0.28,2.93 -0.38,5.83 0.61,8.65 0.73,2.09 1.81,3.9 4.11,4.67 2.49,0.83 4.55,-0.04 6.5,-1.48 0.54,-0.4 0.62,-0.95 0.61,-1.57 -0.02,-2.8 -0.01,-5.6 -0.01,-8.4 z m 28.79,2.53 c 0,3.24 0.04,5.83 -0.02,8.41 -0.02,1 0.19,1.49 1.309998,1.41 0.55,-0.04 1.460003,-0.46 1.520003,0.73 0.05,1.02 0.1,1.89 -1.330003,2.08 -1.289998,0.17 -2.559998,0.51 -3.889998,0.48 -1.88,-0.05 -2.15,-0.26 -2.42,-2.15 -0.04,-0.27 0.14,-0.65 -0.22,-0.79 -0.34,-0.13 -0.5,0.24 -0.72,0.42 -3.61,3 -8.15,3.4 -11.64,1.08 -1.61,-1.07 -2.49,-2.63 -2.67,-4.43 -0.51,-5.13 0.77,-7.91 6.3,-10.22 2.44,-1.02 5.07,-1.27 7.68,-1.49 0.77,-0.07 1.03,-0.28 1.02,-1.05 -0.03,-1.48 -0.05,-2.94 -0.64,-4.36 -0.59,-1.42 -1.67,-1.92 -3.08,-2.03 -3.04,-0.24 -5.88,0.5 -8.63,1.71 -0.51,0.23 -1.19,0.75 -1.48,-0.13 -0.26,-0.77 -1.35,-1.61 0.05,-2.47 3.27,-2 6.7,-3.44 10.61,-3.42 1.44,0.01 2.88,0.27 4.21,0.81 2.67,1.08 3.44,3.4 3.8,5.99 0.46,3.37 0.1,6.73 0.24,9.42 z m -5.09,2.9 c 0,-1.23 -0.01,-2.46 0,-3.69 0,-0.52 -0.06,-0.98 -0.75,-0.84 -1.45,0.3 -2.93,0.28 -4.37,0.69 -3.71,1.04 -5.46,4.48 -3.97,8.03 0.51,1.22 1.48,1.98 2.79,2.16 2.01,0.28 3.86,-0.29 5.6,-1.28 0.54,-0.31 0.73,-0.76 0.72,-1.37 -0.05,-1.23 -0.02,-2.47 -0.02,-3.7 z m 43.060001,-2.89 c 0,2.72 0.01,5.43 -0.01,8.15 0,0.66 0.02,1.21 0.91,1.12 0.54,-0.06 0.99,0.12 0.86,0.75 -0.15,0.71 0.56,1.7 -0.58,2.09 -1.55,0.52 -3.16,0.59 -4.77,0.4 -0.99,-0.12 -1.12,-1.01 -1.18,-1.73 -0.08,-1.15 -0.16,-1.45 -1.24,-0.54 -3.41,2.87 -8.05,3.17 -11.43,0.88 -1.75,-1.18 -2.49,-2.91 -2.7,-4.94 -0.64,-6.24 3.16,-8.74 7.83,-10.17 2.04,-0.62 4.14,-0.8 6.24,-0.99 0.81,-0.07 1,-0.36 0.98,-1.09 -0.04,-1.31 0.04,-2.62 -0.42,-3.89 -0.57,-1.57 -1.53,-2.34 -3.18,-2.45 -3.03,-0.21 -5.88,0.46 -8.64,1.66 -0.6,0.26 -1.25,0.81 -1.68,-0.2 -0.34,-0.8 -1.08,-1.61 0.16,-2.36 4.12,-2.5 8.44,-4.16 13.36,-3.07 3.21,0.71 4.89,2.91 5.26,6.34 0.18,1.69 0.22,3.37 0.22,5.07 0.01,1.66 0.01,3.32 0.01,4.97 z m -5.09,2.54 c 0,-1.27 -0.03,-2.54 0.01,-3.81 0.02,-0.74 -0.27,-1.02 -0.98,-0.92 -1.21,0.17 -2.43,0.28 -3.62,0.55 -3.72,0.83 -5.47,3.48 -4.82,7.21 0.29,1.66 1.57,2.94 3.21,3.16 2.02,0.27 3.85,-0.34 5.57,-1.34 0.49,-0.29 0.64,-0.73 0.63,-1.29 -0.02,-1.18 0,-2.37 0,-3.56 z"
-     fill="#c8dbfb"
-     id="path4" />
-  <path
-     d="m 26.314019,125.77 c 0,-2.89 -0.05,-5.77 0.02,-8.66 0.03,-1.04 -0.33,-1.39 -1.31,-1.24 a 0.7,0.7 0 0 1 -0.25,0 c -0.57,-0.18 -1.44,0.48 -1.68,-0.58 -0.35,-1.48 -0.02,-2.3 1.21,-2.7 1.3,-0.43 2.16,-1.26 2.76,-2.46 0.78,-1.56 1.44,-3.17 1.91,-4.84 0.18,-0.63 0.47,-0.86 1.15,-0.88 3.28,-0.09 3.27,-0.11 3.32,3.17 0.01,1.06 0.09,2.12 0.09,3.18 -0.01,0.67 0.27,0.89 0.91,0.88 1.61,-0.02 3.23,0.03 4.84,-0.02 0.77,-0.02 1.01,0.23 1.03,1.01 0.08,3.27 0.1,3.27 -3.09,3.27 -0.93,0 -1.87,0.03 -2.8,-0.01 -0.67,-0.02 -0.89,0.26 -0.88,0.91 0.04,5.43 0.04,10.86 0.12,16.29 0.02,1.7 0.75,2.26 2.46,2.1 1.1,-0.1 2.19,-0.26 3.23,-0.65 0.59,-0.22 0.89,-0.09 1.14,0.53 0.93,2.29 0.92,2.37 -1.32,3.52 -2.54,1.3 -5.22,1.99 -8.1,1.79 -2.27,-0.16 -3.68,-1.27 -4.35,-3.45 -0.3,-0.98 -0.41,-1.99 -0.41,-3.01 z m -97.67005,-8.99 c 0.57,-0.84 1.11,-1.74 1.76,-2.55 1.68,-2.09 3.68,-3.62 6.54,-3.66 1.08,-0.01 1.63,0.28 1.57,1.52 -0.1,2.08 -0.05,4.16 -0.02,6.24 0.01,0.74 -0.17,0.96 -0.96,0.76 -2.36,-0.59 -4.71,-0.42 -7.03,0.28 -0.8,0.24 -1.16,0.62 -1.15,1.52 0.05,4.5 0.04,9 0,13.5 -0.01,0.89 0.29,1.16 1.15,1.2 1.23,0.06 2.44,0.32 3.67,0.39 0.75,0.05 0.91,0.38 0.89,1.04 -0.06,2.86 0.29,2.28 -2.25,2.3 -4.2,0.04 -8.41,-0.02 -12.61,0.03 -0.91,0.01 -1.39,-0.18 -1.22,-1.18 0.02,-0.12 0,-0.25 0,-0.38 0.02,-2.1 -0.24,-1.88 1.77,-2.04 1.33,-0.11 1.6,-0.67 1.58,-1.9 -0.07,-5.35 -0.04,-10.7 -0.02,-16.05 0,-0.78 -0.17,-1.2 -1,-1.46 -2.21,-0.68 -2.7,-1.69 -2.22,-3.99 0.11,-0.52 0.45,-0.56 0.82,-0.62 2.22,-0.34 4.44,-0.7 6.67,-0.99 0.99,-0.13 1.82,0.7 1.84,1.76 0.03,1.4 0.03,2.8 0.04,4.2 -0.01,0.02 0.06,0.04 0.18,0.08 z m 25.24,6.59 c 0,3.69 0.04,7.38 -0.03,11.07 -0.02,1.04 0.31,1.48 1.32,1.49 0.29,0 0.59,0.12 0.88,0.13 0.93,0.01 1.18,0.47 1.16,1.37 -0.05,2.19 0,2.19 -2.24,2.19 -3.48,0 -6.96,-0.04 -10.44,0.03 -1.09,0.02 -1.47,-0.33 -1.3,-1.36 0.02,-0.12 0.02,-0.26 0,-0.38 -0.28,-1.39 0.39,-1.96 1.7,-1.9 1.36,0.06 1.76,-0.51 1.74,-1.88 -0.09,-5.17 -0.08,-10.35 0,-15.53 0.02,-1.22 -0.32,-1.87 -1.52,-2.17 -0.57,-0.14 -1.47,-0.11 -1.57,-0.85 -0.15,-1.04 -0.05,-2.11 0.01,-3.17 0.02,-0.34 0.44,-0.35 0.73,-0.39 2.81,-0.39 5.63,-0.77 8.44,-1.18 0.92,-0.14 1.15,0.2 1.14,1.09 -0.04,3.8 -0.02,7.62 -0.02,11.44 z"
-     fill="#4280f6"
-     id="path6" />
-  <path
-     d="m 101.44402,125.64 c 0,-3.18 -0.03,-6.37 0.02,-9.55 0.02,-0.94 -0.26,-1.36 -1.22,-1.22 -0.21,0.03 -0.430003,0.04 -0.630003,0 -0.51,-0.12 -1.35,0.39 -1.44,-0.55 -0.08,-0.85 -0.429998,-1.87 0.93,-2.24 2.080003,-0.57 2.720003,-2.39 3.350003,-4.17 0.31,-0.88 0.62,-1.76 0.87,-2.66 0.18,-0.64 0.52,-0.85 1.19,-0.84 2.46,0.05 2,-0.15 2.04,2.04 0.02,1.1 0.08,2.21 -0.02,3.31 -0.11,1.16 0.46,1.52 1.46,1.53 1.78,0.01 3.57,0.04 5.35,-0.01 0.82,-0.02 1.12,0.23 1.11,1.08 -0.05,2.86 0.19,2.49 -2.42,2.51 -1.53,0.01 -3.06,0.02 -4.59,-0.01 -0.65,-0.01 -0.9,0.22 -0.9,0.89 0.02,5.52 0,11.04 0.03,16.56 0,0.67 0.14,1.34 0.25,2.01 0.17,1.04 1.17,1.62 2.59,1.42 1.29,-0.19 2.57,-0.49 3.86,-0.69 0.43,-0.07 1.05,-0.47 1.19,0.4 0.12,0.75 1.05,1.61 -0.09,2.24 -2.09,1.16 -4.28,2.07 -6.71,2.16 -1.05,0.04 -2.13,0.2 -3.16,-0.14 -1.92,-0.65 -3.03,-2.28 -3.05,-4.51 -0.02,-3.19 -0.01,-6.37 -0.01,-9.56 z"
-     fill="#c8dbfb"
-     id="path8" />
-  <path
-     d="m -50.816031,95.21 c 0.19,2.160002 1.85,3.240002 2.82,4.740002 0.25,0.379998 0.48,0.109998 0.67,-0.16 0.21,-0.31 0.6,-1.21 1.15,-1.28 -0.35,1.38 -0.04,3.149998 0.16,4.449998 0.49,3.05 -1.22,5.64 -4.07,6.18 -3.38,0.65 -6.22,-2.21 -5.6,-5.62 0.23,-1.24 1.37,-2.5 0.77,-3.699998 -0.85,-1.7 0.54,-0.52 0.79,-0.22 1.04,1.199998 1.21,0.09 1.45,-0.55 0.24,-0.63 0.31,-1.31 0.47,-1.97 0.19,-0.770002 0.55,-1.400002 1.39,-1.870002 z"
-     fill="#4280f6"
-     id="path10" />
-</svg>

changedetectionio/static/js/browser-steps.js

@@ -208,7 +208,7 @@ $(document).ready(function () {
             console.log(x);
             if (x && first_available.length) {
                 // @todo will it let you click shit that has a layer ontop? probably not.
-                if (x['tagtype'] === 'text' || x['tagtype'] === 'number' || x['tagtype'] === 'email' || x['tagName'] === 'textarea' || x['tagtype'] === 'password' || x['tagtype'] === 'search') {
+                if (x['tagtype'] === 'text' || x['tagtype'] === 'email' || x['tagName'] === 'textarea' || x['tagtype'] === 'password' || x['tagtype'] === 'search') {
                     $('select', first_available).val('Enter text in field').change();
                     $('input[type=text]', first_available).first().val(x['xpath']);
                     $('input[placeholder="Value"]', first_available).addClass('ok').click().focus();
@@ -321,14 +321,8 @@ $(document).ready(function () {
             var s = '<div class="control">' + '<a data-step-index=' + i + ' class="pure-button button-secondary button-green button-xsmall apply" >Apply</a>&nbsp;';
             if (i > 0) {
                 // The first step never gets these (Goto-site)
-                s += `<a data-step-index="${i}" class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;` +
-                    `<a data-step-index="${i}" class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>`;
-
-                // if a screenshot is available
-                if (browser_steps_available_screenshots.includes(i.toString())) {
-                    var d = (browser_steps_last_error_step === i+1) ? 'before' : 'after';
-                    s += `&nbsp;<a data-step-index="${i}" class="pure-button button-secondary button-xsmall show-screenshot" title="Show screenshot from last run" data-type="${d}">Pic</a>&nbsp;`;
-                }
+                s += '<a data-step-index=' + i + ' class="pure-button button-secondary button-xsmall clear" >Clear</a>&nbsp;' +
+                    '<a data-step-index=' + i + ' class="pure-button button-secondary button-red button-xsmall remove" >Remove</a>';
             }
             s += '</div>';
             $(this).append(s)
@@ -443,24 +437,6 @@ $(document).ready(function () {
 
     });
 
-    $('ul#browser_steps li .control .show-screenshot').click(function (element) {
-        var step_n = $(event.currentTarget).data('step-index');
-        w = window.open(this.href, "_blank", "width=640,height=480");
-        const t = $(event.currentTarget).data('type');
-
-        const url = browser_steps_fetch_screenshot_image_url + `&step_n=${step_n}&type=${t}`;
-        w.document.body.innerHTML = `<!DOCTYPE html>
-            <html lang="en">
-                <body>
-                    <img src="${url}" style="width: 100%" alt="Browser Step at step ${step_n} from last run." title="Browser Step at step ${step_n} from last run."/>
-                </body>
-        </html>`;
-        w.document.title = `Browser Step at step ${step_n} from last run.`;
-    });
-
-    if (browser_steps_last_error_step) {
-        $("ul#browser_steps>li:nth-child("+browser_steps_last_error_step+")").addClass("browser-step-with-error");
-    }
 
     $("ul#browser_steps select").change(function () {
         set_greyed_state();

changedetectionio/static/js/diff-overview.js

@@ -1,13 +1,4 @@
 $(document).ready(function () {
-    var csrftoken = $('input[name=csrf_token]').val();
-    $.ajaxSetup({
-        beforeSend: function (xhr, settings) {
-            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
-                xhr.setRequestHeader("X-CSRFToken", csrftoken)
-            }
-        }
-    })
-
     // Load it when the #screenshot tab is in use, so we dont give a slow experience when waiting for the text diff to load
     window.addEventListener('hashchange', function (e) {
         toggle(location.hash);
@@ -24,71 +15,11 @@ $(document).ready(function () {
             $("#settings").hide();
         } else if (hash_name === '#extract') {
             $("#settings").hide();
-        } else {
+        }
+
+
+        else {
             $("#settings").show();
         }
     }
-
-    const article = $('.highlightable-filter')[0];
-
-    // We could also add the  'touchend' event for touch devices, but since
-    // most iOS/Android browsers already show a dialog when you select
-    // text (often with a Share option) we'll skip that
-    article.addEventListener('mouseup', dragTextHandler, false);
-    article.addEventListener('mousedown', clean, false);
-
-    function clean(event) {
-      $("#highlightSnippet").remove();
-    }
-
-
-    function dragTextHandler(event) {
-        console.log('mouseupped');
-
-        // Check if any text was selected
-        if (window.getSelection().toString().length > 0) {
-
-            // Find out how much (if any) user has scrolled
-            var scrollTop = (window.pageYOffset !== undefined) ? window.pageYOffset : (document.documentElement || document.body.parentNode || document.body).scrollTop;
-
-            // Get cursor position
-            const posX = event.clientX;
-            const posY = event.clientY + 20 + scrollTop;
-
-            // Append HTML to the body, create the "Tweet Selection" dialog
-            document.body.insertAdjacentHTML('beforeend', '<div id="highlightSnippet" style="position: absolute; top: ' + posY + 'px; left: ' + posX + 'px;"><div class="pure-form-message-inline" style="font-size: 70%">Ignore any change on any line which contains the selected text.</div><br><a data-mode="exact" href="javascript:void(0);" class="pure-button button-secondary button-xsmall">Ignore exact text</a>&nbsp;</div>');
-
-            if (/\d/.test(window.getSelection().toString())) {
-                // Offer regex replacement
-                document.getElementById("highlightSnippet").insertAdjacentHTML('beforeend', '<a data-mode="digit-regex"  href="javascript:void(0);" class="pure-button button-secondary button-xsmall">Ignore text including number changes</a>');
-            }
-
-            $('#highlightSnippet a').bind('click', function (e) {
-                if(!window.getSelection().toString().trim().length) {
-                    alert('Oops no text selected!');
-                    return;
-                }
-
-                $.ajax({
-                    type: "POST",
-                    url: highlight_submit_ignore_url,
-                    data: {'mode': $(this).data('mode'), 'selection': window.getSelection().toString()},
-                    statusCode: {
-                        400: function () {
-                            // More than likely the CSRF token was lost when the server restarted
-                            alert("There was a problem processing the request, please reload the page.");
-                        }
-                    }
-                }).done(function (data) {
-                    $("#highlightSnippet").html(data)
-                }).fail(function (data) {
-                    console.log(data);
-                    alert('There was an error communicating with the server.');
-                });
-            });
-
-        }
-    }
-
-
 });

changedetectionio/static/js/diff-render.js

@@ -1,120 +1,110 @@
-$(document).ready(function () {
-    var a = document.getElementById("a");
-    var b = document.getElementById("b");
-    var result = document.getElementById("result");
-    var inputs;
+var a = document.getElementById("a");
+var b = document.getElementById("b");
+var result = document.getElementById("result");
 
-    $('#jump-next-diff').click(function () {
+function changed() {
+  // https://github.com/kpdecker/jsdiff/issues/389
+  // I would love to use `{ignoreWhitespace: true}` here but it breaks the formatting
+  options = {
+    ignoreWhitespace: document.getElementById("ignoreWhitespace").checked,
+  };
 
-        var element = inputs[inputs.current];
-        var headerOffset = 80;
-        var elementPosition = element.getBoundingClientRect().top;
-        var offsetPosition = elementPosition - headerOffset + window.scrollY;
-
-        window.scrollTo({
-            top: offsetPosition,
-            behavior: "smooth",
-        });
-
-        inputs.current++;
-        if (inputs.current >= inputs.length) {
-            inputs.current = 0;
-        }
-    });
-
-    function changed() {
-        // https://github.com/kpdecker/jsdiff/issues/389
-        // I would love to use `{ignoreWhitespace: true}` here but it breaks the formatting
-        options = {
-            ignoreWhitespace: document.getElementById("ignoreWhitespace").checked,
-        };
-
-        var diff = Diff[window.diffType](a.textContent, b.textContent, options);
-        var fragment = document.createDocumentFragment();
-        for (var i = 0; i < diff.length; i++) {
-            if (diff[i].added && diff[i + 1] && diff[i + 1].removed) {
-                var swap = diff[i];
-                diff[i] = diff[i + 1];
-                diff[i + 1] = swap;
-            }
-
-            var node;
-            if (diff[i].removed) {
-                node = document.createElement("del");
-                node.classList.add("change");
-                const wrapper = node.appendChild(document.createElement("span"));
-                wrapper.appendChild(document.createTextNode(diff[i].value));
-            } else if (diff[i].added) {
-                node = document.createElement("ins");
-                node.classList.add("change");
-                const wrapper = node.appendChild(document.createElement("span"));
-                wrapper.appendChild(document.createTextNode(diff[i].value));
-            } else {
-                node = document.createTextNode(diff[i].value);
-            }
-            fragment.appendChild(node);
-        }
-
-        result.textContent = "";
-        result.appendChild(fragment);
-
-        // For nice mouse-over hover/title information
-        const removed_current_option = $('#diff-version option:selected')
-        if (removed_current_option) {
-            $('del').each(function () {
-                $(this).prop('title', 'Removed '+removed_current_option[0].label);
-            });
-        }
-        const inserted_current_option = $('#current-version option:selected')
-        if (removed_current_option) {
-            $('ins').each(function () {
-                $(this).prop('title', 'Inserted '+inserted_current_option[0].label);
-            });
-        }
-        // Set the list of possible differences to jump to
-        inputs = document.querySelectorAll('#diff-ui .change')
-        // Set the "current" diff pointer
-        inputs.current = 0;
-        // Goto diff
-        $('#jump-next-diff').click();
+  var diff = Diff[window.diffType](a.textContent, b.textContent, options);
+  var fragment = document.createDocumentFragment();
+  for (var i = 0; i < diff.length; i++) {
+    if (diff[i].added && diff[i + 1] && diff[i + 1].removed) {
+      var swap = diff[i];
+      diff[i] = diff[i + 1];
+      diff[i + 1] = swap;
     }
 
-    $('.needs-localtime').each(function () {
-        for (var option of this.options) {
-            var dateObject = new Date(option.value * 1000);
-            option.label = dateObject.toLocaleString(undefined, {dateStyle: "full", timeStyle: "medium"});
-        }
-    })
-    onDiffTypeChange(
-        document.querySelector('#settings [name="diff_type"]:checked'),
-    );
-    changed();
-
-    a.onpaste = a.onchange = b.onpaste = b.onchange = changed;
-
-    if ("oninput" in a) {
-        a.oninput = b.oninput = changed;
+    var node;
+    if (diff[i].removed) {
+      node = document.createElement("del");
+      node.classList.add("change");
+      const wrapper = node.appendChild(document.createElement("span"));
+      wrapper.appendChild(document.createTextNode(diff[i].value));
+    } else if (diff[i].added) {
+      node = document.createElement("ins");
+      node.classList.add("change");
+      const wrapper = node.appendChild(document.createElement("span"));
+      wrapper.appendChild(document.createTextNode(diff[i].value));
     } else {
-        a.onkeyup = b.onkeyup = changed;
+      node = document.createTextNode(diff[i].value);
     }
+    fragment.appendChild(node);
+  }
 
-    function onDiffTypeChange(radio) {
-        window.diffType = radio.value;
-        // Not necessary
-        //	document.title = "Diff " + radio.value.slice(4);
+  result.textContent = "";
+  result.appendChild(fragment);
+
+  // Jump at start
+  inputs.current = 0;
+  next_diff();
+}
+
+window.onload = function () {
+  /* Convert what is options from UTC time.time() to local browser time */
+  var diffList = document.getElementById("diff-version");
+  if (typeof diffList != "undefined" && diffList != null) {
+    for (var option of diffList.options) {
+      var dateObject = new Date(option.value * 1000);
+      option.label = dateObject.toLocaleString();
     }
+  }
 
-    var radio = document.getElementsByName("diff_type");
-    for (var i = 0; i < radio.length; i++) {
-        radio[i].onchange = function (e) {
-            onDiffTypeChange(e.target);
-            changed();
-        };
-    }
+  /* Set current version date as local time in the browser also */
+  var current_v = document.getElementById("current-v-date");
+  var dateObject = new Date(newest_version_timestamp * 1000);
+  current_v.innerHTML = dateObject.toLocaleString();
+  onDiffTypeChange(
+    document.querySelector('#settings [name="diff_type"]:checked'),
+  );
+  changed();
+};
 
-    document.getElementById("ignoreWhitespace").onchange = function (e) {
-        changed();
-    };
+a.onpaste = a.onchange = b.onpaste = b.onchange = changed;
 
-});
+if ("oninput" in a) {
+  a.oninput = b.oninput = changed;
+} else {
+  a.onkeyup = b.onkeyup = changed;
+}
 
+function onDiffTypeChange(radio) {
+  window.diffType = radio.value;
+  // Not necessary
+  //	document.title = "Diff " + radio.value.slice(4);
+}
+
+var radio = document.getElementsByName("diff_type");
+for (var i = 0; i < radio.length; i++) {
+  radio[i].onchange = function (e) {
+    onDiffTypeChange(e.target);
+    changed();
+  };
+}
+
+document.getElementById("ignoreWhitespace").onchange = function (e) {
+  changed();
+};
+
+var inputs = document.getElementsByClassName("change");
+inputs.current = 0;
+
+function next_diff() {
+  var element = inputs[inputs.current];
+  var headerOffset = 80;
+  var elementPosition = element.getBoundingClientRect().top;
+  var offsetPosition = elementPosition - headerOffset + window.scrollY;
+
+  window.scrollTo({
+    top: offsetPosition,
+    behavior: "smooth",
+  });
+
+  inputs.current++;
+  if (inputs.current >= inputs.length) {
+    inputs.current = 0;
+  }
+}

changedetectionio/static/js/global-settings.js

@@ -32,10 +32,5 @@ $(document).ready(function () {
         window.getSelection().removeAllRanges();
 
     });
-
-    $("#notification-token-toggle").click(function (e) {
-        e.preventDefault();
-        $('#notification-tokens-info').toggle();
-    });
 });
 

changedetectionio/static/js/recheck-proxy.js

@@ -1,87 +0,0 @@
-$(function () {
-    /* add container before each proxy location to show status */
-
-    var option_li = $('.fetch-backend-proxy li').filter(function() {
-        return $("input",this)[0].value.length >0;
-    });
-
-    //var option_li = $('.fetch-backend-proxy li');
-    var isActive = false;
-    $(option_li).prepend('<div class="proxy-status"></div>');
-    $(option_li).append('<div class="proxy-timing"></div><div class="proxy-check-details"></div>');
-
-    function set_proxy_check_status(proxy_key, state) {
-        // select input by value name
-        const proxy_li = $('input[value="' + proxy_key + '" ]').parent();
-        if (state['status'] === 'RUNNING') {
-            $('.proxy-status', proxy_li).html('<span class="spinner"></span>');
-        }
-        if (state['status'] === 'OK') {
-            $('.proxy-status', proxy_li).html('<span style="color: green; font-weight: bold" >OK</span>');
-            $('.proxy-check-details', proxy_li).html(state['text']);
-        }
-        if (state['status'] === 'ERROR' || state['status'] === 'ERROR OTHER') {
-            $('.proxy-status', proxy_li).html('<span style="color: red; font-weight: bold" >X</span>');
-            $('.proxy-check-details', proxy_li).html(state['text']);
-        }
-        $('.proxy-timing', proxy_li).html(state['time']);
-    }
-
-
-    function pollServer() {
-        if (isActive) {
-            window.setTimeout(function () {
-                $.ajax({
-                    url: proxy_recheck_status_url,
-                    success: function (data) {
-                        var all_done = true;
-                        $.each(data, function (proxy_key, state) {
-                            set_proxy_check_status(proxy_key, state);
-                            if (state['status'] === 'RUNNING') {
-                                all_done = false;
-                            }
-                        });
-
-                        if (all_done) {
-                            console.log("Shutting down poller, all done.")
-                            isActive = false;
-                        } else {
-                            pollServer();
-                        }
-                    },
-                    error: function () {
-                        //ERROR HANDLING
-                        pollServer();
-                    }
-                });
-            }, 2000);
-        }
-    }
-
-    $('#check-all-proxies').click(function (e) {
-        e.preventDefault()
-        $('body').addClass('proxy-check-active');
-        $('.proxy-check-details').html('');
-        $('.proxy-status').html('<span class="spinner"></span>').fadeIn();
-        $('.proxy-timing').html('');
-
-        // Request start, needs CSRF?
-        $.ajax({
-            type: "GET",
-            url: recheck_proxy_start_url,
-        }).done(function (data) {
-            $.each(data, function (proxy_key, state) {
-                set_proxy_check_status(proxy_key, state['status'])
-            });
-            isActive = true;
-            pollServer();
-
-        }).fail(function (data) {
-            console.log(data);
-            alert('There was an error communicating with the server.');
-        });
-
-    });
-
-});
-

changedetectionio/static/js/toggle-theme.js

@@ -3,50 +3,45 @@
  * Toggles theme between light and dark mode.
  */
 $(document).ready(function () {
-    const button = document.getElementById("toggle-light-mode");
+  const button = document.getElementById("toggle-light-mode");
 
-    button.onclick = () => {
-        const htmlElement = document.getElementsByTagName("html");
-        const isDarkMode = htmlElement[0].dataset.darkmode === "true";
-        htmlElement[0].dataset.darkmode = !isDarkMode;
-        setCookieValue(!isDarkMode);
-    };
+  button.onclick = () => {
+    const htmlElement = document.getElementsByTagName("html");
+    const isDarkMode = htmlElement[0].dataset.darkmode === "true";
+    htmlElement[0].dataset.darkmode = !isDarkMode;
+    setCookieValue(!isDarkMode);
+  };
 
-    const setCookieValue = (value) => {
-        document.cookie = `css_dark_mode=${value};max-age=31536000;path=/`
-    }
+  const setCookieValue = (value) => {
+    document.cookie = `css_dark_mode=${value};max-age=31536000;path=/`
+  }
 
-    // Search input box behaviour
+  // Search input box behaviour
     const toggle_search = document.getElementById("toggle-search");
-    const search_q = document.getElementById("search-q");
-    if(search_q) {
-      window.addEventListener('keydown', function (e) {
-        if (e.altKey == true && e.keyCode == 83) {
-          search_q.classList.toggle('expanded');
-          search_q.focus();
-        }
-      });
+  const search_q = document.getElementById("search-q");
+  window.addEventListener('keydown', function (e) {
 
-      search_q.onkeydown = (e) => {
-        var key = e.keyCode || e.which;
-        if (key === 13) {
-          document.searchForm.submit();
-        }
-      };
-      toggle_search.onclick = () => {
-        // Could be that they want to search something once text is in there
-        if (search_q.value.length) {
-          document.searchForm.submit();
-        } else {
-          // If not..
-          search_q.classList.toggle('expanded');
-          search_q.focus();
-        }
-      };
+    if (e.altKey == true && e.keyCode == 83)
+      search_q.classList.toggle('expanded');
+      search_q.focus();
+  });
+
+
+  search_q.onkeydown = (e) => {
+    var key = e.keyCode || e.which;
+    if (key === 13) {
+      document.searchForm.submit();
     }
+  };
+  toggle_search.onclick = () => {
+    // Could be that they want to search something once text is in there
+    if (search_q.value.length) {
+      document.searchForm.submit();
+    } else {
+      // If not..
+      search_q.classList.toggle('expanded');
+      search_q.focus();
+    }
+  };
 
-    $('#heart-us').click(function () {
-        $("#overlay").toggleClass('visible');
-        heartpath.style.fill = document.getElementById("overlay").classList.contains("visible") ? '#ff0000' : 'var(--color-background)';
-    });
 });

changedetectionio/static/js/watch-overview.js

@@ -4,14 +4,6 @@ $(function () {
         $(this).closest('.unviewed').removeClass('unviewed');
     });
 
-    $('td[data-timestamp]').each(function () {
-        $(this).prop('title', new Intl.DateTimeFormat(undefined,
-            {
-                dateStyle: 'full',
-                timeStyle: 'long'
-            }).format($(this).data('timestamp') * 1000));
-    })
-
     $("#checkbox-assign-tag").click(function (e) {
         $('#op_extradata').val(prompt("Enter a tag name"));
     });

changedetectionio/static/js/watch-settings.js

@@ -42,8 +42,4 @@ $(document).ready(function () {
         $('#notification_urls').val('');
         e.preventDefault();
     });
-    $("#notification-token-toggle").click(function (e) {
-        e.preventDefault();
-        $('#notification-tokens-info').toggle();
-    });
 });

changedetectionio/static/styles/diff.css

@@ -187,10 +187,6 @@ ins {
     padding: 0.5em; }
   #settings ins {
     padding: 0.5em; }
-  #settings option:checked {
-    font-weight: bold; }
-  #settings [type=radio], #settings [type=checkbox] {
-    vertical-align: middle; }
 
 .source {
   position: absolute;
@@ -222,10 +218,3 @@ td#diff-col div {
   text-align: center; }
   .tab-pane-inner#screenshot img {
     max-width: 99%; }
-
-#highlightSnippet {
-  background: var(--color-background);
-  padding: 1em;
-  border-radius: 5px;
-  background: var(--color-background);
-  box-shadow: 1px 1px 4px var(--color-shadow-jump); }

changedetectionio/static/styles/scss/diff.scss

@@ -77,13 +77,6 @@ ins {
   ins {
     padding: 0.5em;
   }
-
-  option:checked {
-    font-weight: bold;
-  }
-  [type=radio],[type=checkbox] {
-    vertical-align: middle;
-  }
 }
 
 .source {
@@ -126,11 +119,3 @@ td#diff-col div {
     max-width: 99%;
   }
 }
-
-#highlightSnippet {
-  background: var(--color-background);
-  padding: 1em;
-  border-radius: 5px;
-  background: var(--color-background);
-  box-shadow: 1px 1px 4px var(--color-shadow-jump);
-}

changedetectionio/static/styles/scss/parts/_browser-steps.scss

@@ -6,10 +6,6 @@
   }
 
   li {
-    &.browser-step-with-error {
-      background-color: #ffd6d6;
-      border-radius: 4px;
-    }
     &:not(:first-child) {
       &:hover {
         opacity: 1.0;
@@ -48,7 +44,7 @@
 #browser-steps .flex-wrapper {
   display: flex;
   flex-flow: row;
-  height: 70vh;
+  height: 600px; /*@todo make this dynamic */
 }
 
 /*  this is duplicate :( */

changedetectionio/static/styles/scss/parts/_darkmode.scss

@@ -1,25 +0,0 @@
-
-#toggle-light-mode {
-/*  width: 3rem;*/
-  /* default */
-  .icon-dark {
-    display: none;
-  }
-
-}
-
-html[data-darkmode="true"] {
-  #toggle-light-mode {
-    .icon-light {
-      display: none;
-    }
-
-    .icon-dark {
-      display: block;
-    }
-  }
-}
-
-
-
-

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -7,7 +7,6 @@ ul#requests-extra_proxies {
     }
 
   }
-
   /* each proxy entry is a `table` */
   table {
     tr {
@@ -16,47 +15,3 @@ ul#requests-extra_proxies {
   }
 }
 
-#request {
-  /* Auto proxy scan/checker */
-  label[for=proxy] {
-    display: inline-block;
-  }
-}
-
-body.proxy-check-active {
-  #request {
-    .proxy-status {
-      width: 2em;
-    }
-
-    .proxy-check-details {
-      font-size: 80%;
-      color: #555;
-      display: block;
-      padding-left: 4em;
-    }
-
-    .proxy-timing {
-      font-size: 80%;
-      padding-left: 1rem;
-      color: var(--color-link);
-    }
-  }
-}
-
-
-#recommended-proxy {
-  display: grid;
-  gap: 2rem;
-    @media  (min-width: 991px) {
-      grid-template-columns: repeat(2, 1fr);
-    }
-
-  > div {
-    border: 1px #aaa solid;
-    border-radius: 4px;
-    padding: 1em;
-  }
-
-  padding-bottom: 1em;
-}

changedetectionio/static/styles/scss/parts/_love.scss

@@ -1,38 +0,0 @@
-#overlay {
-
-  opacity: 0.95;
-  position: fixed;
-
-  width: 350px;
-  max-width: 100%;
-  height: 100%;
-  top: 0;
-  right: -350px;
-  background-color: var(--color-table-stripe);
-  z-index: 2;
-
-  transform: translateX(0);
-  transition: transform .5s ease;
-
-
-  &.visible {
-    transform: translateX(-100%);
-
-  }
-
-  .content {
-    font-size: 0.875rem;
-    padding: 1rem;
-    margin-top: 5rem;
-    max-width: 400px;
-    color: var(--color-watch-table-row-text);
-  }
-}
-
-#heartpath {
-  &:hover {
-    fill: #ff0000 !important;
-    transition: all ease 0.3s !important;
-  }
-  transition: all ease 0.3s !important;
-}

changedetectionio/static/styles/scss/parts/_menu.scss

@@ -1,25 +0,0 @@
-.pure-menu-link {
-  padding: 0.5rem 1em;
-  line-height: 1.2rem;
-}
-
-.pure-menu-item {
-  svg {
-    height: 1.2rem;
-  }
-  * {
-    vertical-align: middle;
-  }
-  .github-link {
-    height: 1.8rem;
-    display: block;
-    svg {
-      height: 100%;
-    }
-  }
-  .bi-heart {
-    &:hover {
-      cursor: pointer;
-    }
-  }
-}

changedetectionio/static/styles/scss/parts/_visualselector.scss

@@ -1,28 +0,0 @@
-
-#selector-wrapper {
-  height: 100%;
-  max-height: 70vh;
-  overflow-y: scroll;
-  position: relative;
-
-  //width: 100%;
-  >img {
-    position: absolute;
-    z-index: 4;
-    max-width: 100%;
-  }
-
-  >canvas {
-    position: relative;
-    z-index: 5;
-    max-width: 100%;
-
-    &:hover {
-      cursor: pointer;
-    }
-  }
-}
-
-#selector-current-xpath {
-  font-size: 80%;
-}

changedetectionio/static/styles/scss/styles.scss

@@ -8,14 +8,10 @@
 @import "parts/_pagination";
 @import "parts/_spinners";
 @import "parts/_variables";
-@import "parts/_darkmode";
-@import "parts/_menu";
-@import "parts/_love";
 
 body {
   color: var(--color-text);
   background: var(--color-background-page);
-  font-family: Helvetica Neue, Helvetica, Lucida Grande, Arial, Ubuntu, Cantarell, Fira Sans, sans-serif;
 }
 
 .visually-hidden {
@@ -58,6 +54,27 @@ a.github-link {
   }
 }
 
+#toggle-light-mode {
+  width: 3rem;
+  .icon-dark {
+    display: none;
+  }
+
+  &.dark {
+    .icon-light {
+      display: none;
+    }
+
+    .icon-dark {
+      display: block;
+    }
+  }
+}
+
+#toggle-search {
+  width: 2rem;
+}
+
 #search-q {
   opacity: 0;
   -webkit-transition: all .9s ease;
@@ -469,11 +486,7 @@ footer {
   padding: 10px;
 
   &#left-sticky {
-    left: 0;
-    position: fixed;
-    border-top-right-radius: 5px;
-    border-bottom-right-radius: 5px;
-    box-shadow: 1px 1px 4px var(--color-shadow-jump);
+    left: 0px;
   }
 
   &#right-sticky {
@@ -941,7 +954,32 @@ ul {
   }
 }
 
-@import "parts/_visualselector";
+#selector-wrapper {
+  height: 100%;
+  overflow-y: scroll;
+  position: relative;
+
+  //width: 100%;
+  >img {
+    position: absolute;
+    z-index: 4;
+    max-width: 100%;
+  }
+
+  >canvas {
+    position: relative;
+    z-index: 5;
+    max-width: 100%;
+
+    &:hover {
+      cursor: pointer;
+    }
+  }
+}
+
+#selector-current-xpath {
+  font-size: 80%;
+}
 
 #webdriver-override-options {
   input[type="number"] {
@@ -1080,4 +1118,3 @@ ul {
   border-radius: 3px;
   white-space: nowrap;
 }
-

changedetectionio/static/styles/styles.css

@@ -26,9 +26,6 @@
   #browser_steps li {
     list-style: decimal;
     padding: 5px; }
-    #browser_steps li.browser-step-with-error {
-      background-color: #ffd6d6;
-      border-radius: 4px; }
     #browser_steps li:not(:first-child):hover {
       opacity: 1.0; }
     #browser_steps li .control {
@@ -53,7 +50,8 @@
 #browser-steps .flex-wrapper {
   display: flex;
   flex-flow: row;
-  height: 70vh; }
+  height: 600px;
+  /*@todo make this dynamic */ }
 
 /*  this is duplicate :( */
 #browsersteps-selector-wrapper {
@@ -97,37 +95,6 @@ ul#requests-extra_proxies {
   ul#requests-extra_proxies table tr {
     display: inline; }
 
-#request {
-  /* Auto proxy scan/checker */ }
-  #request label[for=proxy] {
-    display: inline-block; }
-
-body.proxy-check-active #request .proxy-status {
-  width: 2em; }
-
-body.proxy-check-active #request .proxy-check-details {
-  font-size: 80%;
-  color: #555;
-  display: block;
-  padding-left: 4em; }
-
-body.proxy-check-active #request .proxy-timing {
-  font-size: 80%;
-  padding-left: 1rem;
-  color: var(--color-link); }
-
-#recommended-proxy {
-  display: grid;
-  gap: 2rem;
-  padding-bottom: 1em; }
-  @media (min-width: 991px) {
-    #recommended-proxy {
-      grid-template-columns: repeat(2, 1fr); } }
-  #recommended-proxy > div {
-    border: 1px #aaa solid;
-    border-radius: 4px;
-    padding: 1em; }
-
 .pagination-page-info {
   color: #fff;
   font-size: 0.85rem;
@@ -316,6 +283,10 @@ html[data-darkmode="true"] {
   --color-icon-github-hover: var(--color-grey-700);
   --color-watch-table-error: var(--color-light-red);
   --color-watch-table-row-text: var(--color-grey-800); }
+  html[data-darkmode="true"] #toggle-light-mode .icon-light {
+    display: none; }
+  html[data-darkmode="true"] #toggle-light-mode .icon-dark {
+    display: block; }
   html[data-darkmode="true"] .icon-spread {
     filter: hue-rotate(-10deg) brightness(1.5); }
   html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
@@ -330,68 +301,9 @@ html[data-darkmode="true"] {
     html[data-darkmode="true"] .watch-table .unviewed.error {
       color: var(--color-watch-table-error); }
 
-#toggle-light-mode {
-  /*  width: 3rem;*/
-  /* default */ }
-  #toggle-light-mode .icon-dark {
-    display: none; }
-
-html[data-darkmode="true"] #toggle-light-mode .icon-light {
-  display: none; }
-
-html[data-darkmode="true"] #toggle-light-mode .icon-dark {
-  display: block; }
-
-.pure-menu-link {
-  padding: 0.5rem 1em;
-  line-height: 1.2rem; }
-
-.pure-menu-item svg {
-  height: 1.2rem; }
-
-.pure-menu-item * {
-  vertical-align: middle; }
-
-.pure-menu-item .github-link {
-  height: 1.8rem;
-  display: block; }
-  .pure-menu-item .github-link svg {
-    height: 100%; }
-
-.pure-menu-item .bi-heart:hover {
-  cursor: pointer; }
-
-#overlay {
-  opacity: 0.95;
-  position: fixed;
-  width: 350px;
-  max-width: 100%;
-  height: 100%;
-  top: 0;
-  right: -350px;
-  background-color: var(--color-table-stripe);
-  z-index: 2;
-  transform: translateX(0);
-  transition: transform .5s ease; }
-  #overlay.visible {
-    transform: translateX(-100%); }
-  #overlay .content {
-    font-size: 0.875rem;
-    padding: 1rem;
-    margin-top: 5rem;
-    max-width: 400px;
-    color: var(--color-watch-table-row-text); }
-
-#heartpath {
-  transition: all ease 0.3s !important; }
-  #heartpath:hover {
-    fill: #ff0000 !important;
-    transition: all ease 0.3s !important; }
-
 body {
   color: var(--color-text);
-  background: var(--color-background-page);
-  font-family: Helvetica Neue, Helvetica, Lucida Grande, Arial, Ubuntu, Cantarell, Fira Sans, sans-serif; }
+  background: var(--color-background-page); }
 
 .visually-hidden {
   clip: rect(0 0 0 0);
@@ -423,6 +335,14 @@ a.github-link {
   a.github-link:hover {
     color: var(--color-icon-github-hover); }
 
+#toggle-light-mode {
+  width: 3rem; }
+  #toggle-light-mode .icon-dark {
+    display: none; }
+
+#toggle-search {
+  width: 2rem; }
+
 #search-q {
   opacity: 0;
   -webkit-transition: all .9s ease;
@@ -714,11 +634,7 @@ footer {
   background: var(--color-background);
   padding: 10px; }
   .sticky-tab#left-sticky {
-    left: 0;
-    position: fixed;
-    border-top-right-radius: 5px;
-    border-bottom-right-radius: 5px;
-    box-shadow: 1px 1px 4px var(--color-shadow-jump); }
+    left: 0px; }
   .sticky-tab#right-sticky {
     right: 0px; }
   .sticky-tab#hosted-sticky {
@@ -1027,7 +943,6 @@ ul {
 
 #selector-wrapper {
   height: 100%;
-  max-height: 70vh;
   overflow-y: scroll;
   position: relative; }
   #selector-wrapper > img {

changedetectionio/store.py

@@ -1,5 +1,3 @@
-from distutils.util import strtobool
-
 from flask import (
     flash
 )
@@ -18,9 +16,6 @@ import threading
 import time
 import uuid as uuid_builder
 
-# Because the server will run as a daemon and wont know the URL for notification links when firing off a notification
-BASE_URL_NOT_SET_TEXT = '("Base URL" not set - see settings - notifications)'
-
 dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
 
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
@@ -42,7 +37,6 @@ class ChangeDetectionStore:
         self.__data = App.model()
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
-        print(">>> Datastore path is ", self.json_store_path)
         self.needs_write = False
         self.start_time = time.time()
         self.stop_thread = False
@@ -96,14 +90,6 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://changedetection.io/CHANGELOG.txt',
                                tag='changedetection.io',
                                extras={'fetch_backend': 'html_requests'})
-
-            updates_available = self.get_updates_available()
-            self.__data['settings']['application']['schema_version'] = updates_available.pop()
-
-        else:
-            # Bump the update version by running updates
-            self.run_updates()
-
         self.__data['version_tag'] = version_tag
 
         # Just to test that proxies.json if it exists, doesnt throw a parsing error on startup
@@ -133,6 +119,9 @@ class ChangeDetectionStore:
             secret = secrets.token_hex(16)
             self.__data['settings']['application']['api_access_token'] = secret
 
+        # Bump the update version by running updates
+        self.run_updates()
+
         self.needs_write = True
 
         # Finally start the thread that will manage periodic data saves to JSON
@@ -184,21 +173,12 @@ class ChangeDetectionStore:
 
     @property
     def data(self):
-        # Re #152, Return env base_url if not overriden
-        # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services
-        #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.
+        # Re #152, Return env base_url if not overriden, @todo also prefer the proxy pass url
+        env_base_url = os.getenv('BASE_URL','')
+        if not self.__data['settings']['application']['base_url']:
+          self.__data['settings']['application']['base_url'] = env_base_url.strip('" ')
 
-        active_base_url = BASE_URL_NOT_SET_TEXT
-        if self.__data['settings']['application'].get('base_url'):
-            active_base_url = self.__data['settings']['application'].get('base_url')
-        elif os.getenv('BASE_URL'):
-            active_base_url = os.getenv('BASE_URL')
-
-        # I looked at various ways todo the following, but in the end just copying the dict seemed simplest/most reliable
-        # even given the memory tradeoff - if you know a better way.. maybe return d|self.__data.. or something
-        d = self.__data
-        d['settings']['application']['active_base_url'] = active_base_url.strip('" ')
-        return d
+        return self.__data
 
     # Delete a single watch by UUID
     def delete(self, uuid):
@@ -244,16 +224,12 @@ class ChangeDetectionStore:
         import pathlib
 
         self.__data['watching'][uuid].update({
-                'browser_steps_last_error_step' : None,
-                'check_count': 0,
-                'fetch_time' : 0.0,
-                'has_ldjson_price_data': None,
                 'last_checked': 0,
+                'has_ldjson_price_data': None,
                 'last_error': False,
                 'last_notification_error': False,
                 'last_viewed': 0,
                 'previous_md5': False,
-                'previous_md5_before_filters': False,
                 'track_ldjson_price_data': None,
             })
 
@@ -349,9 +325,6 @@ class ChangeDetectionStore:
             if k in apply_extras:
                 del apply_extras[k]
 
-        if not apply_extras.get('date_created'):
-            apply_extras['date_created'] = int(time.time())
-
         new_watch.update(apply_extras)
         new_watch.ensure_data_dir_exists()
         self.__data['watching'][new_uuid] = new_watch
@@ -360,8 +333,6 @@ class ChangeDetectionStore:
         if write_to_disk_now:
             self.sync_to_json()
 
-        print("added ", url)
-
         return new_uuid
 
     def visualselector_data_is_ready(self, watch_uuid):
@@ -497,8 +468,6 @@ class ChangeDetectionStore:
                     k = "ui-" + str(i) + proxy.get('proxy_name')
                     proxy_list[k] = {'label': proxy.get('proxy_name'), 'url': proxy.get('proxy_url')}
 
-        if proxy_list and strtobool(os.getenv('ENABLE_NO_PROXY_OPTION', 'True')):
-            proxy_list["no-proxy"] = {'label': "No proxy", 'url': ''}
 
         return proxy_list if len(proxy_list) else None
 
@@ -516,9 +485,6 @@ class ChangeDetectionStore:
         # If it's a valid one
         watch = self.data['watching'].get(uuid)
 
-        if strtobool(os.getenv('ENABLE_NO_PROXY_OPTION', 'True')) and watch.get('proxy') == "no-proxy":
-            return None
-
         if watch.get('proxy') and watch.get('proxy') in list(self.proxy_list.keys()):
             return watch.get('proxy')
 
@@ -600,7 +566,7 @@ class ChangeDetectionStore:
     def add_tag(self, name):
         # If name exists, return that
         n = name.strip().lower()
-        print (f">>> Adding new tag - '{n}'")
+        print (f">>> Adding new tag - '{n}")
         if not n:
             return False
 
@@ -636,8 +602,14 @@ class ChangeDetectionStore:
     def tag_exists_by_name(self, tag_name):
         return any(v.get('title', '').lower() == tag_name.lower() for k, v in self.__data['settings']['application']['tags'].items())
 
-    def get_updates_available(self):
+    # Run all updates
+    # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
+    #             So therefor - each `update_n` should be very careful about checking if it needs to actually run
+    #             Probably we should bump the current update schema version with each tag release version?
+    def run_updates(self):
         import inspect
+        import shutil
+
         updates_available = []
         for i, o in inspect.getmembers(self, predicate=inspect.ismethod):
             m = re.search(r'update_(\d+)$', i)
@@ -645,15 +617,6 @@ class ChangeDetectionStore:
                 updates_available.append(int(m.group(1)))
         updates_available.sort()
 
-        return updates_available
-
-    # Run all updates
-    # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
-    #             So therefor - each `update_n` should be very careful about checking if it needs to actually run
-    #             Probably we should bump the current update schema version with each tag release version?
-    def run_updates(self):
-        import shutil
-        updates_available = self.get_updates_available()
         for update_n in updates_available:
             if update_n > self.__data['settings']['application']['schema_version']:
                 print ("Applying update_{}".format((update_n)))
@@ -815,6 +778,15 @@ class ChangeDetectionStore:
                 continue
         return
 
+    # We don't know when the date_created was in the past until now, so just add an index number for now.
+    def update_11(self):
+        i = 0
+        for uuid, watch in self.data['watching'].items():
+            if not watch.get('date_created'):
+                watch['date_created'] = i
+            i+=1
+        return
+
     # Create tag objects and their references from existing tag text
     def update_12(self):
         i = 0
@@ -828,11 +800,3 @@ class ChangeDetectionStore:
 
                 self.data['watching'][uuid]['tags'] = tag_uuids
 
-    # #1775 - Update 11 did not update the records correctly when adding 'date_created' values for sorting
-    def update_13(self):
-        i = 0
-        for uuid, watch in self.data['watching'].items():
-            if not watch.get('date_created'):
-                self.data['watching'][uuid]['date_created'] = i
-            i+=1
-        return

changedetectionio/templates/_common_fields.jinja

@@ -13,9 +13,9 @@
                             <div class="pure-form-message-inline">
                               <ul>
                                 <li>Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) </code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
+                                <li><code>discord://</code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
+                                <li><code>tgram://</code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
                                 <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
                                   <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                               </ul>
@@ -35,14 +35,18 @@
                             </div>
                             <div class="pure-control-group">
                                 {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }}
-                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
-                                </span>
-
+                                <span class="pure-form-message-inline">Body for all notifications</span>
+                            </div>
+                            <div class="pure-control-group">
+                            <!-- unsure -->
+                                {{ render_field(form.notification_format , class="notification-format") }}
+                                <span class="pure-form-message-inline">Format for all notifications</span>
                             </div>
                             <div class="pure-controls">
-                                <div id="notification-token-toggle" class="pure-button button-tag button-xsmall">Show token/placeholders</div>
-                            </div>
-                            <div class="pure-controls" style="display: none;" id="notification-tokens-info">
+                            <p class="pure-form-message-inline">
+                                You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL.
+                            </p>
+
                                 <table class="pure-table" id="token-table">
                                     <thead>
                                     <tr>
@@ -95,13 +99,9 @@
                                         <td><code>{{ '{{diff_full}}' }}</code></td>
                                         <td>The diff output - full difference output</td>
                                     </tr>
-                                    <tr>
-                                        <td><code>{{ '{{diff_patch}}' }}</code></td>
-                                        <td>The diff output - patch in unified format</td>
-                                    </tr>
                                     <tr>
                                         <td><code>{{ '{{current_snapshot}}' }}</code></td>
-                                        <td>The current snapshot text contents value, useful when combined with JSON or CSS filters
+                                        <td>The current snapshot value, useful when combined with JSON or CSS filters
                                         </td>
                                     </tr>
                                     <tr>
@@ -111,15 +111,12 @@
                                     </tbody>
                                 </table>
                                 <div class="pure-form-message-inline">
-                                    <p>
-									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
-                                    For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
-                                    </p>
+                                    <br>
+                                    URLs generated by changedetection.io (such as <code>{{ '{{diff_url}}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br>
+                                    Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
+									<br>
+									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
                                 </div>
                             </div>
-                            <div class="pure-control-group">
-                                {{ render_field(form.notification_format , class="notification-format") }}
-                                <span class="pure-form-message-inline">Format for all notifications</span>
-                            </div>
                         </div>
 {% endmacro %}

changedetectionio/templates/_helpers.jinja

@@ -1,6 +1,7 @@
 {% macro render_field(field) %}
-  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
   <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
+
   {% if field.errors %}
     <ul class=errors>
     {% for error in field.errors %}
@@ -24,6 +25,18 @@
   </div>
 {% endmacro %}
 
+{% macro render_field(field) %}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
+  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </div>
+{% endmacro %}
 
 {% macro render_simple_field(field) %}
   <span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>

changedetectionio/templates/base.html

@@ -8,10 +8,10 @@
     <title>Change Detection{{extra_title}}</title>
     <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag %}- {{active_tag}}{% endif %}" href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}" >
     <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}" >
-    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}?v={{ get_css_version() }}" >
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}" >
     {% if extra_stylesheets %}
       {% for m in extra_stylesheets %}
-        <link rel="stylesheet" href="{{ m }}?ver={{ get_css_version() }}" >
+        <link rel="stylesheet" href="{{ m }}?ver=1000" >
       {% endfor %}
     {% endif %}
 
@@ -37,7 +37,7 @@
     <div class="header">
       <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
         {% if has_password and not current_user.is_authenticated %}
-          <a class="pure-menu-heading" href="https://changedetection.io" rel="noopener">
+          <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener">
             <strong>Change</strong>Detection.io</a>
         {% else %}
           <a class="pure-menu-heading" href="{{url_for('index')}}">
@@ -49,7 +49,7 @@
         {% else %}
           {% if new_version_available and not(has_password and not current_user.is_authenticated) %}
             <span id="new-version-text" class="pure-menu-heading">
-              <a href="https://changedetection.io">A new version is available</a>
+              <a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a>
             </span>
           {% endif %}
         {% endif %}
@@ -77,7 +77,7 @@
             {% endif %}
           {% else %}
             <li class="pure-menu-item">
-              <a class="pure-menu-link" href="https://changedetection.io">Website Change Detection and Notification.</a>
+              <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
             </li>
           {% endif %}
           {% if current_user.is_authenticated %}
@@ -85,7 +85,6 @@
               <a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a>
             </li>
           {% endif %}
-          {% if current_user.is_authenticated or not has_password %}
           <li class="pure-menu-item pure-form" id="search-menu-item">
             <!-- We use GET here so it offers people a chance to set bookmarks etc -->
             <form name="searchForm" action="" method="GET">
@@ -96,7 +95,6 @@
               </button>
             </form>
           </li>
-          {% endif %}
           <li class="pure-menu-item">
             <button class="toggle-button" id ="toggle-light-mode" type="button" title="Toggle Light/Dark Mode">
               <span class="visually-hidden">Toggle light/dark mode</span>
@@ -108,20 +106,6 @@
               </span>
             </button>
           </li>
-          <li class="pure-menu-item" id="heart-us">
-                <svg
-                   fill="#ff0000"
-                   class="bi bi-heart"
-                   preserveAspectRatio="xMidYMid meet"
-                   viewBox="0 0 16.9 16.1"
-                   id="svg-heart"
-                   xmlns="http://www.w3.org/2000/svg"
-                   xmlns:svg="http://www.w3.org/2000/svg">
-                  <path id="heartpath" d="M 5.338316,0.50302766 C 0.71136983,0.50647126 -3.9576371,7.2707777 8.5004254,15.503028 23.833425,5.3700277 13.220206,-2.5384409 8.6762066,1.6475589 c -0.060791,0.054322 -0.11943,0.1110064 -0.1757812,0.1699219 -0.057,-0.059 -0.1157813,-0.116875 -0.1757812,-0.171875 C 7.4724566,0.86129334 6.4060729,0.50223298 5.338316,0.50302766 Z"
-                     style="fill:var(--color-background);fill-opacity:1;stroke:#ff0000;stroke-opacity:1" />
-                </svg>
-
-          </li>
           <li class="pure-menu-item">
             <a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
               {% include "svgs/github.svg" %}
@@ -137,52 +121,14 @@
     {% endif %}
     {% if left_sticky %}
       <div class="sticky-tab" id="left-sticky">
-        <a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a><br>
-          Visualise <strong>triggers</strong> and <strong>ignored text</strong>
+        <a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a>
       </div>
     {% endif %}
     {% if right_sticky %}
       <div class="sticky-tab" id="right-sticky">{{ right_sticky }}</div>
     {% endif %}
     <section class="content">
-        <div id="overlay">
-            <div class="content">
-                <strong>changedetection.io needs your support!</strong><br>
-                <p>
-                    You can help us by supporting changedetection.io on these platforms;
-                </p>
-                <p>
-                <ul>
-                    <li>
-                        <a href="https://alternativeto.net/software/changedetection-io/about/">Rate us at
-                        AlternativeTo.net</a>
-                    </li>
-                <li>
-                    <a href="https://github.com/dgtlmoon/changedetection.io">Star us on GitHub</a>
-                </li>
-                <li>
-                    <a href="https://twitter.com/change_det_io">Follow us at Twitter/X</a>
-                </li>
-                <li>
-                    <a href="https://www.linkedin.com/company/changedetection-io">Check us out on LinkedIn</a>
-                </li>
-                <li>
-                    And tell your friends and colleagues :)
-                </li>
-                </ul>
-                </p>
-                <p>
-                    The more popular changedetection.io is, the more time we can dedicate to adding amazing features!
-                </p>
-                <p>
-                    Many thanks :)<br>
-                </p>
-                <p>
-                    <i>changedetection.io team</i>
-                </p>
-            </div>
-        </div>
-        <header>
+      <header>
         {% block header %}{% endblock %}
       </header>
 

changedetectionio/templates/diff.html

@@ -6,38 +6,14 @@
     {% if last_error_screenshot %}
     const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
     {% endif %}
-
-    const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
-
 </script>
 <script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
 <div id="settings">
+    <h1>Differences</h1>
     <form class="pure-form " action="" method="GET">
         <fieldset>
-            {% if versions|length >= 1 %}
-                <strong>Compare</strong>
-                <del class="change"><span>from</span></del>
-                <select id="diff-version" name="from_version" class="needs-localtime">
-                    {% for version in versions|reverse %}
-                        <option value="{{ version }}" {% if version== from_version %} selected="" {% endif %}>
-                            {{ version }}
-                        </option>
-                    {% endfor %}
-                </select>
-                <ins class="change"><span>to</span></ins>
-                <select id="current-version" name="to_version" class="needs-localtime">
-                    {% for version in versions|reverse %}
-                        <option value="{{ version }}" {% if version== to_version %} selected="" {% endif %}>
-                            {{ version }}
-                        </option>
-                    {% endfor %}
-                </select>
-                <button type="submit" class="pure-button pure-button-primary">Go</button>
-            {% endif %}
-        </fieldset>
-        <fieldset>
-            <strong>Style</strong>
+
             <label for="diffWords" class="pure-checkbox">
                 <input type="radio" name="diff_type" id="diffWords" value="diffWords"> Words</label>
             <label for="diffLines" class="pure-checkbox">
@@ -47,20 +23,32 @@
                 <input type="radio" name="diff_type" id="diffChars" value="diffChars"> Chars</label>
             <!-- @todo - when mimetype is JSON, select this by default? -->
             <label for="diffJson" class="pure-checkbox">
-                <input type="radio" name="diff_type" id="diffJson" value="diffJson"> JSON</label>
+                <input type="radio" name="diff_type" id="diffJson" value="diffJson" > JSON</label>
 
-            <span>
-        <!-- https://github.com/kpdecker/jsdiff/issues/389 ? -->
-        <label for="ignoreWhitespace" class="pure-checkbox" id="label-diff-ignorewhitespace">
-            <input type="checkbox" id="ignoreWhitespace" name="ignoreWhitespace"> Ignore Whitespace</label>
-    </span>
+            {% if versions|length >= 1 %}
+            <label for="diff-version">Compare newest (<span id="current-v-date"></span>) with</label>
+            <select id="diff-version" name="previous_version">
+                {% for version in versions|reverse %}
+                <option value="{{version}}" {% if version== current_previous_version %} selected="" {% endif %}>
+                    {{version}}
+                </option>
+                {% endfor %}
+            </select>
+            <button type="submit" class="pure-button pure-button-primary">Go</button>
+            {% endif %}
         </fieldset>
     </form>
-
+    <del>Removed text</del>
+    <ins>Inserted Text</ins>
+    <span>
+        <!-- https://github.com/kpdecker/jsdiff/issues/389 ? -->
+        <label for="ignoreWhitespace" class="pure-checkbox" id="label-diff-ignorewhitespace">
+            <input type="checkbox" id="ignoreWhitespace" name="ignoreWhitespace" > Ignore Whitespace</label>
+    </span>
 </div>
 
 <div id="diff-jump">
-    <a id="jump-next-diff" title="Jump to next difference">Jump</a>
+    <a onclick="next_diff();">Jump</a>
 </div>
 
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
@@ -88,6 +76,8 @@
     </div>
 
      <div class="tab-pane-inner" id="text">
+         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.</div>
+
          {% if password_enabled_and_share_is_off %}
            <div class="tip">Pro-tip: You can enable <strong>"share access when password is enabled"</strong> from settings</div>
          {% endif %}
@@ -98,10 +88,10 @@
              <tbody>
              <tr>
                  <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
-                 <td id="a" style="display: none;">{{from_version_file_contents}}</td>
-                 <td id="b" style="display: none;">{{to_version_file_contents}}</td>
+                 <td id="a" style="display: none;">{{previous}}</td>
+                 <td id="b" style="display: none;">{{newest}}</td>
                  <td id="diff-col">
-                     <span id="result" class="highlightable-filter"></span>
+                     <span id="result"></span>
                  </td>
              </tr>
              </tbody>

changedetectionio/templates/edit.html

@@ -4,21 +4,18 @@
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script>
-    const browser_steps_available_screenshots=JSON.parse('{{ watch.get_browsersteps_available_screenshots|tojson }}');
-    const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
-    const browser_steps_fetch_screenshot_image_url="{{url_for('browser_steps.browser_steps_fetch_screenshot_image', uuid=uuid)}}";
-    const browser_steps_last_error_step={{ watch.browser_steps_last_error_step|tojson }};
-    const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
-    const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
+    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
+    const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
+
 {% if emailprefix %}
     const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
 {% endif %}
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
-    const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
-    const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
-    const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
-    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
-    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
+
+    const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
+    const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
+    const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
 
 </script>
 
@@ -30,8 +27,6 @@
 <script src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
 {% endif %}
 
-<script src="{{url_for('static_content', group='js', filename='recheck-proxy.js')}}" defer></script>
-
 <div class="edit-form monospaced-textarea">
 
     <div class="tabs collapsable">
@@ -51,7 +46,6 @@
             <li class="tab"><a href="#restock">Restock Detection</a></li>
             {% endif %}
             <li class="tab"><a href="#notifications">Notifications</a></li>
-            <li class="tab"><a href="#stats">Stats</a></li>
         </ul>
     </div>
 
@@ -112,13 +106,12 @@
                         <span class="pure-form-message-inline">
                             <p>Use the <strong>Basic</strong> method (default) where your watched site doesn't need Javascript to render.</p>
                             <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
-                            Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
+                            Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using BrightData Proxies, find out more here.</a>
                         </span>
                     </div>
                 {% if form.proxy %}
                     <div class="pure-control-group inline-radio">
-                          <div>{{ form.proxy.label }} <a href="" id="check-all-proxies" class="pure-button button-secondary button-xsmall" >Check/Scan all</a></div>
-                          <div>{{ form.proxy(class="fetch-backend-proxy") }}</div>
+                        {{ render_field(form.proxy, class="fetch-backend-proxy") }}
                         <span class="pure-form-message-inline">
                         Choose a proxy for this watch
                         </span>
@@ -381,16 +374,15 @@ Unavailable") }}
                         {{ render_field(form.extract_text, rows=5, placeholder="\d+ online") }}
                         <span class="pure-form-message-inline">
                     <ul>
-                        <li>Extracts text in the final output (line by line) after other filters using regular expressions or string match;
+                        <li>Extracts text in the final output (line by line) after other filters using regular expressions;
                             <ul>
                                 <li>Regular expression &dash; example <code>/reports.+?2022/i</code></li>
-                                <li>Don't forget to consider the white-space at the start of a line <code>/.+?reports.+?2022/i</code></li>
                                 <li>Use <code>//(?aiLmsux))</code> type flags (more <a href="https://docs.python.org/3/library/re.html#index-15">information here</a>)<br></li>
                                 <li>Keyword example &dash; example <code>Out of stock</code></li>
                                 <li>Use groups to extract just that text &dash; example <code>/reports.+?(\d+)/i</code> returns a list of years only</li>
                             </ul>
                         </li>
-                        <li>One line per regular-expression/string match</li>
+                        <li>One line per regular-expression/ string match</li>
                     </ul>
                         </span>
                     </div>
@@ -444,35 +436,7 @@ Unavailable") }}
                 </fieldset>
             </div>
             {% endif %}
-            <div class="tab-pane-inner" id="stats">
-                <div class="pure-control-group">
-                    <style>
-                    #stats-table tr > td:first-child {
-                        font-weight: bold;
-                    }
-                    </style>
-                    <table class="pure-table" id="stats-table">
-                        <tbody>
-                        <tr>
-                            <td>Check count</td>
-                            <td>{{ "{:,}".format( watch.check_count) }}</td>
-                        </tr>
-                        <tr>
-                            <td>Consecutive filter failures</td>
-                            <td>{{ "{:,}".format( watch.consecutive_filter_failures) }}</td>
-                        </tr>
-                        <tr>
-                            <td>History length</td>
-                            <td>{{ "{:,}".format(watch.history|length) }}</td>
-                        </tr>
-                        <tr>
-                            <td>Last fetch time</td>
-                            <td>{{ watch.fetch_time }}s</td>
-                        </tr>
-                        </tbody>
-                    </table>
-                </div>
-            </div>
+
             <div id="actions">
                 <div class="pure-control-group">
                     {{ render_button(form.save_button) }}

changedetectionio/templates/import.html

@@ -8,12 +8,11 @@
         <ul>
             <li class="tab" id=""><a href="#url-list">URL List</a></li>
             <li class="tab"><a href="#distill-io">Distill.io</a></li>
-            <li class="tab"><a href="#xlsx">.XLSX &amp; Wachete</a></li>
         </ul>
     </div>
 
     <div class="box-wrap inner">
-        <form class="pure-form" action="{{url_for('import_page')}}" method="POST" enctype="multipart/form-data">
+        <form class="pure-form pure-form-aligned" action="{{url_for('import_page')}}" method="POST">
             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
             <div class="tab-pane-inner" id="url-list">
                     <legend>
@@ -80,42 +79,6 @@
 " rows="25">{{ original_distill_json }}</textarea>
 
             </div>
-            <div class="tab-pane-inner" id="xlsx">
-            <fieldset>
-                <div class="pure-control-group">
-                {{ render_field(form.xlsx_file, class="processor") }}
-                </div>
-                <div class="pure-control-group">
-                    {{ render_field(form.file_mapping, class="processor") }}
-                </div>
-            </fieldset>
-                <div class="pure-control-group">
-                <span class="pure-form-message-inline">
-                    Table of custom column and data types mapping for the <strong>Custom mapping</strong> File mapping type.
-                </span>
-                    <table style="border: 1px solid #aaa; padding: 0.5rem; border-radius: 4px;">
-                        <tr>
-                            <td><strong>Column #</strong></td>
-                            {% for n in range(4) %}
-                                <td><input type="number" name="custom_xlsx[col_{{n}}]" style="width: 4rem;" min="1"></td>
-                            {%  endfor %}
-                        </tr>
-                        <tr>
-                            <td><strong>Type</strong></td>
-                            {% for n in range(4) %}
-                                <td><select name="custom_xlsx[col_type_{{n}}]">
-                                    <option value="" style="color: #aaa"> -- none --</option>
-                                    <option value="url">URL</option>
-                                    <option value="title">Title</option>
-                                    <option value="include_filter">CSS/xPath filter</option>
-                                    <option value="tag">Group / Tag name(s)</option>
-                                    <option value="interval_minutes">Recheck time (minutes)</option>
-                                </select></td>
-                            {%  endfor %}
-                        </tr>
-                    </table>
-                </div>
-            </div>
             <button type="submit" class="pure-button pure-input-1-2 pure-button-primary">Import</button>
         </form>
 

changedetectionio/templates/preview.html

@@ -6,7 +6,6 @@
     {% if last_error_screenshot %}
     const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
     {% endif %}
-    const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
 </script>
 <script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>
 
@@ -21,7 +20,7 @@
         {% endif %}
     </ul>
 </div>
-<form><input type="hidden" name="csrf_token" value="{{ csrf_token() }}"></form>
+
 <div id="diff-ui">
     <div class="tab-pane-inner" id="error-text">
         <div class="snapshot-age error">{{watch.error_text_ctime|format_seconds_ago}} seconds ago</div>
@@ -37,12 +36,11 @@
 
     <div class="tab-pane-inner" id="text">
         <div class="snapshot-age">{{watch.snapshot_text_ctime|format_timestamp_timeago}}</div>
-        <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span> <span class="tip"><strong>Pro-tip</strong>: Highlight text to add to ignore filters</span>
-
+        <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
         <table>
             <tbody>
             <tr>
-                <td id="diff-col" class="highlightable-filter">
+                <td id="diff-col">
                     {% for row in content %}
                     <div class="{{row.classes}}">{{row.line}}</div>
                     {% endfor %}

changedetectionio/templates/settings.html

@@ -62,6 +62,14 @@
                         <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page)
                         </span>
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
+                        class="m-d") }}
+                        <span class="pure-form-message-inline">
+                            Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notifications and RSS links.<br>Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"),
+                            <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
+                        </span>
+                    </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.pager_size) }}
                         <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
@@ -92,13 +100,6 @@
                         {{ render_common_settings_form(form.application.form, emailprefix, settings_application) }}
                     </div>
                 </fieldset>
-                <div class="pure-control-group" id="notification-base-url">
-                    {{ render_field(form.application.form.base_url, class="m-d") }}
-                    <span class="pure-form-message-inline">
-                        Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notification links.<br>
-                        Default value is the system environment variable '<code>BASE_URL</code>' - <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
-                    </span>
-                </div>
             </div>
 
             <div class="tab-pane-inner" id="fetching">
@@ -109,7 +110,7 @@
                         <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                     </span>
                     <br>
-                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using Bright Data and Oxylabs Proxies, find out more here.</a>
+                    Tip: <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support">Connect using BrightData Proxies, find out more here.</a>
                 </div>
                 <fieldset class="pure-group" id="webdriver-override-options">
                     <div class="pure-form-message-inline">
@@ -180,57 +181,20 @@ nav
                 </div>
             </div>
             <div class="tab-pane-inner" id="proxies">
-                <div id="recommended-proxy">
-                    <div>
-                        <img style="height: 2em;" src="{{url_for('static_content', group='images', filename='brightdata.svg')}}" alt="BrightData Proxy Provider">
-                        <p>BrightData offer world-class proxy services, "Data Center" proxies are a very affordable way to proxy your requests, whilst <strong><a href="https://brightdata.grsm.io/n0r16zf7eivq">WebUnlocker</a></strong> can help solve most CAPTCHAs.</p>
-                        <p>
-                            BrightData offer many <a href="https://brightdata.com/proxy-types" target="new">many different types of proxies</a>, it is worth reading about what is best for your use-case.
-                        </p>
 
-                        <p>
-                            When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the access Proxy URL into the "Extra Proxies" boxes below.<br>
-                        </p>
-                        <p>
-                            The Proxy URL with BrightData should start with <code>http://brd-customer...</code>
-                        </p>
-                        <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p>
-                    </div>
-                    <div>
-                        <img style="height: 2em;"
-                             src="{{url_for('static_content', group='images', filename='oxylabs.svg')}}"
-                             alt="Oxylabs Proxy Provider">
-                        <p>
-                            Collect public data at scale with industry-leading web scraping solutions and the world’s
-                            largest ethical proxy network.
-                        </p>
-                        <p>
-                            Oxylabs also provide a <a href="https://oxylabs.io/products/web-unblocker"><strong>WebUnlocker</strong></a>
-                            proxy that bypasses sophisticated anti-bot systems, so you don’t have to.<br>
-                        </p>
-                        <p>
-                            Serve over <a href="https://oxylabs.io/location-proxy">195 countries</a>, providing <a
-                                href="https://oxylabs.io/products/residential-proxy-pool">Residential</a>, <a
-                                href="https://oxylabs.io/products/mobile-proxies">Mobile</a> and <a
-                                href="https://oxylabs.io/products/rotating-isp-proxies">ISP proxies</a> and much more.
-                        </p>
-                        <p>
-                            Use the promo code <strong>boost35</strong> with this link <a href="https://oxylabs.go2cloud.org/SH2d">https://oxylabs.go2cloud.org/SH2d</a> for 35% off Residential, Mobile proxies, Web Unblocker, and Scraper APIs. Built-in proxies enable you to access data from all around the world and help overcome anti-bot solutions.
-
-                        </p>
-
-                        
-                    </div>
-                </div>
+                <p><strong>Tip</strong>: You can connect to websites using <a href="https://brightdata.grsm.io/n0r16zf7eivq">BrightData</a> proxies, their service <strong>WebUnlocker</strong> will solve most CAPTCHAs, whilst their <strong>Residential Proxies</strong> may help to avoid CAPTCHA altogether. </p>
+                <p>It may be easier to try <strong>WebUnlocker</strong> first, WebUnlocker also supports country selection.</p>
                 <p>
-                    Your proxy provider may need to whitelist our IP of <code>204.15.192.195</code>
+                    When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the example URL below<br>
+                    The Proxy URL with BrightData should start with <code>http://brd-customer...</code>
                 </p>
-               <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.
+
+                <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p>
+
 
                 <div class="pure-control-group">
-                {{ render_field(form.requests.form.extra_proxies) }}
-                <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span><br>
-                <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
+                    {{ render_field(form.requests.form.extra_proxies) }}
+                    <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span>
                 </div>
             </div>
             <div id="actions">

changedetectionio/templates/svgs/github.svg

@@ -1,6 +1,3 @@
-<svg class="octicon octicon-mark-github v-align-middle"  viewbox="0 0 16 16" version="1.1" aria-hidden="true">
-    <path
-     fill-rule="evenodd"
-     d="M 8,0 C 3.58,0 0,3.58 0,8 c 0,3.54 2.29,6.53 5.47,7.59 0.4,0.07 0.55,-0.17 0.55,-0.38 0,-0.19 -0.01,-0.82 -0.01,-1.49 C 4,14.09 3.48,13.23 3.32,12.78 3.23,12.55 2.84,11.84 2.5,11.65 2.22,11.5 1.82,11.13 2.49,11.12 3.12,11.11 3.57,11.7 3.72,11.94 4.44,13.15 5.59,12.81 6.05,12.6 6.12,12.08 6.33,11.73 6.56,11.53 4.78,11.33 2.92,10.64 2.92,7.58 2.92,6.71 3.23,5.99 3.74,5.43 3.66,5.23 3.38,4.41 3.82,3.31 c 0,0 0.67,-0.21 2.2,0.82 0.64,-0.18 1.32,-0.27 2,-0.27 0.68,0 1.36,0.09 2,0.27 1.53,-1.04 2.2,-0.82 2.2,-0.82 0.44,1.1 0.16,1.92 0.08,2.12 0.51,0.56 0.82,1.27 0.82,2.15 0,3.07 -1.87,3.75 -3.65,3.95 0.29,0.25 0.54,0.73 0.54,1.48 0,1.07 -0.01,1.93 -0.01,2.2 0,0.21 0.15,0.46 0.55,0.38 A 8.013,8.013 0 0 0 16,8 C 16,3.58 12.42,0 8,0 Z"
-     id="path2" />
+<svg class="octicon octicon-mark-github v-align-middle" height="32" viewbox="0 0 16 16" version="1.1" width="32" aria-hidden="true">
+  <path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
 </svg>

changedetectionio/templates/watch-overview.html

@@ -119,9 +119,6 @@
                             <a href="{{ url_for('settings_page', uuid=watch.uuid) }}#proxies">Try adding external proxies/locations</a>
                         
                         {% endif %}
-                        {% if 'empty result or contain only an image' in watch.last_error %}
-                            <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Detecting-changes-in-images">more help here</a>.
-                        {% endif %}
                     </div>
                     {% endif %}
                     {% if watch.last_notification_error is defined and watch.last_notification_error != False %}
@@ -154,8 +151,8 @@
                     {% endfor %}
 
                 </td>
-                <td class="last-checked" data-timestamp="{{ watch.last_checked }}">{{watch|format_last_checked_time|safe}}</td>
-                <td class="last-changed" data-timestamp="{{ watch.last_changed }}">{% if watch.history_n >=2 and watch.last_changed >0 %}
+                <td class="last-checked">{{watch|format_last_checked_time|safe}}</td>
+                <td class="last-changed">{% if watch.history_n >=2 and watch.last_changed >0 %}
                     {{watch.last_changed|format_timestamp_timeago}}
                     {% else %}
                     Not yet
@@ -178,18 +175,13 @@
             </tbody>
         </table>
         <ul id="post-list-buttons">
-            {% if errored_count %}
-            <li>
-                <a href="{{url_for('index', with_errors=1, tag=request.args.get('tag')) }}" class="pure-button button-tag button-error ">With errors ({{ errored_count }})</a>
-            </li>
-            {% endif %}
             {% if has_unviewed %}
             <li>
-                <a href="{{url_for('mark_all_viewed',with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Mark all viewed</a>
+                <a href="{{url_for('mark_all_viewed', tag=request.args.get('tag')) }}" class="pure-button button-tag ">Mark all viewed</a>
             </li>
             {% endif %}
             <li>
-               <a href="{{ url_for('form_watch_checknow', tag=active_tag, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('form_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
                 all {% if active_tag%} in "{{tags[active_tag].title}}"{%endif%}</a>
             </li>
             <li>

changedetectionio/tests/fetchers/test_content.py

@@ -28,6 +28,8 @@ def test_fetch_webdriver_content(client, live_server):
     )
 
     assert b"1 Imported" in res.data
+    time.sleep(3)
+
     wait_for_all_checks(client)
 
 

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -2,11 +2,12 @@
 
 import time
 from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks
-
+from ..util import live_server_setup
 
 def test_preferred_proxy(client, live_server):
+    time.sleep(1)
     live_server_setup(live_server)
+    time.sleep(1)
     url = "http://chosen.changedetection.io"
 
     res = client.post(
@@ -19,7 +20,7 @@ def test_preferred_proxy(client, live_server):
 
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(2)
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
@@ -33,5 +34,5 @@ def test_preferred_proxy(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
+    time.sleep(2)
     # Now the request should appear in the second-squid logs

changedetectionio/tests/proxy_list/test_noproxy.py

@@ -1,77 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
-
-
-def test_noproxy_option(client, live_server):
-    live_server_setup(live_server)
-    # Run by run_proxy_tests.sh
-    # Call this URL then scan the containers that it never went through them
-    url = "http://noproxy.changedetection.io"
-
-    # Should only be available when a proxy is setup
-    res = client.get(
-        url_for("edit_page", uuid="first", unpause_on_save=1))
-    assert b'No proxy' not in res.data
-
-    # Setup a proxy
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-ignore_whitespace": "y",
-            "application-fetch_backend": "html_requests",
-            "requests-extra_proxies-0-proxy_name": "custom-one-proxy",
-            "requests-extra_proxies-0-proxy_url": "http://test:awesome@squid-one:3128",
-            "requests-extra_proxies-1-proxy_name": "custom-two-proxy",
-            "requests-extra_proxies-1-proxy_url": "http://test:awesome@squid-two:3128",
-            "requests-extra_proxies-2-proxy_name": "custom-proxy",
-            "requests-extra_proxies-2-proxy_url": "http://test:awesome@squid-custom:3128",
-        },
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    # Should be available as an option
-    res = client.get(
-        url_for("settings_page", unpause_on_save=1))
-    assert b'No proxy' in res.data
-
-
-    # This will add it paused
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
-        follow_redirects=True
-    )
-    assert b"Watch added in Paused state, saving will unpause" in res.data
-    uuid = extract_UUID_from_client(client)
-    res = client.get(
-        url_for("edit_page", uuid=uuid, unpause_on_save=1))
-    assert b'No proxy' in res.data
-
-    res = client.post(
-        url_for("edit_page", uuid=uuid, unpause_on_save=1),
-        data={
-                "include_filters": "",
-                "fetch_backend": "html_requests",
-                "headers": "",
-                "proxy": "no-proxy",
-                "tags": "",
-                "url": url,
-              },
-        follow_redirects=True
-    )
-    assert b"unpaused" in res.data
-    wait_for_all_checks(client)
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-    # Now the request should NOT appear in the second-squid logs (handled by the run_test_proxies.sh script)
-
-    # Prove that it actually checked
-
-    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] != 0
-

changedetectionio/tests/proxy_socks5/proxies.json-example

@@ -1,6 +0,0 @@
-{
-  "socks5proxy": {
-    "label": "socks5proxy",
-    "url": "socks5://proxy_user123:proxy_pass123@socks5proxy:1080"
-  }
-}

changedetectionio/tests/proxy_socks5/proxies.json-example-noauth

@@ -1,6 +0,0 @@
-{
-  "socks5proxy": {
-    "label": "socks5proxy",
-    "url": "socks5://socks5proxy-noauth:1080"
-  }
-}

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -1,63 +0,0 @@
-#!/usr/bin/python3
-import os
-import time
-from flask import url_for
-from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
-
-
-def test_socks5(client, live_server):
-    live_server_setup(live_server)
-
-    # Setup a proxy
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "requests-time_between_check-minutes": 180,
-            "application-ignore_whitespace": "y",
-            "application-fetch_backend": "html_requests",
-            # set in .github/workflows/test-only.yml
-            "requests-extra_proxies-0-proxy_url": "socks5://proxy_user123:proxy_pass123@socks5proxy:1080",
-            "requests-extra_proxies-0-proxy_name": "socks5proxy",
-        },
-        follow_redirects=True
-    )
-
-    assert b"Settings updated." in res.data
-
-    test_url = "https://changedetection.io/CHANGELOG.txt?socks-test-tag=" + os.getenv('SOCKSTEST', '')
-
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
-        follow_redirects=True
-    )
-    assert b"Watch added in Paused state, saving will unpause" in res.data
-
-    res = client.get(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
-    )
-    # check the proxy is offered as expected
-    assert b'ui-0socks5proxy' in res.data
-
-    res = client.post(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
-        data={
-            "include_filters": "",
-            "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
-            "headers": "",
-            "proxy": "ui-0socks5proxy",
-            "tags": "",
-            "url": test_url,
-        },
-        follow_redirects=True
-    )
-    assert b"unpaused" in res.data
-    wait_for_all_checks(client)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    # Should see the proper string
-    assert "+0200:".encode('utf-8') in res.data

changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py

@@ -1,52 +0,0 @@
-#!/usr/bin/python3
-import os
-import time
-from flask import url_for
-from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
-
-
-# should be proxies.json mounted from run_proxy_tests.sh already
-# -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json
-def test_socks5_from_proxiesjson_file(client, live_server):
-    live_server_setup(live_server)
-
-    test_url = "https://changedetection.io/CHANGELOG.txt?socks-test-tag=" + os.getenv('SOCKSTEST', '')
-
-    res = client.get(url_for("settings_page"))
-    assert b'name="requests-proxy" type="radio" value="socks5proxy"' in res.data
-
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
-        follow_redirects=True
-    )
-    assert b"Watch added in Paused state, saving will unpause" in res.data
-
-    res = client.get(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
-    )
-    # check the proxy is offered as expected
-    assert b'name="proxy" type="radio" value="socks5proxy"' in res.data
-
-    res = client.post(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
-        data={
-            "include_filters": "",
-            "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
-            "headers": "",
-            "proxy": "socks5proxy",
-            "tags": "",
-            "url": test_url,
-        },
-        follow_redirects=True
-    )
-    assert b"unpaused" in res.data
-    wait_for_all_checks(client)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    # Should see the proper string
-    assert "+0200:".encode('utf-8') in res.data

changedetectionio/tests/smtp/smtp-test-server.py

@@ -1,42 +0,0 @@
-#!/usr/bin/python3
-import smtpd
-import asyncore
-
-# Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
-
-last_received_message = b"Nothing"
-
-
-class CustomSMTPServer(smtpd.SMTPServer):
-
-    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
-        global last_received_message
-        last_received_message = data
-        print('Receiving message from:', peer)
-        print('Message addressed from:', mailfrom)
-        print('Message addressed to  :', rcpttos)
-        print('Message length        :', len(data))
-        print(data.decode('utf8'))
-        return
-
-
-# Just print out the last message received on plain TCP socket server
-class EchoServer(asyncore.dispatcher):
-
-    def __init__(self, host, port):
-        asyncore.dispatcher.__init__(self)
-        self.create_socket()
-        self.set_reuse_addr()
-        self.bind((host, port))
-        self.listen(5)
-
-    def handle_accepted(self, sock, addr):
-        global last_received_message
-        print('Incoming connection from %s' % repr(addr))
-        sock.send(last_received_message)
-        last_received_message = b''
-
-
-server = CustomSMTPServer(('0.0.0.0', 11025), None)  # SMTP mail goes here
-server2 = EchoServer('0.0.0.0', 11080)  # Echo back last message received
-asyncore.loop()

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -1,165 +0,0 @@
-import json
-import os
-import time
-import re
-from flask import url_for
-from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
-    wait_for_all_checks, \
-    set_longer_modified_response
-from changedetectionio.tests.util import extract_UUID_from_client
-import logging
-import base64
-
-# NOTE - RELIES ON mailserver as hostname running, see github build recipes
-smtp_test_server = 'mailserver'
-
-from changedetectionio.notification import (
-    default_notification_body,
-    default_notification_format,
-    default_notification_title,
-    valid_notification_formats,
-)
-
-def test_setup(live_server):
-    live_server_setup(live_server)
-
-def get_last_message_from_smtp_server():
-    import socket
-    global smtp_test_server
-    port = 11080  # socket server port number
-
-    client_socket = socket.socket()  # instantiate
-    client_socket.connect((smtp_test_server, port))  # connect to the server
-
-    data = client_socket.recv(50024).decode()  # receive response
-    client_socket.close()  # close the connection
-    return data
-
-
-# Requires running the test SMTP server
-
-def test_check_notification_email_formats_default_HTML(client, live_server):
-    # live_server_setup(live_server)
-    set_original_response()
-
-    global smtp_test_server
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": "fallback-body<br> " + default_notification_body,
-              "application-notification_format": 'HTML',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-    set_longer_modified_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
-
-    # The email should have two bodies, and the text/html part should be <br>
-    assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
-    assert 'Content-Type: text/html' in msg
-    assert '(added) So let\'s see what happens.<br>' in msg  # the html part
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-
-def test_check_notification_email_formats_default_Text_override_HTML(client, live_server):
-    # live_server_setup(live_server)
-
-    # HTML problems? see this
-    # https://github.com/caronc/apprise/issues/633
-
-    set_original_response()
-    global smtp_test_server
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": default_notification_body,
-              "application-notification_format": 'Text',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-    set_longer_modified_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
-    #    with open('/tmp/m.txt', 'w') as f:
-    #        f.write(msg)
-
-    # The email should not have two bodies, should be TEXT only
-
-    assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
-
-    set_original_response()
-    # Now override as HTML format
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-            "url": test_url,
-            "notification_format": 'HTML',
-            'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
-
-    # The email should have two bodies, and the text/html part should be <br>
-    assert 'Content-Type: text/plain' in msg
-    assert '(removed) So let\'s see what happens.\n' in msg  # The plaintext part with \n
-    assert 'Content-Type: text/html' in msg
-    assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_access_control.py

@@ -1,4 +1,4 @@
-from .util import live_server_setup, extract_UUID_from_client, wait_for_all_checks
+from . util import live_server_setup, extract_UUID_from_client
 from flask import url_for
 import time
 
@@ -19,16 +19,10 @@ def test_check_access_control(app, client, live_server):
         )
 
         assert b"1 Imported" in res.data
-        time.sleep(3)
-        # causes a 'Popped wrong request context.' error when client. is accessed?
-        #wait_for_all_checks(client)
-
-        res = c.get(url_for("form_watch_checknow"), follow_redirects=True)
+        time.sleep(2)
+        res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
         assert b'1 watches queued for rechecking.' in res.data
-        time.sleep(3)
-        # causes a 'Popped wrong request context.' error when client. is accessed?
-        #wait_for_all_checks(client)
-
+        time.sleep(2)
 
         # Enable password check and diff page access bypass
         res = c.post(
@@ -48,7 +42,7 @@ def test_check_access_control(app, client, live_server):
         assert b"Login" in res.data
 
         # The diff page should return something valid when logged out
-        res = c.get(url_for("diff_history_page", uuid="first"))
+        res = client.get(url_for("diff_history_page", uuid="first"))
         assert b'Random content' in res.data
 
         # Check wrong password does not let us in
@@ -89,8 +83,6 @@ def test_check_access_control(app, client, live_server):
         res = c.get(url_for("logout"),
             follow_redirects=True)
 
-        assert b"Login" in res.data
-
         res = c.get(url_for("settings_page"),
             follow_redirects=True)
 
@@ -168,5 +160,5 @@ def test_check_access_control(app, client, live_server):
         assert b"Login" in res.data
 
         # The diff page should return something valid when logged out
-        res = c.get(url_for("diff_history_page", uuid="first"))
+        res = client.get(url_for("diff_history_page", uuid="first"))
         assert b'Random content' not in res.data

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -2,8 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, extract_UUID_from_client, extract_api_key_from_UI, wait_for_all_checks
-
+from .util import live_server_setup, extract_UUID_from_client, extract_api_key_from_UI
 
 def set_response_with_ldjson():
     test_return_data = """<html>
@@ -28,7 +27,7 @@ def set_response_with_ldjson():
            "description":"You dont need it",
            "mpn":"111111",
            "sku":"22222",
-           "Offers":{
+           "offers":{
               "@type":"AggregateOffer",
               "lowPrice":8097000,
               "highPrice":8099900,
@@ -76,11 +75,12 @@ def set_response_without_ldjson():
         f.write(test_return_data)
     return None
 
-def test_setup(client, live_server):
-    live_server_setup(live_server)
-
 # actually only really used by the distll.io importer, but could be handy too
 def test_check_ldjson_price_autodetect(client, live_server):
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
 
     set_response_with_ldjson()
 
@@ -92,7 +92,7 @@ def test_check_ldjson_price_autodetect(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Should get a notice that it's available
     res = client.get(url_for("index"))
@@ -102,11 +102,11 @@ def test_check_ldjson_price_autodetect(client, live_server):
     uuid = extract_UUID_from_client(client)
 
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(2)
     # Offer should be gone
     res = client.get(url_for("index"))
     assert b'Embedded price data' not in res.data
@@ -138,97 +138,9 @@ def test_check_ldjson_price_autodetect(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
+    time.sleep(3)
     res = client.get(url_for("index"))
     assert b'ldjson-price-track-offer' not in res.data
     
     ##########################################################################################
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-
-
-def _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_data):
-
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-    for k,v in client.application.config.get('DATASTORE').data['watching'].items():
-        assert v.get('last_error') == False
-        assert v.get('has_ldjson_price_data') == has_ldjson_price_data
-
-
-    ##########################################################################################
-    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-
-
-def test_bad_ldjson_is_correctly_ignored(client, live_server):
-    #live_server_setup(live_server)
-    test_return_data = """
-            <html>
-            <head>
-                <script type="application/ld+json">
-                    {
-                        "@context": "http://schema.org",
-                        "@type": ["Product", "SubType"],
-                        "name": "My test product",
-                        "description": "",
-                        "offers": {
-                            "note" : "You can see the case-insensitive OffERS key, it should work",
-                            "@type": "Offer",
-                            "offeredBy": {
-                                "@type": "Organization",
-                                "name":"Person",
-                                "telephone":"+1 999 999 999"
-                            },
-                            "price": "1",
-                            "priceCurrency": "EUR",
-                            "url": "/some/url"
-                        }
-                    }
-                </script>
-            </head>
-            <body>
-            <div class="yes">Some extra stuff</div>
-            </body></html>
-     """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-    _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=True)
-    test_return_data = """
-            <html>
-            <head>
-                <script type="application/ld+json">
-                    {
-                        "@context": "http://schema.org",
-                        "@type": ["Product", "SubType"],
-                        "name": "My test product",
-                        "description": "",
-                        "BrokenOffers": {
-                            "@type": "Offer",
-                            "offeredBy": {
-                                "@type": "Organization",
-                                "name":"Person",
-                                "telephone":"+1 999 999 999"
-                            },
-                            "price": "1",
-                            "priceCurrency": "EUR",
-                            "url": "/some/url"
-                        }
-                    }
-                </script>
-            </head>
-            <body>
-            <div class="yes">Some extra stuff</div>
-            </body></html>
-     """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-    _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=False)
-

changedetectionio/tests/test_backend.py

@@ -89,7 +89,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
     res = client.get(url_for("diff_history_page", uuid="first"))
-    assert b'selected=""' in res.data, "Confirm diff history page loaded"
+    assert b'Compare newest' in res.data
 
     # Check the [preview] pulls the right one
     res = client.get(

changedetectionio/tests/test_backup.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, live_server_setup
 from flask import url_for
 from urllib.request import urlopen
 from zipfile import ZipFile
@@ -19,12 +19,12 @@ def test_backup(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("import_page"),
-        data={"urls": url_for('test_endpoint', _external=True)+"?somechar=őőőőőőőő"},
+        data={"urls": url_for('test_endpoint', _external=True)},
         follow_redirects=True
     )
 
     assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     res = client.get(
         url_for("get_backup"),

changedetectionio/tests/test_css_selector.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from . util import live_server_setup
 
 from ..html_tools import *
 
@@ -176,77 +176,3 @@ def test_check_multiple_filters(client, live_server):
     assert b"Blob A" in res.data # CSS was ok
     assert b"Blob B" in res.data # xPath was ok
     assert b"Blob C" not in res.data # Should not be included
-
-# The filter exists, but did not contain anything useful
-# Mainly used when the filter contains just an IMG, this can happen when someone selects an image in the visual-selector
-# Tests fetcher can throw a "ReplyWithContentButNoText" exception after applying filter and extracting text
-def test_filter_is_empty_help_suggestion(client, live_server):
-    #live_server_setup(live_server)
-
-    include_filters = "#blob-a"
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write("""<html><body>
-         <div id="blob-a">
-           <img src="something.jpg">
-         </div>
-         </body>
-         </html>
-        """)
-
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-    # Goto the edit page, add our ignore text
-    # Add our URL to the import page
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"include_filters": include_filters,
-              "url": test_url,
-              "tags": "",
-              "headers": "",
-              'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
-    wait_for_all_checks(client)
-
-
-    res = client.get(
-        url_for("index"),
-        follow_redirects=True
-    )
-
-    assert b'empty result or contain only an image' in res.data
-
-
-    ### Just an empty selector, no image
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write("""<html><body>
-         <div id="blob-a">
-           <!-- doo doo -->
-         </div>
-         </body>
-         </html>
-        """)
-
-    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    res = client.get(
-        url_for("index"),
-        follow_redirects=True
-    )
-
-    assert b'empty result or contain only an image' not in res.data
-    assert b'but contained no usable text' in res.data

changedetectionio/tests/test_extract_regex.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from .util import live_server_setup
 
 from ..html_tools import *
 
@@ -55,8 +55,6 @@ def set_multiline_response():
      </p>
      
      <div>aaand something lines</div>
-     <br>
-     <div>and this should be</div>
      </body>
      </html>
     """
@@ -68,10 +66,11 @@ def set_multiline_response():
 
 
 def test_setup(client, live_server):
+
     live_server_setup(live_server)
 
 def test_check_filter_multiline(client, live_server):
-    #live_server_setup(live_server)
+
     set_multiline_response()
 
     # Add our URL to the import page
@@ -83,15 +82,14 @@ def test_check_filter_multiline(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"include_filters": '',
-              # Test a regex and a plaintext
-              'extract_text': '/something.+?6 billion.+?lines/si\r\nand this should be',
+              'extract_text': '/something.+?6 billion.+?lines/si',
               "url": test_url,
               "tags": "",
               "headers": "",
@@ -101,19 +99,13 @@ def test_check_filter_multiline(client, live_server):
     )
 
     assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("index"))
-
-    # Issue 1828
-    assert b'not at the start of the expression' not in res.data
+    time.sleep(3)
 
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True
     )
-    # Plaintext that doesnt look like a regex should match also
-    assert b'and this should be' in res.data
+
 
     assert b'<div class="">Something' in res.data
     assert b'<div class="">across 6 billion multiple' in res.data
@@ -123,11 +115,14 @@ def test_check_filter_multiline(client, live_server):
     assert b'aaand something lines' not in res.data
 
 def test_check_filter_and_regex_extract(client, live_server):
-    
+    sleep_time_for_fetch_thread = 3
     include_filters = ".changetext"
 
     set_original_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -137,15 +132,19 @@ def test_check_filter_and_regex_extract(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    time.sleep(1)
+    # Trigger a check
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"include_filters": include_filters,
-              'extract_text': '/\d+ online/\r\n/\d+ guests/\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i\r\n/issue1828.+?2022/i',
+              'extract_text': '\d+ online\r\n\d+ guests\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i',
               "url": test_url,
               "tags": "",
               "headers": "",
@@ -156,13 +155,8 @@ def test_check_filter_and_regex_extract(client, live_server):
 
     assert b"Updated watch." in res.data
 
-
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("index"))
-    #issue 1828
-    assert b'not at the start of the expression' not in res.data
+    time.sleep(sleep_time_for_fetch_thread)
 
     #  Make a change
     set_modified_response()
@@ -170,7 +164,7 @@ def test_check_filter_and_regex_extract(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id
@@ -202,35 +196,3 @@ def test_check_filter_and_regex_extract(client, live_server):
 
     # Should not be here
     assert b'Some text that did change' not in res.data
-
-
-
-def test_regex_error_handling(client, live_server):
-
-    #live_server_setup(live_server)
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    ### test regex error handling
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"extract_text": '/something bad\d{3/XYZ',
-              "url": test_url,
-              "fetch_backend": "html_requests"},
-        follow_redirects=True
-    )
-
-    with open('/tmp/fuck.html', 'wb') as f:
-        f.write(res.data)
-
-    assert b'is not a valid regular expression.' in res.data
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_filter_exist_changes.py

@@ -15,7 +15,7 @@ def set_response_without_filter():
      <p>Which is across multiple lines</p>
      <br>
      So let's see what happens.  <br>
-     <div id="nope-doesnt-exist">Some text thats the same</div>
+     <div id="nope-doesnt-exist">Some text thats the same</div>     
      </body>
      </html>
     """
@@ -32,7 +32,7 @@ def set_response_with_filter():
      <p>Which is across multiple lines</p>
      <br>
      So let's see what happens.  <br>
-     <div class="ticket-available">Ticket now on sale!</div>
+     <div class="ticket-available">Ticket now on sale!</div>     
      </body>
      </html>
     """
@@ -84,7 +84,6 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
                                                    "Snapshot: {{current_snapshot}}\n"
                                                    "Diff: {{diff}}\n"
                                                    "Diff Full: {{diff_full}}\n"
-                                                   "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
                               "notification_format": "Text"}
 

changedetectionio/tests/test_filter_failure_notification.py

@@ -12,7 +12,7 @@ def set_response_with_filter():
      <p>Which is across multiple lines</p>
      <br>
      So let's see what happens.  <br>
-     <div id="nope-doesnt-exist">Some text thats the same</div>
+     <div id="nope-doesnt-exist">Some text thats the same</div>     
      </body>
      </html>
     """
@@ -66,7 +66,6 @@ def run_filter_test(client, content_filter):
                                                    "Snapshot: {{current_snapshot}}\n"
                                                    "Diff: {{diff}}\n"
                                                    "Diff Full: {{diff_full}}\n"
-                                                   "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
                               "notification_format": "Text"}
 

changedetectionio/tests/test_group.py

@@ -15,7 +15,7 @@ def set_original_response():
      Some initial text<br>
      <p id="only-this">Should be only this</p>
      <br>
-     <p id="not-this">And never this</p>
+     <p id="not-this">And never this</p>     
      </body>
      </html>
     """
@@ -30,7 +30,7 @@ def set_modified_response():
      Some initial text<br>
      <p id="only-this">Should be REALLY only this</p>
      <br>
-     <p id="not-this">And never this</p>
+     <p id="not-this">And never this</p>     
      </body>
      </html>
     """
@@ -189,7 +189,6 @@ def test_group_tag_notification(client, live_server):
                                                    "Diff Added: {{diff_added}}\n"
                                                    "Diff Removed: {{diff_removed}}\n"
                                                    "Diff Full: {{diff_full}}\n"
-                                                   "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
                               "notification_screenshot": True,
                               "notification_format": "Text",
@@ -320,4 +319,4 @@ def test_clone_tag_on_quickwatchform_add(client, live_server):
     assert b'Deleted' in res.data
 
     res = client.get(url_for("tags.delete_all"), follow_redirects=True)
-    assert b'All tags deleted' in res.data
+    assert b'All tags deleted' in res.data

changedetectionio/tests/test_ignore_regex_text.py

@@ -15,24 +15,11 @@ def test_strip_regex_text_func():
     but sometimes we want to remove the lines.
     
     but 1 lines
-    skip 5 lines
-    really? yes man
-#/not this tries weirdly formed regex or just strings starting with /
-/not this
     but including 1234 lines
     igNORe-cAse text we dont want to keep    
     but not always."""
 
-
-    ignore_lines = [
-        "sometimes",
-        "/\s\d{2,3}\s/",
-        "/ignore-case text/",
-        "really?",
-        "/skip \d lines/i",
-        "/not"
-    ]
-
+    ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"]
 
     fetcher = fetch_site_status.perform_site_check(datastore=False)
     stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
@@ -40,10 +27,4 @@ def test_strip_regex_text_func():
     assert b"but 1 lines" in stripped_content
     assert b"igNORe-cAse text" not in stripped_content
     assert b"but 1234 lines" not in stripped_content
-    assert b"really" not in stripped_content
-    assert b"not this" not in stripped_content
-
-    # Check line number reporting
-    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines, mode="line numbers")
-    assert stripped_content == [2, 5, 6, 7, 8, 10]
 

changedetectionio/tests/test_ignorehighlighter.py

@@ -1,57 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
-from changedetectionio import html_tools
-from . util import  extract_UUID_from_client
-
-def set_original_ignore_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text<br>
-     <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
-     <p>oh yeah 456</p>
-     </body>
-     </html>
-
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-
-def test_highlight_ignore(client, live_server):
-    live_server_setup(live_server)
-    set_original_ignore_response()
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    # Give the thread time to pick it up
-    wait_for_all_checks(client)
-    uuid = extract_UUID_from_client(client)
-    # use the highlighter endpoint
-    res = client.post(
-        url_for("highlight_submit_ignore_url", uuid=uuid),
-        data={"mode": 'digit-regex', 'selection': 'oh yeah 123'},
-        follow_redirects=True
-    )
-
-    res = client.get(url_for("edit_page", uuid=uuid))
-
-    # should be a regex now
-    assert b'/oh\ yeah\ \d+/' in res.data
-
-    # Should return a link
-    assert b'href' in res.data
-
-    # And it should register in the preview page
-    res = client.get(url_for("preview_page", uuid=uuid))
-    assert b'<div class="ignored">oh yeah 456' in res.data

changedetectionio/tests/test_import.py

@@ -1,19 +1,16 @@
 #!/usr/bin/python3
-import io
-import os
+
 import time
 
 from flask import url_for
 
-from .util import live_server_setup, wait_for_all_checks
-
-
+from .util import live_server_setup
 def test_setup(client, live_server):
     live_server_setup(live_server)
 
 def test_import(client, live_server):
     # Give the endpoint time to spin up
-    wait_for_all_checks(client)
+    time.sleep(1)
 
     res = client.post(
         url_for("import_page"),
@@ -122,97 +119,3 @@ def test_import_distillio(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     # Clear flask alerts
     res = client.get(url_for("index"))
-
-def test_import_custom_xlsx(client, live_server):
-    """Test can upload a excel spreadsheet and the watches are created correctly"""
-
-    #live_server_setup(live_server)
-
-    dirname = os.path.dirname(__file__)
-    filename = os.path.join(dirname, 'import/spreadsheet.xlsx')
-    with open(filename, 'rb') as f:
-
-        data= {
-            'file_mapping': 'custom',
-            'custom_xlsx[col_0]': '1',
-            'custom_xlsx[col_1]': '3',
-            'custom_xlsx[col_2]': '5',
-            'custom_xlsx[col_3]': '4',
-            'custom_xlsx[col_type_0]': 'title',
-            'custom_xlsx[col_type_1]': 'url',
-            'custom_xlsx[col_type_2]': 'include_filters',
-            'custom_xlsx[col_type_3]': 'interval_minutes',
-            'xlsx_file': (io.BytesIO(f.read()), 'spreadsheet.xlsx')
-        }
-
-    res = client.post(
-        url_for("import_page"),
-        data=data,
-        follow_redirects=True,
-    )
-
-    assert b'4 imported from custom .xlsx' in res.data
-    # Because this row was actually just a header with no usable URL, we should get an error
-    assert b'Error processing row number 1' in res.data
-
-    res = client.get(
-        url_for("index")
-    )
-
-    assert b'Somesite results ABC' in res.data
-    assert b'City news results' in res.data
-
-    # Just find one to check over
-    for uuid, watch in live_server.app.config['DATASTORE'].data['watching'].items():
-        if watch.get('title') == 'Somesite results ABC':
-            filters = watch.get('include_filters')
-            assert filters[0] == '/html[1]/body[1]/div[4]/div[1]/div[1]/div[1]||//*[@id=\'content\']/div[3]/div[1]/div[1]||//*[@id=\'content\']/div[1]'
-            assert watch.get('time_between_check') == {'weeks': 0, 'days': 1, 'hours': 6, 'minutes': 24, 'seconds': 0}
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-def test_import_watchete_xlsx(client, live_server):
-    """Test can upload a excel spreadsheet and the watches are created correctly"""
-
-    #live_server_setup(live_server)
-    dirname = os.path.dirname(__file__)
-    filename = os.path.join(dirname, 'import/spreadsheet.xlsx')
-    with open(filename, 'rb') as f:
-
-        data= {
-            'file_mapping': 'wachete',
-            'xlsx_file': (io.BytesIO(f.read()), 'spreadsheet.xlsx')
-        }
-
-    res = client.post(
-        url_for("import_page"),
-        data=data,
-        follow_redirects=True,
-    )
-
-    assert b'4 imported from Wachete .xlsx' in res.data
-
-    res = client.get(
-        url_for("index")
-    )
-
-    assert b'Somesite results ABC' in res.data
-    assert b'City news results' in res.data
-
-    # Just find one to check over
-    for uuid, watch in live_server.app.config['DATASTORE'].data['watching'].items():
-        if watch.get('title') == 'Somesite results ABC':
-            filters = watch.get('include_filters')
-            assert filters[0] == '/html[1]/body[1]/div[4]/div[1]/div[1]/div[1]||//*[@id=\'content\']/div[3]/div[1]/div[1]||//*[@id=\'content\']/div[1]'
-            assert watch.get('time_between_check') == {'weeks': 0, 'days': 1, 'hours': 6, 'minutes': 24, 'seconds': 0}
-            assert watch.get('fetch_backend') == 'html_requests' # Has inactive 'dynamic wachet'
-
-        if watch.get('title') == 'JS website':
-            assert watch.get('fetch_backend') == 'html_webdriver' # Has active 'dynamic wachet'
-
-        if watch.get('title') == 'system default website':
-            assert watch.get('fetch_backend') == 'system' # uses default if blank
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_notification.py

@@ -3,8 +3,7 @@ import os
 import time
 import re
 from flask import url_for
-from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
-    set_longer_modified_response
+from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -25,6 +24,9 @@ def test_check_notification(client, live_server):
     #live_server_setup(live_server)
     set_original_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Re 360 - new install should have defaults set
     res = client.get(url_for("settings_page"))
     notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
@@ -98,7 +100,6 @@ def test_check_notification(client, live_server):
                                                    "Diff Added: {{diff_added}}\n"
                                                    "Diff Removed: {{diff_removed}}\n"
                                                    "Diff Full: {{diff_full}}\n"
-                                                   "Diff as Patch: {{diff_patch}}\n"
                                                    ":-)",
                               "notification_screenshot": True,
                               "notification_format": "Text"}
@@ -141,7 +142,8 @@ def test_check_notification(client, live_server):
 
     # Did we see the URL that had a change, in the notification?
     # Diff was correctly executed
-
+    assert test_url in notification_submission
+    assert ':-)' in notification_submission
     assert "Diff Full: Some initial text" in notification_submission
     assert "Diff: (changed) Which is across multiple lines" in notification_submission
     assert "(into) which has this one new line" in notification_submission
@@ -154,8 +156,7 @@ def test_check_notification(client, live_server):
     assert "preview/" in notification_submission
     assert ":-)" in notification_submission
     assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
-    assert test_url in notification_submission
-    assert ':-)' in notification_submission
+
     # Check the attachment was added, and that it is a JPEG from the original PNG
     notification_submission_object = json.loads(notification_submission)
     # We keep PNG screenshots for now
@@ -274,7 +275,7 @@ def test_notification_validation(client, live_server):
 
 
 def test_notification_custom_endpoint_and_jinja2(client, live_server):
-    #live_server_setup(live_server)
+    time.sleep(1)
 
     # test_endpoint - that sends the contents of a file
     # test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt)
@@ -285,14 +286,12 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
 
     res = client.post(
         url_for("settings_page"),
-        data={
-              "application-fetch_backend": "html_requests",
-              "application-minutes_between_check": 180,
+        data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
               "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }',
-              "application-notification_format": default_notification_format,
-              "application-notification_urls": test_notification_url,
               # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-              "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
+              "application-notification_urls": test_notification_url,
+              "application-minutes_between_check": 180,
+              "application-fetch_backend": "html_requests"
               },
         follow_redirects=True
     )
@@ -317,8 +316,9 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(2)
 
+
     with open("test-datastore/notification.txt", 'r') as f:
-        x = f.read()
+        x=f.read()
         j = json.loads(x)
         assert j['url'].startswith('http://localhost')
         assert j['secret'] == 444
@@ -329,9 +329,5 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         notification_url = f.read()
         assert 'xxx=http' in notification_url
 
-    # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
-    assert os.path.isfile("test-datastore/notification-content-type.txt")
-    with open("test-datastore/notification-content-type.txt", 'r') as f:
-        assert 'application/json' in f.read()
-
     os.unlink("test-datastore/notification-url.txt")
+

changedetectionio/tests/test_rss.py

@@ -2,61 +2,12 @@
 
 import time
 from flask import url_for
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
-    extract_UUID_from_client
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI
 
 
-def set_original_cdata_xml():
-    test_return_data = """<rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:media="http://search.yahoo.com/mrss/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
-    <channel>
-    <title>Gizi</title>
-    <link>https://test.com</link>
-    <atom:link href="https://testsite.com" rel="self" type="application/rss+xml"/>
-    <description>
-    <![CDATA[ The Future Could Be Here ]]>
-    </description>
-    <language>en</language>
-    <item>
-    <title>
-    <![CDATA[ <img src="https://testsite.com/hacked.jpg"> Hackers can access your computer ]]>
-    </title>
-    <link>https://testsite.com/news/12341234234</link>
-    <description>
-    <![CDATA[ <img class="type:primaryImage" src="https://testsite.com/701c981da04869e.jpg"/><p>The days of Terminator and The Matrix could be closer. But be positive.</p><p><a href="https://testsite.com">Read more link...</a></p> ]]>
-    </description>
-    <category>cybernetics</category>
-    <category>rand corporation</category>
-    <pubDate>Tue, 17 Oct 2023 15:10:00 GMT</pubDate>
-    <guid isPermaLink="false">1850933241</guid>
-    <dc:creator>
-    <![CDATA[ Mr Hacker News ]]>
-    </dc:creator>
-    <media:thumbnail url="https://testsite.com/thumbnail-c224e10d81488e818701c981da04869e.jpg"/>
-    </item>
-
-    <item>
-        <title>    Some other title    </title>
-        <link>https://testsite.com/news/12341234236</link>
-        <description>
-        Some other description
-        </description>
-    </item>    
-    </channel>
-    </rss>
-            """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-
-def test_setup(client, live_server):
-    live_server_setup(live_server)
-
 def test_rss_and_token(client, live_server):
-    #    live_server_setup(live_server)
-
     set_original_response()
-    rss_token = extract_rss_token_from_UI(client)
+    live_server_setup(live_server)
 
     # Add our URL to the import page
     res = client.post(
@@ -66,11 +17,11 @@ def test_rss_and_token(client, live_server):
     )
 
     assert b"1 Imported" in res.data
+    rss_token = extract_rss_token_from_UI(client)
 
-    wait_for_all_checks(client)
-    set_modified_response()
+    time.sleep(2)
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     # Add our URL to the import page
     res = client.get(
@@ -86,80 +37,3 @@ def test_rss_and_token(client, live_server):
     )
     assert b"Access denied, bad token" not in res.data
     assert b"Random content" in res.data
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-
-def test_basic_cdata_rss_markup(client, live_server):
-    #live_server_setup(live_server)
-
-    set_original_cdata_xml()
-
-    test_url = url_for('test_endpoint', content_type="application/xml", _external=True)
-
-    # Add our URL to the import page
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-
-    wait_for_all_checks(client)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    assert b'CDATA' not in res.data
-    assert b'<![' not in res.data
-    assert b'Hackers can access your computer' in res.data
-    assert b'The days of Terminator' in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-
-def test_rss_xpath_filtering(client, live_server):
-    #live_server_setup(live_server)
-
-    set_original_cdata_xml()
-
-    test_url = url_for('test_endpoint', content_type="application/xml", _external=True)
-
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
-        follow_redirects=True
-    )
-    assert b"Watch added in Paused state, saving will unpause" in res.data
-
-    uuid = extract_UUID_from_client(client)
-    res = client.post(
-        url_for("edit_page", uuid=uuid, unpause_on_save=1),
-        data={
-                "include_filters": "//item/title",
-                "fetch_backend": "html_requests",
-                "headers": "",
-                "proxy": "no-proxy",
-                "tags": "",
-                "url": test_url,
-              },
-        follow_redirects=True
-    )
-    assert b"unpaused" in res.data
-
-    wait_for_all_checks(client)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    assert b'CDATA' not in res.data
-    assert b'<![' not in res.data
-    # #1874  All but the first <title was getting selected
-    # Convert any HTML with just a top level <title> to <h1> to be sure title renders
-
-    assert b'Hackers can access your computer' in res.data # Should ONLY be selected by the xpath
-    assert b'Some other title' in res.data  # Should ONLY be selected by the xpath
-    assert b'The days of Terminator' not in res.data # Should NOT be selected by the xpath
-    assert b'Some other description' not in res.data  # Should NOT be selected by the xpath
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_trigger_regex.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from . util import live_server_setup
 
 
 def set_original_ignore_response():
@@ -26,8 +26,13 @@ def test_trigger_regex_functionality(client, live_server):
 
     live_server_setup(live_server)
 
+    sleep_time_for_fetch_thread = 3
+
     set_original_ignore_response()
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -38,7 +43,7 @@ def test_trigger_regex_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (just a new one shouldnt have anything)
     res = client.get(url_for("index"))
@@ -52,7 +57,7 @@ def test_trigger_regex_functionality(client, live_server):
               "fetch_backend": "html_requests"},
         follow_redirects=True
     )
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     # so that we set the state to 'unviewed' after all the edits
     client.get(url_for("diff_history_page", uuid="first"))
 
@@ -60,7 +65,7 @@ def test_trigger_regex_functionality(client, live_server):
         f.write("some new noise")
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # It should report nothing found (nothing should match the regex)
     res = client.get(url_for("index"))
@@ -70,7 +75,7 @@ def test_trigger_regex_functionality(client, live_server):
         f.write("regex test123<br>\nsomething 123")
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 

changedetectionio/tests/test_xpath_selector.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks
+from . util import live_server_setup
 
 from ..html_tools import *
 
@@ -86,14 +86,14 @@ def test_check_xpath_filter_utf8(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
+    time.sleep(1)
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
+    time.sleep(3)
     res = client.get(url_for("index"))
     assert b'Unicode strings with encoding declaration are not supported.' not in res.data
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
@@ -140,14 +140,14 @@ def test_check_xpath_text_function_utf8(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
+    time.sleep(1)
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"include_filters": filter, "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
+    time.sleep(3)
     res = client.get(url_for("index"))
     assert b'Unicode strings with encoding declaration are not supported.' not in res.data
 
@@ -164,6 +164,7 @@ def test_check_xpath_text_function_utf8(client, live_server):
     assert b'Deleted' in res.data
 
 def test_check_markup_xpath_filter_restriction(client, live_server):
+    sleep_time_for_fetch_thread = 3
 
     xpath_filter = "//*[contains(@class, 'sametext')]"
 
@@ -182,7 +183,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
@@ -194,7 +195,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     assert b"Updated watch." in res.data
 
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     # view it/reset state back to viewed
     client.get(url_for("diff_history_page", uuid="first"), follow_redirects=True)
@@ -205,7 +206,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    wait_for_all_checks(client)
+    time.sleep(sleep_time_for_fetch_thread)
 
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
@@ -215,6 +216,9 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
 
 def test_xpath_validation(client, live_server):
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -223,7 +227,7 @@ def test_xpath_validation(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
+    time.sleep(2)
 
     res = client.post(
         url_for("edit_page", uuid="first"),
@@ -240,8 +244,11 @@ def test_check_with_prefix_include_filters(client, live_server):
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
     set_original_response()
-    wait_for_all_checks(client)
+
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -250,7 +257,7 @@ def test_check_with_prefix_include_filters(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     res = client.post(
         url_for("edit_page", uuid="first"),
@@ -259,7 +266,7 @@ def test_check_with_prefix_include_filters(client, live_server):
     )
 
     assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
+    time.sleep(3)
 
     res = client.get(
         url_for("preview_page", uuid="first"),
@@ -270,46 +277,3 @@ def test_check_with_prefix_include_filters(client, live_server):
     assert b"Some text that will change" not in res.data #not in selector
 
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-def test_various_rules(client, live_server):
-    # Just check these don't error
-    #live_server_setup(live_server)
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write("""<html>
-       <body>
-     Some initial text<br>
-     <p>Which is across multiple lines</p>
-     <br>
-     So let's see what happens.  <br>
-     <div class="sametext">Some text thats the same</div>
-     <div class="changetext">Some text that will change</div>
-     <a href=''>some linky </a>
-     <a href=''>another some linky </a>
-     <!-- related to https://github.com/dgtlmoon/changedetection.io/pull/1774 -->
-     <input   type="email"   id="email" />
-     </body>
-     </html>
-    """)
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    wait_for_all_checks(client)
-
-
-    for r in ['//div', '//a', 'xpath://div', 'xpath://a']:
-        res = client.post(
-            url_for("edit_page", uuid="first"),
-            data={"include_filters": r,
-                  "url": test_url,
-                  "tags": "",
-                  "headers": "",
-                  'fetch_backend': "html_requests"},
-            follow_redirects=True
-        )
-        wait_for_all_checks(client)
-        assert b"Updated watch." in res.data
-        res = client.get(url_for("index"))
-        assert b'fetch-error' not in res.data, f"Should not see errors after '{r} filter"

changedetectionio/tests/unit/test_notification_diff.py

@@ -36,7 +36,7 @@ class TestDiffBuilder(unittest.TestCase):
         output = output.split("\n")
         self.assertIn('(removed) for having learned computerese,', output)
         self.assertIn('(removed) I continue to examine bits, bytes and words', output)
-
+        
         #diff_removed
         with open(base_dir + "/test-content/before.txt", 'r') as f:
             previous_version_file_contents = f.read()
@@ -49,7 +49,7 @@ class TestDiffBuilder(unittest.TestCase):
         self.assertIn('(into) xok', output)
         self.assertIn('(into) next-x-ok', output)
         self.assertNotIn('(added) and something new', output)
-
+        
         #diff_removed
         with open(base_dir + "/test-content/after-2.txt", 'r') as f:
             newest_version_file_contents = f.read()
@@ -57,25 +57,9 @@ class TestDiffBuilder(unittest.TestCase):
         output = output.split("\n")
         self.assertIn('(removed) for having learned computerese,', output)
         self.assertIn('(removed) I continue to examine bits, bytes and words', output)
-
-    def test_expected_diff_patch_output(self):
-        base_dir = os.path.dirname(__file__)
-        with open(base_dir + "/test-content/before.txt", 'r') as f:
-            before = f.read()
-        with open(base_dir + "/test-content/after.txt", 'r') as f:
-            after = f.read()
-
-        output = diff.render_diff(previous_version_file_contents=before,
-                                  newest_version_file_contents=after,
-                                  patch_format=True)
-        output = output.split("\n")
-
-        self.assertIn('-ok', output)
-        self.assertIn('+xok', output)
-        self.assertIn('+next-x-ok', output)
-        self.assertIn('+and something new', output)
+        
 
         # @todo test blocks of changed, blocks of added, blocks of removed
 
 if __name__ == '__main__':
-    unittest.main()
+    unittest.main()

changedetectionio/tests/util.py

@@ -38,25 +38,7 @@ def set_modified_response():
         f.write(test_return_data)
 
     return None
-def set_longer_modified_response():
-    test_return_data = """<html>
-    <head><title>modified head title</title></head>
-    <body>
-     Some initial text<br>
-     <p>which has this one new line</p>
-     <br>
-     So let's see what happens.  <br>
-     So let's see what happens.  <br>
-      So let's see what happens.  <br>
-     So let's see what happens.  <br>           
-     </body>
-     </html>
-    """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-    return None
 def set_more_modified_response():
     test_return_data = """<html>
     <head><title>modified head title</title></head>
@@ -205,10 +187,6 @@ def live_server_setup(live_server):
         with open("test-datastore/notification-url.txt", "w") as f:
             f.write(request.url)
 
-        if request.content_type:
-            with open("test-datastore/notification-content-type.txt", "w") as f:
-                f.write(request.content_type)
-
         print("\n>> Test notification endpoint was hit.\n", data)
         return "Text was set"
 

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -1,19 +1,18 @@
 #!/usr/bin/python3
 
 import time
-import os
 from flask import url_for
 from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
 
-def test_setup(client, live_server):
-    live_server_setup(live_server)
-
 # Add a site in paused mode, add an invalid filter, we should still have visual selector data ready
 def test_visual_selector_content_ready(client, live_server):
     import os
     import json
 
     assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
+    time.sleep(1)
+    live_server_setup(live_server)
+
 
     # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url
     test_url = "https://changedetection.io/ci-test/test-runjs.html"
@@ -54,13 +53,6 @@ def test_visual_selector_content_ready(client, live_server):
     with open(os.path.join('test-datastore', uuid, 'elements.json'), 'r') as f:
         json.load(f)
 
-    # Attempt to fetch it via the web hook that the browser would use
-    res = client.get(url_for('static_content', group='visual_selector_data', filename=uuid))
-    json.loads(res.data)
-    assert res.mimetype == 'application/json'
-    assert res.status_code == 200
-
-
     # Some options should be enabled
     # @todo - in the future, the visibility should be toggled by JS from the request type setting
     res = client.get(
@@ -68,75 +60,4 @@ def test_visual_selector_content_ready(client, live_server):
         follow_redirects=True
     )
     assert b'notification_screenshot' in res.data
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )
 
-def test_basic_browserstep(client, live_server):
-
-    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), "Needs PLAYWRIGHT_DRIVER_URL set for this test"
-    #live_server_setup(live_server)
-
-    # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url
-    test_url = "https://changedetection.io/ci-test/test-runjs.html"
-
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
-        follow_redirects=True
-    )
-    assert b"Watch added in Paused state, saving will unpause" in res.data
-
-    res = client.post(
-        url_for("edit_page", uuid="first", unpause_on_save=1),
-        data={
-              "url": test_url,
-              "tags": "",
-              "headers": "",
-              'fetch_backend': "html_webdriver",
-              'browser_steps-0-operation': 'Goto site',
-              'browser_steps-1-operation': 'Click element',
-              'browser_steps-1-selector': 'button[name=test-button]',
-              'browser_steps-1-optional_value': ''
-        },
-        follow_redirects=True
-    )
-    assert b"unpaused" in res.data
-    wait_for_all_checks(client)
-
-    uuid = extract_UUID_from_client(client)
-
-    # Check HTML conversion detected and workd
-    res = client.get(
-        url_for("preview_page", uuid=uuid),
-        follow_redirects=True
-    )
-    assert b"This text should be removed" not in res.data
-    assert b"I smell JavaScript because the button was pressed" in res.data
-
-    # now test for 404 errors
-    res = client.post(
-        url_for("edit_page", uuid=uuid, unpause_on_save=1),
-        data={
-              "url": "https://changedetection.io/404",
-              "tags": "",
-              "headers": "",
-              'fetch_backend': "html_webdriver",
-              'browser_steps-0-operation': 'Goto site',
-              'browser_steps-1-operation': 'Click element',
-              'browser_steps-1-selector': 'button[name=test-button]',
-              'browser_steps-1-optional_value': ''
-        },
-        follow_redirects=True
-    )
-    assert b"unpaused" in res.data
-    wait_for_all_checks(client)
-
-    res = client.get(url_for("index"))
-    assert b'Error - 404' in res.data
-
-    client.get(
-        url_for("form_delete", uuid="all"),
-        follow_redirects=True
-    )

changedetectionio/update_worker.py

@@ -3,7 +3,7 @@ import threading
 import queue
 import time
 
-from changedetectionio import content_fetcher, html_tools
+from changedetectionio import content_fetcher
 from .processors.text_json_diff import FilterNotFoundInResponse
 from .processors.restock_diff import UnableToExtractRestockData
 
@@ -32,17 +32,15 @@ class update_worker(threading.Thread):
 
         watch_history = watch.history
         dates = list(watch_history.keys())
-        # Add text that was triggered
-        snapshot_contents = watch.get_history_snapshot(dates[-1])
 
         # HTML needs linebreak, but MarkDown and Text can use a linefeed
         if n_object['notification_format'] == 'HTML':
             line_feed_sep = "<br>"
-            # Snapshot will be plaintext on the disk, convert to some kind of HTML
-            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
         else:
             line_feed_sep = "\n"
 
+        # Add text that was triggered
+        snapshot_contents = watch.get_history_snapshot(dates[-1])
         trigger_text = watch.get('trigger_text', [])
         triggered_text = ''
 
@@ -58,7 +56,6 @@ class update_worker(threading.Thread):
             'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
             'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
             'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
-            'diff_patch': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep, patch_format=True),
             'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
             'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
             'triggered_text': triggered_text,
@@ -68,48 +65,15 @@ class update_worker(threading.Thread):
         logging.info (">> SENDING NOTIFICATION")
         self.notification_q.put(n_object)
 
-    # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
-    def _check_cascading_vars(self, var_name, watch):
-
-        from changedetectionio.notification import (
-            default_notification_format_for_watch,
-            default_notification_body,
-            default_notification_title
-        )
-
-
-        # Would be better if this was some kind of Object where Watch can reference the parent datastore etc
-        v = watch.get(var_name)
-        if v and not watch.get('notification_muted'):
-            if var_name == 'notification_format' and v == default_notification_format_for_watch:
-                return self.datastore.data['settings']['application'].get('notification_format')
-
-            return v
-
-        tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
-        if tags:
-            for tag_uuid, tag in tags.items():
-                v = tag.get(var_name)
-                if v and not tag.get('notification_muted'):
-                    return v
-
-        if self.datastore.data['settings']['application'].get(var_name):
-            return self.datastore.data['settings']['application'].get(var_name)
-
-        # Otherwise could be defaults
-        if var_name == 'notification_format':
-            return default_notification_format_for_watch
-        if var_name == 'notification_body':
-            return default_notification_body
-        if var_name == 'notification_title':
-            return default_notification_title
-
-        return None
 
     def send_content_changed_notification(self, watch_uuid):
 
+        from changedetectionio.notification import (
+            default_notification_format_for_watch
+        )
+
         n_object = {}
-        watch = self.datastore.data['watching'].get(watch_uuid)
+        watch = self.datastore.data['watching'].get(watch_uuid, False)
         if not watch:
             return
 
@@ -123,20 +87,57 @@ class update_worker(threading.Thread):
             )
 
         # Should be a better parent getter in the model object
-
         # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
-        n_object['notification_urls'] = self._check_cascading_vars('notification_urls', watch)
-        n_object['notification_title'] = self._check_cascading_vars('notification_title', watch)
-        n_object['notification_body'] = self._check_cascading_vars('notification_body', watch)
-        n_object['notification_format'] = self._check_cascading_vars('notification_format', watch)
+        n_object['notification_urls'] = watch.get('notification_urls')
+
+        n_object['notification_title'] = watch['notification_title'] if watch['notification_title'] else \
+            self.datastore.data['settings']['application']['notification_title']
+
+        n_object['notification_body'] = watch['notification_body'] if watch['notification_body'] else \
+            self.datastore.data['settings']['application']['notification_body']
+
+        n_object['notification_format'] = watch['notification_format'] if watch['notification_format'] != default_notification_format_for_watch else \
+            self.datastore.data['settings']['application']['notification_format']
 
         # (Individual watch) Only prepare to notify if the rules above matched
-        queued = False
-        if n_object and n_object.get('notification_urls'):
-            queued = True
+        sent = False
+        if 'notification_urls' in n_object and n_object['notification_urls']:
+            sent = True
             self.queue_notification_for_watch(n_object, watch)
 
-        return queued
+        # (Group tags) try by group tag
+        if not sent:
+            # Else, Try by tag, and use system default vars for format, body etc as fallback
+            tags = self.datastore.get_all_tags_for_watch(uuid=watch_uuid)
+            for tag_uuid, tag in tags.items():
+                n_object = {}
+                n_object['notification_urls'] = tag.get('notification_urls')
+
+                n_object['notification_title'] = tag.get('notification_title') if tag.get('notification_title') else \
+                    self.datastore.data['settings']['application']['notification_title']
+
+                n_object['notification_body'] = tag.get('notification_body') if tag.get('notification_body') else \
+                    self.datastore.data['settings']['application']['notification_body']
+
+                n_object['notification_format'] = tag.get('notification_format') if tag.get('notification_format') != default_notification_format_for_watch else \
+                    self.datastore.data['settings']['application']['notification_format']
+
+                if 'notification_urls' in n_object and n_object.get('notification_urls') and not tag.get('notification_muted'):
+                    sent = True
+                    self.queue_notification_for_watch(n_object, watch)
+
+        # (Group tags) try by global
+        if not sent:
+            # leave this as is, but repeat in a loop for each tag also
+            n_object['notification_urls'] = self.datastore.data['settings']['application'].get('notification_urls')
+            n_object['notification_title'] = self.datastore.data['settings']['application'].get('notification_title')
+            n_object['notification_body'] = self.datastore.data['settings']['application'].get('notification_body')
+            n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
+            if n_object.get('notification_urls') and n_object.get('notification_body') and n_object.get('notification_title'):
+                sent = True
+                self.queue_notification_for_watch(n_object, watch)
+
+        return sent
 
 
     def send_filter_failure_notification(self, watch_uuid):
@@ -238,9 +239,7 @@ class update_worker(threading.Thread):
                             # Used as a default and also by some tests
                             update_handler = text_json_diff.perform_site_check(datastore=self.datastore)
 
-                        self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
                         changed_detected, update_obj, contents = update_handler.run(uuid, skip_when_checksum_same=queued_item_data.item.get('skip_when_checksum_same'))
-
                         # Re #342
                         # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
                         # We then convert/.decode('utf-8') for the notification etc
@@ -253,20 +252,7 @@ class update_worker(threading.Thread):
                         # Totally fine, it's by choice - just continue on, nothing more to care about
                         # Page had elements/content but no renderable text
                         # Backend (not filters) gave zero output
-                        extra_help = ""
-                        if e.has_filters:
-                            # Maybe it contains an image? offer a more helpful link
-                            has_img = html_tools.include_filters(include_filters='img',
-                                                                 html_content=e.html_content)
-                            if has_img:
-                                extra_help = ", it's possible that the filters you have give an empty result or contain only an image."
-                            else:
-                                extra_help = ", it's possible that the filters were found, but contained no usable text."
-
-                        self.datastore.update_watch(uuid=uuid, update_obj={
-                            'last_error': f"Got HTML content but no text found (With {e.status_code} reply code){extra_help}"
-                        })
-
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Got HTML content but no text found (With {} reply code).".format(e.status_code)})
                         if e.screenshot:
                             self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot)
                         process_changedetection_results = False
@@ -326,13 +312,8 @@ class update_worker(threading.Thread):
                         if not self.datastore.data['watching'].get(uuid):
                             continue
 
-                        error_step = e.step_n + 1
-                        err_text = f"Warning, browser step at position {error_step} could not run, target not found, check the watch, add a delay if necessary, view Browser Steps to see screenshot at that step"
-                        self.datastore.update_watch(uuid=uuid,
-                                                    update_obj={'last_error': err_text,
-                                                                'browser_steps_last_error_step': error_step
-                                                                }
-                                                    )
+                        err_text = "Warning, browser step at position {} could not run, target not found, check the watch, add a delay if necessary.".format(e.step_n+1)
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
 
 
                         if self.datastore.data['watching'][uuid].get('filter_failure_notification_send', False):
@@ -400,9 +381,6 @@ class update_worker(threading.Thread):
                         if not self.datastore.data['watching'][uuid].get('ignore_status_codes'):
                             update_obj['consecutive_filter_failures'] = 0
 
-                        # Everything ran OK, clean off any previous error
-                        update_obj['last_error'] = False
-
                         self.cleanup_error_artifacts(uuid)
 
                     #

docker-compose.yml

@@ -47,9 +47,6 @@ services:
   #
   #        Hides the `Referer` header so that monitored websites can't see the changedetection.io hostname.
   #      - HIDE_REFERER=true
-  #        
-  #        Default number of parallel/concurrent fetchers
-  #      - FETCH_WORKERS=10
 
       # Comment out ports: when using behind a reverse proxy , enable networks: etc.
       ports:
@@ -66,12 +63,25 @@ services:
 #        browser-chrome:
 #            condition: service_started
 
+#    browser-chrome:
+#        hostname: browser-chrome
+#        image: selenium/standalone-chrome-debug:3.141.59
+#        environment:
+#            - VNC_NO_PASSWORD=1
+#            - SCREEN_WIDTH=1920
+#            - SCREEN_HEIGHT=1080
+#            - SCREEN_DEPTH=24
+#        volumes:
+#            # Workaround to avoid the browser crashing inside a docker container
+#            # See https://github.com/SeleniumHQ/docker-selenium#quick-start
+#            - /dev/shm:/dev/shm
+#        restart: unless-stopped
+
      # Used for fetching pages via Playwright+Chrome where you need Javascript support.
-     # Note: Playwright/browserless not supported on ARM type devices (rPi etc)
-     # RECOMMENDED FOR FETCHING PAGES WITH CHROME
+
 #    playwright-chrome:
 #        hostname: playwright-chrome
-#        image: browserless/chrome:1.60-chrome-stable
+#        image: browserless/chrome
 #        restart: unless-stopped
 #        environment:
 #            - SCREEN_WIDTH=1920
@@ -88,23 +98,6 @@ services:
 #             Ignore HTTPS errors, like for self-signed certs
 #            - DEFAULT_IGNORE_HTTPS_ERRORS=true
 #
-
-     # Used for fetching pages via Playwright+Chrome where you need Javascript support.
-     # Note: works well but is deprecated, doesnt fetch full page screenshots and other issues
-#    browser-chrome:
-#        hostname: browser-chrome
-#        image: selenium/standalone-chrome:4
-#        environment:
-#            - VNC_NO_PASSWORD=1
-#            - SCREEN_WIDTH=1920
-#            - SCREEN_HEIGHT=1080
-#            - SCREEN_DEPTH=24
-#        volumes:
-#            # Workaround to avoid the browser crashing inside a docker container
-#            # See https://github.com/SeleniumHQ/docker-selenium#quick-start
-#            - /dev/shm:/dev/shm
-#        restart: unless-stopped
-
 volumes:
   changedetection-data:
 

requirements.txt

@@ -1,23 +1,21 @@
-eventlet>=0.33.3 # related to dnspython fixes
+eventlet>=0.31.0
 feedgen~=0.9
 flask-compress
-# 0.6.3 included compatibility fix for werkzeug 3.x (2.x had deprecation of url handlers)
-flask-login>=0.6.3
+flask-login~=0.5
 flask-paginate
 flask_expects_json~=1.7
 flask_restful
-flask_wtf~=1.2
-flask~=2.3
+flask_wtf
+flask~=2.0
 inscriptis~=2.2
 pytz
 timeago~=1.0
-validators~=0.21
-
+validators
 
 # Set these versions together to avoid a RequestsDependencyWarning
 # >= 2.26 also adds Brotli support if brotli is installed
 brotli~=1.0
-requests[socks]
+requests[socks] ~=2.28
 
 urllib3>1.26
 chardet>2.3.0
@@ -25,12 +23,16 @@ chardet>2.3.0
 wtforms~=3.0
 jsonpath-ng~=1.5.3
 
-dnspython~=2.4 # related to eventlet fixes
+
+# dnspython 2.3.0 is not compatible with eventlet
+# * https://github.com/eventlet/eventlet/issues/781
+# * https://datastax-oss.atlassian.net/browse/PYTHON-1320
+dnspython<2.3.0
 
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise~=1.6.0
+apprise~=1.3.0
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
@@ -46,18 +48,21 @@ beautifulsoup4
 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
 lxml
 
-selenium~=4.14.0
+# 3.141 was missing socksVersion, 3.150 was not in pypi, so we try 4.1.0
+selenium~=4.1.0
 
-werkzeug~=3.0
+# https://stackoverflow.com/questions/71652965/importerror-cannot-import-name-safe-str-cmp-from-werkzeug-security/71653849#71653849
+# ImportError: cannot import name 'safe_str_cmp' from 'werkzeug.security'
+# need to revisit flask login versions
+werkzeug~=2.0.0
 
 # Templating, so far just in the URLs but in the future can be for the notifications also
 jinja2~=3.1
 jinja2-time
-openpyxl
+
 # https://peps.python.org/pep-0508/#environment-markers
 # https://github.com/dgtlmoon/changedetection.io/pull/1009
-jq~=1.3; python_version >= "3.8" and sys_platform == "darwin"
-jq~=1.3; python_version >= "3.8" and sys_platform == "linux"
+jq~=1.3 ;python_version >= "3.8" and sys_platform == "linux"
 
 # Any current modern version, required so far for screenshot PNG->JPEG conversion but will be used more in the future
 pillow
@@ -66,6 +71,3 @@ pillow
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup
 pytest ~=7.2
 pytest-flask ~=1.2
-
-# Pin jsonschema version to prevent build errors on armv6 while rpds-py wheels aren't available (1708)
-jsonschema==4.17.3

runtime.txt

@@ -1 +1 @@
-python-3.11.5
+python-3.9.15

setup.py

@@ -41,7 +41,7 @@ setup(
     include_package_data=True,
     install_requires=install_requires,
     license="Apache License 2.0",
-    python_requires=">= 3.7",
+    python_requires=">= 3.6",
     classifiers=['Intended Audience :: Customer Service',
                  'Intended Audience :: Developers',
                  'Intended Audience :: Education',