Add more debug

.github/test/Dockerfile-alpine

@@ -2,7 +2,7 @@
 # Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)
 # Some packages wont install via pypi because they dont have a wheel available under this architecture.
 
-FROM ghcr.io/linuxserver/baseimage-alpine:3.16
+FROM ghcr.io/linuxserver/baseimage-alpine:3.18
 ENV PYTHONUNBUFFERED=1
 
 COPY requirements.txt /requirements.txt
@@ -26,6 +26,6 @@ RUN \
     py3-pip && \
   echo "**** pip3 install test of changedetection.io ****" && \
   pip3 install -U pip wheel setuptools && \
-  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.16/ -r /requirements.txt && \
+  pip3 install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.18/ -r /requirements.txt && \
   apk del --purge \
     build-dependencies

.github/workflows/codeql-analysis.yml

@@ -30,11 +30,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -45,7 +45,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -59,4 +59,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/containers.yml

@@ -39,11 +39,11 @@ jobs:
     # Or if we are in a tagged release scenario.
     if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: 3.11
 
       - name: Install dependencies
         run: |
@@ -58,27 +58,27 @@ jobs:
           echo ${{ github.ref }} > changedetectionio/tag.txt
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
         with:
           image: tonistiigi/binfmt:latest
           platforms: all
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Login to Docker Hub Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
         with:
           install: true
           version: latest
@@ -88,7 +88,7 @@ jobs:
       - name: Build and push :dev
         id: docker_build
         if: ${{ github.ref }} == "refs/heads/master"
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: ./
           file: ./Dockerfile
@@ -105,7 +105,7 @@ jobs:
       - name: Build and push :tag
         id: docker_build_tag_release
         if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: ./
           file: ./Dockerfile
@@ -125,7 +125,7 @@ jobs:
         run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
 
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}

.github/workflows/test-container-build.yml

@@ -24,22 +24,22 @@ jobs:
   test-container-build:
     runs-on: ubuntu-latest
     steps:
-        - uses: actions/checkout@v2
-        - name: Set up Python 3.9
-          uses: actions/setup-python@v2
+        - uses: actions/checkout@v4
+        - name: Set up Python 3.11
+          uses: actions/setup-python@v4
           with:
-            python-version: 3.9
+            python-version: 3.11
 
         # Just test that the build works, some libraries won't compile on ARM/rPi etc
         - name: Set up QEMU
-          uses: docker/setup-qemu-action@v1
+          uses: docker/setup-qemu-action@v3
           with:
             image: tonistiigi/binfmt:latest
             platforms: all
 
         - name: Set up Docker Buildx
           id: buildx
-          uses: docker/setup-buildx-action@v1
+          uses: docker/setup-buildx-action@v3
           with:
             install: true
             version: latest
@@ -49,7 +49,7 @@ jobs:
         # Check we can still build under alpine/musl
         - name: Test that the docker containers can build (musl via alpine check)
           id: docker_build_musl
-          uses: docker/build-push-action@v2
+          uses: docker/build-push-action@v5
           with:
             context: ./
             file: ./.github/test/Dockerfile-alpine
@@ -57,7 +57,7 @@ jobs:
 
         - name: Test that the docker containers can build
           id: docker_build
-          uses: docker/build-push-action@v2
+          uses: docker/build-push-action@v5
           # https://github.com/docker/build-push-action#customizing
           with:
             context: ./

.github/workflows/test-only.yml

@@ -7,13 +7,13 @@ jobs:
   test-application:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       # Mainly just for link/flake8
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v2
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Lint with flake8
         run: |
@@ -83,19 +83,59 @@ jobs:
         run: |
           cd changedetectionio
           ./run_proxy_tests.sh
+          # And again with PLAYWRIGHT_DRIVER_URL=..
           cd ..
 
       - name: Test changedetection.io container starts+runs basically without error
         run: |
-          docker run -p 5556:5000 -d test-changedetectionio
+          docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
           sleep 3
           # Should return 0 (no error) when grep finds it
           curl -s http://localhost:5556 |grep -q checkbox-uuid
           
           # and IPv6
           curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+          docker kill test-changedetectionio
+
+      - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
+        run: |
           
+          echo SIGINT Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGINT to sig-test container"
+          docker kill --signal=SIGINT sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          docker logs sig-test
+          docker inspect --format '{{json .State.Running}}' sig-test
+          if [ $? -eq 0 ]
+          then
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists 
+          docker rm sig-test
+          
+          echo SIGTERM Shutdown request test
+          docker run --name sig-test -d test-changedetectionio
+          sleep 3
+          echo ">>> Sending SIGTERM to sig-test container"
+          docker kill --signal=SIGTERM sig-test
+          sleep 3
+          # invert the check (it should be not 0/not running)
+          docker ps
+          docker logs sig-test
+          docker inspect --format '{{json .State.Running}}' sig-test
+          if [ $? -eq 0 ]
+          then
+            exit 1
+          fi
+          
+          # @todo - scan the container log to see the right "graceful shutdown" text exists           
+          docker rm sig-test
 
 #export WEBDRIVER_URL=http://localhost:4444/wd/hub
 #pytest tests/fetchers/test_content.py
-#pytest tests/test_errorhandling.py
+#pytest tests/test_errorhandling.py

.github/workflows/test-pip-build.yml

@@ -11,12 +11,12 @@ jobs:
   test-pip-build-basics:
     runs-on: ubuntu-latest
     steps:
-        - uses: actions/checkout@v2
+        - uses: actions/checkout@v4
 
-        - name: Set up Python 3.9
-          uses: actions/setup-python@v2
+        - name: Set up Python 3.11
+          uses: actions/setup-python@v4
           with:
-            python-version: 3.9
+            python-version: 3.11
 
 
         - name: Test that the basic pip built package runs without error

Dockerfile

@@ -1,5 +1,5 @@
 # pip dependencies install stage
-FROM python:3.10-slim-bullseye as builder
+FROM python:3.11-slim-bullseye as builder
 
 # See `cryptography` pin comment in requirements.txt
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
@@ -29,7 +29,7 @@ RUN pip install --target=/dependencies playwright~=1.27.1 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage
-FROM python:3.10-slim-bullseye
+FROM python:3.11-slim-bullseye
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libssl1.1 \

README-pip.md

@@ -2,19 +2,44 @@
 
 Live your data-life pro-actively, track website content changes and receive notifications via Discord, Email, Slack, Telegram and 70+ more
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://changedetection.io)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring, list of websites with changes"  title="Self-hosted web page change monitoring, list of websites with changes"  />](https://changedetection.io)
 
 
 [**Don't have time? Let us host it for you! try our extremely affordable subscription use our proxies and support!**](https://changedetection.io) 
 
 
-#### Example use cases
+### Target specific parts of the webpage using the Visual Selector tool.
+
+Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Select parts and elements of a web page to monitor for changes"  title="Select parts and elements of a web page to monitor for changes" />](https://changedetection.io?src=pip)
+
+### Easily see what changed, examine by word, line, or individual character.
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://changedetection.io?src=pip)
+
+
+### Perform interactive browser steps
+
+Fill in text boxes, click buttons and more, setup your changedetection scenario. 
+
+Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
+
+[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more"  title="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more" />](https://changedetection.io?src=pip)
+
+After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
+Requires Playwright to be enabled.
+
+
+### Example use cases
 
 - Products and services have a change in pricing
 - _Out of stock notification_ and _Back In stock notification_
+- Monitor and track PDF file changes, know when a PDF file has text changes.
 - Governmental department updates (changes are often only on their websites)
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
+- Discogs restock alerts and monitoring
 - Realestate listing changes
 - Know when your favourite whiskey is on sale, or other special deals are announced before anyone else
 - COVID related news from government websites
@@ -27,18 +52,34 @@ Live your data-life pro-actively, track website content changes and receive noti
 - Create RSS feeds based on changes in web content
 - Monitor HTML source code for unexpected changes, strengthen your PCI compliance
 - You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
+- Get notified when certain keywords appear in Twitter search results
+- Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
+- Get alerts when new job positions are open on Bamboo HR and other job platforms
+- Website defacement monitoring
+- Pokémon Card Restock Tracker / Pokémon TCG Tracker
+- RegTech - stay ahead of regulatory changes, regulatory compliance
 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 
 #### Key Features
 
 - Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
-- Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
+- Target elements with xPath(1.0) and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
 - Switch between fast non-JS and Chrome JS based "fetchers"
+- Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)
 - Easily specify how often a site should be checked
 - Execute JS before extracting text (Good for logging in, see examples in the UI!)
 - Override Request Headers, Specify `POST` or `GET` and other methods
 - Use the "Visual Selector" to help target specific elements
+- Configurable [proxy per watch](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration)
+- Send a screenshot with the notification when a change is detected in the web page
+
+We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.
+
+[Oxylabs](https://oxylabs.go2cloud.org/SH2d) is also an excellent proxy provider and well worth using, they offer Residental, ISP, Rotating and many other proxy types to suit your project. 
+
+Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
+
 
 
 ```bash

README.md

@@ -5,7 +5,7 @@
 _Live your data-life pro-actively._ 
 
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://changedetection.io?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web site page change monitoring"  title="Self-hosted web site page change monitoring"  />](https://changedetection.io?src=github)
 
 [![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)
 
@@ -22,7 +22,7 @@ _Live your data-life pro-actively._
 
 Available when connected to a <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Playwright-content-fetcher">playwright content fetcher</a> (included as part of our subscription service)
 
-[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />](https://changedetection.io?src=github)
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Select parts and elements of a web page to monitor for changes"  title="Select parts and elements of a web page to monitor for changes" />](https://changedetection.io?src=github)
 
 ### Easily see what changed, examine by word, line, or individual character.
 
@@ -35,7 +35,7 @@ Fill in text boxes, click buttons and more, setup your changedetection scenario.
 
 Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.
 
-[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Website change detection with interactive browser steps, login, cookies etc" />](https://changedetection.io?src=github)
+[<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more"  title="Website change detection with interactive browser steps, detect changes behind login and password, search queries and more" />](https://changedetection.io?src=github)
 
 After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
 Requires Playwright to be enabled.
@@ -226,7 +226,7 @@ The application also supports notifying you that it can follow this information
 
 ## Proxy Configuration
 
-See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [Bright Data proxy services where possible]( https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support)
+See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [Bright Data proxy services where possible](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support) and [Oxylabs](https://oxylabs.go2cloud.org/SH2d) proxy services.
 
 ## Raspberry Pi support?
 

changedetectionio/__init__.py

@@ -38,7 +38,9 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.45.1'
+__version__ = '0.45.3'
+
+from changedetectionio.store import BASE_URL_NOT_SET_TEXT
 
 datastore = None
 
@@ -184,7 +186,6 @@ class User(flask_login.UserMixin):
 
     pass
 
-
 def login_optionally_required(func):
     @wraps(func)
     def decorated_view(*args, **kwargs):
@@ -197,7 +198,6 @@ def login_optionally_required(func):
         # Permitted
         elif request.endpoint == 'diff_history_page' and datastore.data['settings']['application'].get('shared_diff_access'):
             return func(*args, **kwargs)
-
         elif request.method in flask_login.config.EXEMPT_METHODS:
             return func(*args, **kwargs)
         elif app.config.get('LOGIN_DISABLED'):
@@ -356,12 +356,10 @@ def changedetection_app(config=None, datastore_o=None):
                 # Include a link to the diff page, they will have to login here to see if password protection is enabled.
                 # Description is the page you watch, link takes you to the diff JS UI page
                 # Dict val base_url will get overriden with the env var if it is set.
-                ext_base_url = datastore.data['settings']['application'].get('base_url')
-                if ext_base_url:
-                    # Go with overriden value
-                    diff_link = {'href': "{}{}".format(ext_base_url, url_for('diff_history_page', uuid=watch['uuid'], _external=False))}
-                else:
-                    diff_link = {'href': url_for('diff_history_page', uuid=watch['uuid'], _external=True)}
+                ext_base_url = datastore.data['settings']['application'].get('active_base_url')
+
+                # Because we are called via whatever web server, flask should figure out the right path (
+                diff_link = {'href': url_for('diff_history_page', uuid=watch['uuid'], _external=True)}
 
                 fe.link(link=diff_link)
 
@@ -714,7 +712,6 @@ def changedetection_app(config=None, datastore_o=None):
             output = render_template("edit.html",
                                      available_processors=processors.available_processors(),
                                      browser_steps_config=browser_step_ui_config,
-                                     current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
                                      form=form,
                                      has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
@@ -804,7 +801,6 @@ def changedetection_app(config=None, datastore_o=None):
 
         output = render_template("settings.html",
                                  form=form,
-                                 current_base_url = datastore.data['settings']['application']['base_url'],
                                  hide_remove_pass=os.getenv("SALTED_PASS", False),
                                  api_key=datastore.data['settings']['application'].get('api_access_token'),
                                  emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
@@ -1433,6 +1429,7 @@ def changedetection_app(config=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/highlight_submit_ignore_url", methods=['POST'])
+    @login_optionally_required
     def highlight_submit_ignore_url():
         import re
         mode = request.form.get('mode')

changedetectionio/blueprint/check_proxies/__init__.py

@@ -57,9 +57,11 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 status.update({'status': 'ERROR OTHER', 'length': len(contents), 'text': f"Got empty reply with code {e.status_code} - Access denied"})
             else:
                 status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': f"Empty reply with code {e.status_code}, needs chrome?"})
-
+        except content_fetcher.ReplyWithContentButNoText as e:
+            txt = f"Got reply but with no content - Status code {e.status_code} - It's possible that the filters were found, but contained no usable text (or contained only an image)."
+            status.update({'status': 'ERROR', 'text': txt})
         except Exception as e:
-            status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': 'Error: '+str(e)})
+            status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': 'Error: '+type(e).__name__+str(e)})
         else:
             status.update({'status': 'OK', 'length': len(contents), 'text': ''})
 

changedetectionio/content_fetcher.py

@@ -1,12 +1,15 @@
-import hashlib
 from abc import abstractmethod
+from distutils.util import strtobool
+from urllib.parse import urlparse
 import chardet
+import hashlib
 import json
 import logging
 import os
 import requests
 import sys
 import time
+import urllib.parse
 
 visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary'
 
@@ -77,11 +80,13 @@ class ScreenshotUnavailable(Exception):
 
 
 class ReplyWithContentButNoText(Exception):
-    def __init__(self, status_code, url, screenshot=None):
+    def __init__(self, status_code, url, screenshot=None, has_filters=False, html_content=''):
         # Set this so we can use it in other parts of the app
         self.status_code = status_code
         self.url = url
         self.screenshot = screenshot
+        self.has_filters = has_filters
+        self.html_content = html_content
         return
 
 
@@ -264,7 +269,6 @@ class base_html_playwright(Fetcher):
 
         if self.proxy:
             # Playwright needs separate username and password values
-            from urllib.parse import urlparse
             parsed = urlparse(self.proxy.get('server'))
             if parsed.username:
                 self.proxy['username'] = parsed.username
@@ -319,14 +323,13 @@ class base_html_playwright(Fetcher):
 
         # Append proxy connect string
         if self.proxy:
-            import urllib.parse
             # Remove username/password if it exists in the URL or you will receive "ERR_NO_SUPPORTED_PROXIES" error
             # Actual authentication handled by Puppeteer/node
             o = urlparse(self.proxy.get('server'))
-            proxy_url = urllib.parse.quote(o._replace(netloc="{}:{}".format(o.hostname, o.port)).geturl())
+            # Remove scheme, socks5:// doesnt always work and it will autodetect anyway
+            proxy_url = urllib.parse.quote(o._replace(netloc="{}:{}".format(o.hostname, o.port)).geturl().replace(f"{o.scheme}://", '', 1))
             browserless_function_url = f"{browserless_function_url}&--proxy-server={proxy_url}&dumpio=true"
 
-
         try:
             amp = '&' if '?' in browserless_function_url else '?'
             response = requests.request(
@@ -343,9 +346,9 @@ class base_html_playwright(Fetcher):
                         'req_headers': request_headers,
                         'screenshot_quality': int(os.getenv("PLAYWRIGHT_SCREENSHOT_QUALITY", 72)),
                         'url': url,
-                        'user_agent': request_headers.get('User-Agent', 'Mozilla/5.0'),
-                        'proxy_username': self.proxy.get('username','') if self.proxy else False,
-                        'proxy_password': self.proxy.get('password', '') if self.proxy else False,
+                        'user_agent': {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None),
+                        'proxy_username': self.proxy.get('username', '') if self.proxy else False,
+                        'proxy_password': self.proxy.get('password', '') if self.proxy and self.proxy.get('username') else False,
                         'no_cache_list': [
                             'twitter',
                             '.pdf'
@@ -414,8 +417,8 @@ class base_html_playwright(Fetcher):
                 lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
                 self.browser_steps))
 
-        if not has_browser_steps:
-            if os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
+        if not has_browser_steps and os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH'):
+            if strtobool(os.getenv('USE_EXPERIMENTAL_PUPPETEER_FETCH')):
                 # Temporary backup solution until we rewrite the playwright code
                 return self.run_fetch_browserless_puppeteer(
                     url,
@@ -432,6 +435,7 @@ class base_html_playwright(Fetcher):
 
         self.delete_browser_steps_screenshots()
         response = None
+
         with sync_playwright() as p:
             browser_type = getattr(p, self.browser_type)
 
@@ -440,10 +444,13 @@ class base_html_playwright(Fetcher):
             # 60,000 connection timeout only
             browser = browser_type.connect_over_cdp(self.command_executor, timeout=60000)
 
+            # SOCKS5 with authentication is not supported (yet)
+            # https://github.com/microsoft/playwright/issues/10567
+
             # Set user agent to prevent Cloudflare from blocking the browser
             # Use the default one configured in the App.py model that's passed from fetch_site_status.py
             context = browser.new_context(
-                user_agent=request_headers.get('User-Agent', 'Mozilla/5.0'),
+                user_agent={k.lower(): v for k, v in request_headers.items()}.get('user-agent', None),
                 proxy=self.proxy,
                 # This is needed to enable JavaScript execution on GitHub and others
                 bypass_csp=True,
@@ -476,7 +483,6 @@ class base_html_playwright(Fetcher):
                 print("Content Fetcher > retrying request got error - ", str(e))
                 time.sleep(1)
                 response = self.page.goto(url, wait_until='commit')
-
             except Exception as e:
                 print("Content Fetcher > Other exception when page.goto", str(e))
                 context.close()
@@ -612,7 +618,6 @@ class base_html_webdriver(Fetcher):
         from selenium.common.exceptions import WebDriverException
         # request_body, request_method unused for now, until some magic in the future happens.
 
-        # check env for WEBDRIVER_URL
         self.driver = webdriver.Remote(
             command_executor=self.command_executor,
             desired_capabilities=DesiredCapabilities.CHROME,
@@ -684,13 +689,17 @@ class html_requests(Fetcher):
             is_binary=False):
 
         # Make requests use a more modern looking user-agent
-        if not 'User-Agent' in request_headers:
+        if not {k.lower(): v for k, v in request_headers.items()}.get('user-agent', None):
             request_headers['User-Agent'] = os.getenv("DEFAULT_SETTINGS_HEADERS_USERAGENT",
                                                       'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36')
 
         proxies = {}
 
         # Allows override the proxy on a per-request basis
+
+        # https://requests.readthedocs.io/en/latest/user/advanced/#socks
+        # Should also work with `socks5://user:pass@host:port` type syntax.
+
         if self.proxy_override:
             proxies = {'http': self.proxy_override, 'https': self.proxy_override, 'ftp': self.proxy_override}
         else:

changedetectionio/forms.py

@@ -229,16 +229,19 @@ class ValidateJinja2Template(object):
     def __call__(self, form, field):
         from changedetectionio import notification
 
-        from jinja2 import Environment, BaseLoader, TemplateSyntaxError
+        from jinja2 import Environment, BaseLoader, TemplateSyntaxError, UndefinedError
         from jinja2.meta import find_undeclared_variables
 
 
         try:
             jinja2_env = Environment(loader=BaseLoader)
             jinja2_env.globals.update(notification.valid_tokens)
+
             rendered = jinja2_env.from_string(field.data).render()
         except TemplateSyntaxError as e:
             raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
+        except UndefinedError as e:
+            raise ValidationError(f"A variable or function is not defined: {e}") from e
 
         ast = jinja2_env.parse(field.data)
         undefined = ", ".join(find_undeclared_variables(ast))
@@ -478,7 +481,7 @@ class SingleExtraProxy(Form):
 
     # maybe better to set some <script>var..
     proxy_name = StringField('Name', [validators.Optional()], render_kw={"placeholder": "Name"})
-    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "http://user:pass@...:3128", "size":50})
+    proxy_url = StringField('Proxy URL', [validators.Optional()], render_kw={"placeholder": "socks5:// or regular proxy http://user:pass@...:3128", "size":50})
     # @todo do the validation here instead
 
 # datastore.data['settings']['requests']..
@@ -502,7 +505,10 @@ class globalSettingsRequestForm(Form):
 class globalSettingsApplicationForm(commonSettingsForm):
 
     api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
-    base_url = StringField('Base URL', validators=[validators.Optional()])
+    base_url = StringField('Notification base URL override',
+                           validators=[validators.Optional()],
+                           render_kw={"placeholder": os.getenv('BASE_URL', 'Not set')}
+                           )
     empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
     fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
     global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])

changedetectionio/html_tools.py

@@ -7,17 +7,35 @@ from typing import List
 import json
 import re
 
+
 # HTML added to be sure each result matching a filter (.example) gets converted to a new line by Inscriptis
 TEXT_FILTER_LIST_LINE_SUFFIX = "<br>"
 
+PERL_STYLE_REGEX = r'^/(.*?)/([a-z]*)?$'
 # 'price' , 'lowPrice', 'highPrice' are usually under here
-# all of those may or may not appear on different websites
-LD_JSON_PRODUCT_OFFER_SELECTOR = "json:$..offers"
+# All of those may or may not appear on different websites - I didnt find a way todo case-insensitive searching here
+LD_JSON_PRODUCT_OFFER_SELECTORS = ["json:$..offers", "json:$..Offers"]
 
 class JSONNotFound(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
-        
+
+
+# Doesn't look like python supports forward slash auto enclosure in re.findall
+# So convert it to inline flag "(?i)foobar" type configuration
+def perl_style_slash_enclosed_regex_to_options(regex):
+
+    res = re.search(PERL_STYLE_REGEX, regex, re.IGNORECASE)
+
+    if res:
+        flags = res.group(2) if res.group(2) else 'i'
+        regex = f"(?{flags}){res.group(1)}"
+    else:
+        # Fall back to just ignorecase as an option
+        regex = f"(?i){regex}"
+
+    return regex
+
 # Given a CSS Rule, and a blob of HTML, return the blob of HTML that matches
 def include_filters(include_filters, html_content, append_pretty_line_formatting=False):
     soup = BeautifulSoup(html_content, "html.parser")
@@ -144,7 +162,6 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 
         # Foreach <script json></script> blob.. just return the first that matches json_filter
         # As a last resort, try to parse the whole <body>
-        s = []
         soup = BeautifulSoup(content, 'html.parser')
 
         if ensure_is_ldjson_info_type:
@@ -170,13 +187,24 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
         
         for json_data in bs_jsons:
             stripped_text_from_html = _parse_json(json_data, json_filter)
+
             if ensure_is_ldjson_info_type:
                 # Could sometimes be list, string or something else random
                 if isinstance(json_data, dict):
                     # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search
                     # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)
-                    if json_data.get('@type', False) and json_data.get('@type','').lower() == ensure_is_ldjson_info_type.lower() and stripped_text_from_html:
-                        break
+                    # @type could also be a list (Product, SubType)
+                    # LD_JSON auto-extract also requires some content PLUS the ldjson to be present
+                    # 1833 - could be either str or dict, should not be anything else
+                    if json_data.get('@type') and stripped_text_from_html:
+                        try:
+                            if json_data.get('@type') == str or json_data.get('@type') == dict:
+                                types = [json_data.get('@type')] if isinstance(json_data.get('@type'), str) else json_data.get('@type')
+                                if ensure_is_ldjson_info_type.lower() in [x.lower().strip() for x in types]:
+                                    break
+                        except:
+                            continue
+
             elif stripped_text_from_html:
                 break
 
@@ -195,23 +223,14 @@ def strip_ignore_text(content, wordlist, mode="content"):
     output = []
     ignore_text = []
     ignore_regex = []
-
     ignored_line_numbers = []
 
     for k in wordlist:
         # Is it a regex?
-        x = re.search('^\/(.*)\/(.*)', k.strip())
-        if x:
-            # Starts with / but doesn't look like a regex
-            p = x.group(1)
-            try:
-                # @Todo python regex options can go before the regex str, but not really many of the options apply on a per-line basis
-                ignore_regex.append(re.compile(rf"{p}", re.IGNORECASE))
-            except Exception as e:
-                # Badly formed regex, treat as text
-                ignore_text.append(k.strip())
+        res = re.search(PERL_STYLE_REGEX, k, re.IGNORECASE)
+        if res:
+            ignore_regex.append(re.compile(perl_style_slash_enclosed_regex_to_options(k)))
         else:
-            # Had a / but doesn't work as regex
             ignore_text.append(k.strip())
 
     for line in content.splitlines():
@@ -275,9 +294,18 @@ def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
 
 # Does LD+JSON exist with a @type=='product' and a .price set anywhere?
 def has_ldjson_product_info(content):
+    pricing_data = ''
+
     try:
-        pricing_data = extract_json_as_string(content=content, json_filter=LD_JSON_PRODUCT_OFFER_SELECTOR, ensure_is_ldjson_info_type="product")
-    except JSONNotFound as e:
+        if not 'application/ld+json' in content:
+            return False
+
+        for filter in LD_JSON_PRODUCT_OFFER_SELECTORS:
+            pricing_data += extract_json_as_string(content=content,
+                                                  json_filter=filter,
+                                                  ensure_is_ldjson_info_type="product")
+
+    except Exception as e:
         # Totally fine
         return False
     x=bool(pricing_data)

changedetectionio/notification.py

@@ -208,15 +208,11 @@ def create_notification_parameters(n_object, datastore):
         watch_tag = ''
 
     # Create URLs to customise the notification with
-    base_url = datastore.data['settings']['application']['base_url']
+    # active_base_url - set in store.py data property
+    base_url = datastore.data['settings']['application'].get('active_base_url')
 
     watch_url = n_object['watch_url']
 
-    # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services
-    #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.
-    if base_url == '':
-        base_url = "<base-url-env-var-not-set>"
-
     diff_url = "{}/diff/{}".format(base_url, uuid)
     preview_url = "{}/preview/{}".format(base_url, uuid)
 
@@ -226,7 +222,7 @@ def create_notification_parameters(n_object, datastore):
     # Valid_tokens also used as a field validator
     tokens.update(
         {
-            'base_url': base_url if base_url is not None else '',
+            'base_url': base_url,
             'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '',
             'diff': n_object.get('diff', ''),  # Null default in the case we use a test
             'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test

changedetectionio/processors/text_json_diff.py

@@ -11,17 +11,19 @@ from changedetectionio import content_fetcher, html_tools
 from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
 from copy import deepcopy
 from . import difference_detection_processor
+from ..html_tools import PERL_STYLE_REGEX
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
-
-name =  'Webpage Text/HTML, JSON and PDF changes'
+name = 'Webpage Text/HTML, JSON and PDF changes'
 description = 'Detects all text changes where possible'
+json_filter_prefixes = ['json:', 'jq:']
 
 class FilterNotFoundInResponse(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
 
+
 class PDFToHTMLToolNotFound(ValueError):
     def __init__(self, msg):
         ValueError.__init__(self, msg)
@@ -37,19 +39,6 @@ class perform_site_check(difference_detection_processor):
         super().__init__(*args, **kwargs)
         self.datastore = datastore
 
-    # Doesn't look like python supports forward slash auto enclosure in re.findall
-    # So convert it to inline flag "foobar(?i)" type configuration
-    def forward_slash_enclosed_regex_to_options(self, regex):
-        res = re.search(r'^/(.*?)/(\w+)$', regex, re.IGNORECASE)
-
-        if res:
-            regex = res.group(1)
-            regex += '(?{})'.format(res.group(2))
-        else:
-            regex += '(?{})'.format('i')
-
-        return regex
-
     def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
         changed_detected = False
         screenshot = False  # as bytes
@@ -135,7 +124,8 @@ class perform_site_check(difference_detection_processor):
         # requests for PDF's, images etc should be passwd the is_binary flag
         is_binary = watch.is_pdf
 
-        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'), is_binary=is_binary)
+        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_codes, watch.get('include_filters'),
+                    is_binary=is_binary)
         fetcher.quit()
 
         self.screenshot = fetcher.screenshot
@@ -151,7 +141,6 @@ class perform_site_check(difference_detection_processor):
             if update_obj['previous_md5_before_filters'] == watch.get('previous_md5_before_filters'):
                 raise content_fetcher.checksumFromPreviousCheckWasTheSame()
 
-
         # Fetching complete, now filters
         # @todo move to class / maybe inside of fetcher abstract base?
 
@@ -207,7 +196,7 @@ class perform_site_check(difference_detection_processor):
 
         # Inject a virtual LD+JSON price tracker rule
         if watch.get('track_ldjson_price_data', '') == PRICE_DATA_TRACK_ACCEPT:
-            include_filters_rule.append(html_tools.LD_JSON_PRODUCT_OFFER_SELECTOR)
+            include_filters_rule += html_tools.LD_JSON_PRODUCT_OFFER_SELECTORS
 
         has_filter_rule = len(include_filters_rule) and len(include_filters_rule[0].strip())
         has_subtractive_selectors = len(subtractive_selectors) and len(subtractive_selectors[0].strip())
@@ -225,14 +214,11 @@ class perform_site_check(difference_detection_processor):
                 pass
 
         if has_filter_rule:
-            json_filter_prefixes = ['json:', 'jq:']
             for filter in include_filters_rule:
                 if any(prefix in filter for prefix in json_filter_prefixes):
                     stripped_text_from_html += html_tools.extract_json_as_string(content=fetcher.content, json_filter=filter)
                     is_html = False
 
-
-
         if is_html or is_source:
 
             # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
@@ -283,7 +269,6 @@ class perform_site_check(difference_detection_processor):
         # Re #340 - return the content before the 'ignore text' was applied
         text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
-
         # @todo whitespace coming from missing rtrim()?
         # stripped_text_from_html could be based on their preferences, replace the processed text with only that which they want to know about.
         # Rewrite's the processing text based on only what diff result they want to see
@@ -293,13 +278,13 @@ class perform_site_check(difference_detection_processor):
             # needs to not include (added) etc or it may get used twice
             # Replace the processed text with the preferred result
             rendered_diff = diff.render_diff(previous_version_file_contents=watch.get_last_fetched_before_filters(),
-                                                       newest_version_file_contents=stripped_text_from_html,
-                                                       include_equal=False,  # not the same lines
-                                                       include_added=watch.get('filter_text_added', True),
-                                                       include_removed=watch.get('filter_text_removed', True),
-                                                       include_replaced=watch.get('filter_text_replaced', True),
-                                                       line_feed_sep="\n",
-                                                       include_change_type_prefix=False)
+                                             newest_version_file_contents=stripped_text_from_html,
+                                             include_equal=False,  # not the same lines
+                                             include_added=watch.get('filter_text_added', True),
+                                             include_removed=watch.get('filter_text_removed', True),
+                                             include_replaced=watch.get('filter_text_replaced', True),
+                                             line_feed_sep="\n",
+                                             include_change_type_prefix=False)
 
             watch.save_last_fetched_before_filters(text_content_before_ignored_filter)
 
@@ -314,7 +299,12 @@ class perform_site_check(difference_detection_processor):
         # Treat pages with no renderable text content as a change? No by default
         empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)
         if not is_json and not empty_pages_are_a_change and len(stripped_text_from_html.strip()) == 0:
-            raise content_fetcher.ReplyWithContentButNoText(url=url, status_code=fetcher.get_last_status_code(), screenshot=screenshot)
+            raise content_fetcher.ReplyWithContentButNoText(url=url,
+                                                            status_code=fetcher.get_last_status_code(),
+                                                            screenshot=screenshot,
+                                                            has_filters=has_filter_rule,
+                                                            html_content=html_content
+                                                            )
 
         # We rely on the actual text in the html output.. many sites have random script vars etc,
         # in the future we'll implement other mechanisms.
@@ -335,16 +325,25 @@ class perform_site_check(difference_detection_processor):
             regex_matched_output = []
             for s_re in extract_text:
                 # incase they specified something in '/.../x'
-                regex = self.forward_slash_enclosed_regex_to_options(s_re)
-                result = re.findall(regex.encode('utf-8'), stripped_text_from_html)
+                if re.search(PERL_STYLE_REGEX, s_re, re.IGNORECASE):
+                    regex = html_tools.perl_style_slash_enclosed_regex_to_options(s_re)
+                    result = re.findall(regex.encode('utf-8'), stripped_text_from_html)
 
-                for l in result:
-                    if type(l) is tuple:
-                        # @todo - some formatter option default (between groups)
-                        regex_matched_output += list(l) + [b'\n']
-                    else:
-                        # @todo - some formatter option default (between each ungrouped result)
-                        regex_matched_output += [l] + [b'\n']
+                    for l in result:
+                        if type(l) is tuple:
+                            # @todo - some formatter option default (between groups)
+                            regex_matched_output += list(l) + [b'\n']
+                        else:
+                            # @todo - some formatter option default (between each ungrouped result)
+                            regex_matched_output += [l] + [b'\n']
+                else:
+                    # Doesnt look like regex, just hunt for plaintext and return that which matches
+                    # `stripped_text_from_html` will be bytes, so we must encode s_re also to bytes
+                    r = re.compile(re.escape(s_re.encode('utf-8')), re.IGNORECASE)
+                    res = r.findall(stripped_text_from_html)
+                    if res:
+                        for match in res:
+                            regex_matched_output += [match] + [b'\n']
 
             # Now we will only show what the regex matched
             stripped_text_from_html = b''

changedetectionio/res/puppeteer_fetch.js

@@ -18,12 +18,19 @@ module.exports = async ({page, context}) => {
 
     await page.setBypassCSP(true)
     await page.setExtraHTTPHeaders(req_headers);
-    await page.setUserAgent(user_agent);
+
+    if (user_agent) {
+        await page.setUserAgent(user_agent);
+    }
     // https://ourcodeworld.com/articles/read/1106/how-to-solve-puppeteer-timeouterror-navigation-timeout-of-30000-ms-exceeded
 
     await page.setDefaultNavigationTimeout(0);
 
     if (proxy_username) {
+        // Setting Proxy-Authentication header is deprecated, and doing so can trigger header change errors from Puppeteer
+        // https://github.com/puppeteer/puppeteer/issues/676 ?
+        // https://help.brightdata.com/hc/en-us/articles/12632549957649-Proxy-Manager-How-to-Guides#h_01HAKWR4Q0AFS8RZTNYWRDFJC2
+        // https://cri.dev/posts/2020-03-30-How-to-solve-Puppeteer-Chrome-Error-ERR_INVALID_ARGUMENT/
         await page.authenticate({
             username: proxy_username,
             password: proxy_password

changedetectionio/res/stock-not-in-stock.js

@@ -5,14 +5,19 @@ function isItemInStock() {
     'agotado',
     'artikel zurzeit vergriffen',
     'as soon as stock is available',
+    'ausverkauft', // sold out
     'available for back order',
+    'back-order or out of stock',
     'backordered',
+    'benachrichtigt mich', // notify me
     'brak na stanie',
     'brak w magazynie',
     'coming soon',
     'currently have any tickets for this',
     'currently unavailable',
+    'dostępne wkrótce',
     'en rupture de stock',
+    'ist derzeit nicht auf lager',
     'item is no longer available',
     'message if back in stock',
     'nachricht bei',
@@ -37,6 +42,7 @@ function isItemInStock() {
     'unavailable tickets',
     'we do not currently have an estimate of when this product will be back in stock.',
     'zur zeit nicht an lager',
+    '已售完',
   ];
 
 

changedetectionio/run_proxy_tests.sh

@@ -10,6 +10,40 @@ set -x
 docker run --network changedet-network -d --name squid-one --hostname squid-one --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
 docker run --network changedet-network -d --name squid-two --hostname squid-two --rm -v `pwd`/tests/proxy_list/squid.conf:/etc/squid/conf.d/debian.conf ubuntu/squid:4.13-21.10_edge
 
+# SOCKS5 related - start simple Socks5 proxy server
+# SOCKSTEST=xyz should show in the logs of this service to confirm it fetched
+docker run --network changedet-network -d --hostname socks5proxy --name socks5proxy -p 1080:1080 -e PROXY_USER=proxy_user123 -e PROXY_PASSWORD=proxy_pass123 serjs/go-socks5-proxy
+docker run --network changedet-network -d --hostname socks5proxy-noauth -p 1081:1080 --name socks5proxy-noauth  serjs/go-socks5-proxy
+
+echo "---------------------------------- SOCKS5 -------------------"
+# SOCKS5 related - test from proxies.json
+docker run --network changedet-network \
+  -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
+  --rm \
+  -e "SOCKSTEST=proxiesjson" \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py'
+
+# SOCKS5 related - by manually entering in UI
+docker run --network changedet-network \
+  --rm \
+  -e "SOCKSTEST=manual" \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy.py'
+
+# SOCKS5 related - test from proxies.json via playwright - NOTE- PLAYWRIGHT DOESNT SUPPORT AUTHENTICATING PROXY
+docker run --network changedet-network \
+  -e "SOCKSTEST=manual-playwright" \
+  -v `pwd`/tests/proxy_socks5/proxies.json-example-noauth:/app/changedetectionio/test-datastore/proxies.json \
+  -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" \
+  --rm \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_socks5/test_socks5_proxy_sources.py'
+
+echo "socks5 server logs"
+docker logs socks5proxy
+echo "----------------------------------"
+
 # Used for configuring a custom proxy URL via the UI
 docker run --network changedet-network -d \
   --name squid-custom \

changedetectionio/static/js/browser-steps.js

@@ -208,7 +208,7 @@ $(document).ready(function () {
             console.log(x);
             if (x && first_available.length) {
                 // @todo will it let you click shit that has a layer ontop? probably not.
-                if (x['tagtype'] === 'text' || x['tagtype'] === 'email' || x['tagName'] === 'textarea' || x['tagtype'] === 'password' || x['tagtype'] === 'search') {
+                if (x['tagtype'] === 'text' || x['tagtype'] === 'number' || x['tagtype'] === 'email' || x['tagName'] === 'textarea' || x['tagtype'] === 'password' || x['tagtype'] === 'search') {
                     $('select', first_available).val('Enter text in field').change();
                     $('input[type=text]', first_available).first().val(x['xpath']);
                     $('input[placeholder="Value"]', first_available).addClass('ok').click().focus();

changedetectionio/static/js/global-settings.js

@@ -32,5 +32,10 @@ $(document).ready(function () {
         window.getSelection().removeAllRanges();
 
     });
+
+    $("#notification-token-toggle").click(function (e) {
+        e.preventDefault();
+        $('#notification-tokens-info').toggle();
+    });
 });
 

changedetectionio/static/js/watch-settings.js

@@ -42,4 +42,8 @@ $(document).ready(function () {
         $('#notification_urls').val('');
         e.preventDefault();
     });
+    $("#notification-token-toggle").click(function (e) {
+        e.preventDefault();
+        $('#notification-tokens-info').toggle();
+    });
 });

changedetectionio/static/styles/scss/parts/_browser-steps.scss

@@ -44,7 +44,7 @@
 #browser-steps .flex-wrapper {
   display: flex;
   flex-flow: row;
-  height: 600px; /*@todo make this dynamic */
+  height: 70vh;
 }
 
 /*  this is duplicate :( */

changedetectionio/static/styles/styles.css

@@ -50,8 +50,7 @@
 #browser-steps .flex-wrapper {
   display: flex;
   flex-flow: row;
-  height: 600px;
-  /*@todo make this dynamic */ }
+  height: 70vh; }
 
 /*  this is duplicate :( */
 #browsersteps-selector-wrapper {

changedetectionio/store.py

@@ -18,6 +18,9 @@ import threading
 import time
 import uuid as uuid_builder
 
+# Because the server will run as a daemon and wont know the URL for notification links when firing off a notification
+BASE_URL_NOT_SET_TEXT = '("Base URL" not set - see settings - notifications)'
+
 dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])
 
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
@@ -39,6 +42,7 @@ class ChangeDetectionStore:
         self.__data = App.model()
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
+        print(">>> Datastore path is ", self.json_store_path)
         self.needs_write = False
         self.start_time = time.time()
         self.stop_thread = False
@@ -175,12 +179,21 @@ class ChangeDetectionStore:
 
     @property
     def data(self):
-        # Re #152, Return env base_url if not overriden, @todo also prefer the proxy pass url
-        env_base_url = os.getenv('BASE_URL','')
-        if not self.__data['settings']['application']['base_url']:
-          self.__data['settings']['application']['base_url'] = env_base_url.strip('" ')
+        # Re #152, Return env base_url if not overriden
+        # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services
+        #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.
 
-        return self.__data
+        active_base_url = BASE_URL_NOT_SET_TEXT
+        if self.__data['settings']['application'].get('base_url'):
+            active_base_url = self.__data['settings']['application'].get('base_url')
+        elif os.getenv('BASE_URL'):
+            active_base_url = os.getenv('BASE_URL')
+
+        # I looked at various ways todo the following, but in the end just copying the dict seemed simplest/most reliable
+        # even given the memory tradeoff - if you know a better way.. maybe return d|self.__data.. or something
+        d = self.__data
+        d['settings']['application']['active_base_url'] = active_base_url.strip('" ')
+        return d
 
     # Delete a single watch by UUID
     def delete(self, uuid):

changedetectionio/templates/_common_fields.jinja

@@ -13,7 +13,7 @@
                             <div class="pure-form-message-inline">
                               <ul>
                                 <li>Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.</li>
-                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) </code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
                                 <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
                                 <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
@@ -35,18 +35,14 @@
                             </div>
                             <div class="pure-control-group">
                                 {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }}
-                                <span class="pure-form-message-inline">Body for all notifications</span>
-                            </div>
-                            <div class="pure-control-group">
-                            <!-- unsure -->
-                                {{ render_field(form.notification_format , class="notification-format") }}
-                                <span class="pure-form-message-inline">Format for all notifications</span>
+                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
+                                </span>
+
                             </div>
                             <div class="pure-controls">
-                            <p class="pure-form-message-inline">
-                                You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL.
-                            </p>
-
+                                <div id="notification-token-toggle" class="pure-button button-tag button-xsmall">Show token/placeholders</div>
+                            </div>
+                            <div class="pure-controls" style="display: none;" id="notification-tokens-info">
                                 <table class="pure-table" id="token-table">
                                     <thead>
                                     <tr>
@@ -105,7 +101,7 @@
                                     </tr>
                                     <tr>
                                         <td><code>{{ '{{current_snapshot}}' }}</code></td>
-                                        <td>The current snapshot value, useful when combined with JSON or CSS filters
+                                        <td>The current snapshot text contents value, useful when combined with JSON or CSS filters
                                         </td>
                                     </tr>
                                     <tr>
@@ -115,12 +111,15 @@
                                     </tbody>
                                 </table>
                                 <div class="pure-form-message-inline">
-                                    <br>
-                                    URLs generated by changedetection.io (such as <code>{{ '{{diff_url}}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br>
-                                    Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
-									<br>
-									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
+                                    <p>
+									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
+                                    For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
+                                    </p>
                                 </div>
                             </div>
+                            <div class="pure-control-group">
+                                {{ render_field(form.notification_format , class="notification-format") }}
+                                <span class="pure-form-message-inline">Format for all notifications</span>
+                            </div>
                         </div>
 {% endmacro %}

changedetectionio/templates/edit.html

@@ -378,15 +378,16 @@ Unavailable") }}
                         {{ render_field(form.extract_text, rows=5, placeholder="\d+ online") }}
                         <span class="pure-form-message-inline">
                     <ul>
-                        <li>Extracts text in the final output (line by line) after other filters using regular expressions;
+                        <li>Extracts text in the final output (line by line) after other filters using regular expressions or string match;
                             <ul>
                                 <li>Regular expression &dash; example <code>/reports.+?2022/i</code></li>
+                                <li>Don't forget to consider the white-space at the start of a line <code>/.+?reports.+?2022/i</code></li>
                                 <li>Use <code>//(?aiLmsux))</code> type flags (more <a href="https://docs.python.org/3/library/re.html#index-15">information here</a>)<br></li>
                                 <li>Keyword example &dash; example <code>Out of stock</code></li>
                                 <li>Use groups to extract just that text &dash; example <code>/reports.+?(\d+)/i</code> returns a list of years only</li>
                             </ul>
                         </li>
-                        <li>One line per regular-expression/ string match</li>
+                        <li>One line per regular-expression/string match</li>
                     </ul>
                         </span>
                     </div>

changedetectionio/templates/settings.html

@@ -62,14 +62,6 @@
                         <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page)
                         </span>
                     </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
-                        class="m-d") }}
-                        <span class="pure-form-message-inline">
-                            Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notifications and RSS links.<br>Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"),
-                            <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
-                        </span>
-                    </div>
                     <div class="pure-control-group">
                         {{ render_field(form.application.form.pager_size) }}
                         <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
@@ -100,6 +92,13 @@
                         {{ render_common_settings_form(form.application.form, emailprefix, settings_application) }}
                     </div>
                 </fieldset>
+                <div class="pure-control-group" id="notification-base-url">
+                    {{ render_field(form.application.form.base_url, class="m-d") }}
+                    <span class="pure-form-message-inline">
+                        Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notification links.<br>
+                        Default value is the system environment variable '<code>BASE_URL</code>' - <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
+                    </span>
+                </div>
             </div>
 
             <div class="tab-pane-inner" id="fetching">
@@ -230,7 +229,8 @@ nav
 
                 <div class="pure-control-group">
                 {{ render_field(form.requests.form.extra_proxies) }}
-                <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span>
+                <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span><br>
+                <span class="pure-form-message-inline">SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead</span>
                 </div>
             </div>
             <div id="actions">

changedetectionio/templates/watch-overview.html

@@ -119,6 +119,9 @@
                             <a href="{{ url_for('settings_page', uuid=watch.uuid) }}#proxies">Try adding external proxies/locations</a>
                         
                         {% endif %}
+                        {% if 'empty result or contain only an image' in watch.last_error %}
+                            <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Detecting-changes-in-images">more help here</a>.
+                        {% endif %}
                     </div>
                     {% endif %}
                     {% if watch.last_notification_error is defined and watch.last_notification_error != False %}

changedetectionio/tests/fetchers/test_content.py

@@ -28,8 +28,6 @@ def test_fetch_webdriver_content(client, live_server):
     )
 
     assert b"1 Imported" in res.data
-    time.sleep(3)
-
     wait_for_all_checks(client)
 
 

changedetectionio/tests/proxy_list/test_multiple_proxy.py

@@ -2,12 +2,11 @@
 
 import time
 from flask import url_for
-from ..util import live_server_setup
+from ..util import live_server_setup, wait_for_all_checks
+
 
 def test_preferred_proxy(client, live_server):
-    time.sleep(1)
     live_server_setup(live_server)
-    time.sleep(1)
     url = "http://chosen.changedetection.io"
 
     res = client.post(
@@ -20,7 +19,7 @@ def test_preferred_proxy(client, live_server):
 
     assert b"1 Imported" in res.data
 
-    time.sleep(2)
+    wait_for_all_checks(client)
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={
@@ -34,5 +33,5 @@ def test_preferred_proxy(client, live_server):
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    time.sleep(2)
+    wait_for_all_checks(client)
     # Now the request should appear in the second-squid logs

changedetectionio/tests/proxy_socks5/proxies.json-example

@@ -0,0 +1,6 @@
+{
+  "socks5proxy": {
+    "label": "socks5proxy",
+    "url": "socks5://proxy_user123:proxy_pass123@socks5proxy:1080"
+  }
+}

changedetectionio/tests/proxy_socks5/proxies.json-example-noauth

@@ -0,0 +1,6 @@
+{
+  "socks5proxy": {
+    "label": "socks5proxy",
+    "url": "socks5://socks5proxy-noauth:1080"
+  }
+}

changedetectionio/tests/proxy_socks5/test_socks5_proxy.py

@@ -0,0 +1,63 @@
+#!/usr/bin/python3
+import os
+import time
+from flask import url_for
+from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
+
+
+def test_socks5(client, live_server):
+    live_server_setup(live_server)
+
+    # Setup a proxy
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-ignore_whitespace": "y",
+            "application-fetch_backend": "html_requests",
+            # set in .github/workflows/test-only.yml
+            "requests-extra_proxies-0-proxy_url": "socks5://proxy_user123:proxy_pass123@socks5proxy:1080",
+            "requests-extra_proxies-0-proxy_name": "socks5proxy",
+        },
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    test_url = "https://changedetection.io/CHANGELOG.txt?socks-test-tag=" + os.getenv('SOCKSTEST', '')
+
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first", unpause_on_save=1),
+    )
+    # check the proxy is offered as expected
+    assert b'ui-0socks5proxy' in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid="first", unpause_on_save=1),
+        data={
+            "include_filters": "",
+            "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
+            "headers": "",
+            "proxy": "ui-0socks5proxy",
+            "tags": "",
+            "url": test_url,
+        },
+        follow_redirects=True
+    )
+    assert b"unpaused" in res.data
+    wait_for_all_checks(client)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should see the proper string
+    assert "+0200:".encode('utf-8') in res.data

changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py

@@ -0,0 +1,52 @@
+#!/usr/bin/python3
+import os
+import time
+from flask import url_for
+from changedetectionio.tests.util import live_server_setup, wait_for_all_checks
+
+
+# should be proxies.json mounted from run_proxy_tests.sh already
+# -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json
+def test_socks5_from_proxiesjson_file(client, live_server):
+    live_server_setup(live_server)
+
+    test_url = "https://changedetection.io/CHANGELOG.txt?socks-test-tag=" + os.getenv('SOCKSTEST', '')
+
+    res = client.get(url_for("settings_page"))
+    assert b'name="requests-proxy" type="radio" value="socks5proxy"' in res.data
+
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first", unpause_on_save=1),
+    )
+    # check the proxy is offered as expected
+    assert b'name="proxy" type="radio" value="socks5proxy"' in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid="first", unpause_on_save=1),
+        data={
+            "include_filters": "",
+            "fetch_backend": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',
+            "headers": "",
+            "proxy": "socks5proxy",
+            "tags": "",
+            "url": test_url,
+        },
+        follow_redirects=True
+    )
+    assert b"unpaused" in res.data
+    wait_for_all_checks(client)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should see the proper string
+    assert "+0200:".encode('utf-8') in res.data

changedetectionio/tests/test_automatic_follow_ldjson_price.py

@@ -2,7 +2,8 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup, extract_UUID_from_client, extract_api_key_from_UI
+from .util import live_server_setup, extract_UUID_from_client, extract_api_key_from_UI, wait_for_all_checks
+
 
 def set_response_with_ldjson():
     test_return_data = """<html>
@@ -27,7 +28,7 @@ def set_response_with_ldjson():
            "description":"You dont need it",
            "mpn":"111111",
            "sku":"22222",
-           "offers":{
+           "Offers":{
               "@type":"AggregateOffer",
               "lowPrice":8097000,
               "highPrice":8099900,
@@ -75,12 +76,11 @@ def set_response_without_ldjson():
         f.write(test_return_data)
     return None
 
-# actually only really used by the distll.io importer, but could be handy too
-def test_check_ldjson_price_autodetect(client, live_server):
+def test_setup(client, live_server):
     live_server_setup(live_server)
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
+# actually only really used by the distll.io importer, but could be handy too
+def test_check_ldjson_price_autodetect(client, live_server):
 
     set_response_with_ldjson()
 
@@ -92,7 +92,7 @@ def test_check_ldjson_price_autodetect(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Should get a notice that it's available
     res = client.get(url_for("index"))
@@ -102,11 +102,11 @@ def test_check_ldjson_price_autodetect(client, live_server):
     uuid = extract_UUID_from_client(client)
 
     client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))
-    time.sleep(2)
+    wait_for_all_checks(client)
 
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(2)
+    wait_for_all_checks(client)
     # Offer should be gone
     res = client.get(url_for("index"))
     assert b'Embedded price data' not in res.data
@@ -138,9 +138,97 @@ def test_check_ldjson_price_autodetect(client, live_server):
         follow_redirects=True
     )
     assert b"1 Imported" in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'ldjson-price-track-offer' not in res.data
     
     ##########################################################################################
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+
+
+def _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_data):
+
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    for k,v in client.application.config.get('DATASTORE').data['watching'].items():
+        assert v.get('last_error') == False
+        assert v.get('has_ldjson_price_data') == has_ldjson_price_data
+
+
+    ##########################################################################################
+    client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+
+
+def test_bad_ldjson_is_correctly_ignored(client, live_server):
+    #live_server_setup(live_server)
+    test_return_data = """
+            <html>
+            <head>
+                <script type="application/ld+json">
+                    {
+                        "@context": "http://schema.org",
+                        "@type": ["Product", "SubType"],
+                        "name": "My test product",
+                        "description": "",
+                        "offers": {
+                            "note" : "You can see the case-insensitive OffERS key, it should work",
+                            "@type": "Offer",
+                            "offeredBy": {
+                                "@type": "Organization",
+                                "name":"Person",
+                                "telephone":"+1 999 999 999"
+                            },
+                            "price": "1",
+                            "priceCurrency": "EUR",
+                            "url": "/some/url"
+                        }
+                    }
+                </script>
+            </head>
+            <body>
+            <div class="yes">Some extra stuff</div>
+            </body></html>
+     """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=True)
+    test_return_data = """
+            <html>
+            <head>
+                <script type="application/ld+json">
+                    {
+                        "@context": "http://schema.org",
+                        "@type": ["Product", "SubType"],
+                        "name": "My test product",
+                        "description": "",
+                        "BrokenOffers": {
+                            "@type": "Offer",
+                            "offeredBy": {
+                                "@type": "Organization",
+                                "name":"Person",
+                                "telephone":"+1 999 999 999"
+                            },
+                            "price": "1",
+                            "priceCurrency": "EUR",
+                            "url": "/some/url"
+                        }
+                    }
+                </script>
+            </head>
+            <body>
+            <div class="yes">Some extra stuff</div>
+            </body></html>
+     """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=False)
+

changedetectionio/tests/test_backup.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-from .util import set_original_response, set_modified_response, live_server_setup
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 from flask import url_for
 from urllib.request import urlopen
 from zipfile import ZipFile
@@ -19,12 +19,12 @@ def test_backup(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("import_page"),
-        data={"urls": url_for('test_endpoint', _external=True)},
+        data={"urls": url_for('test_endpoint', _external=True)+"?somechar=őőőőőőőő"},
         follow_redirects=True
     )
 
     assert b"1 Imported" in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     res = client.get(
         url_for("get_backup"),

changedetectionio/tests/test_css_selector.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 from ..html_tools import *
 
@@ -176,3 +176,77 @@ def test_check_multiple_filters(client, live_server):
     assert b"Blob A" in res.data # CSS was ok
     assert b"Blob B" in res.data # xPath was ok
     assert b"Blob C" not in res.data # Should not be included
+
+# The filter exists, but did not contain anything useful
+# Mainly used when the filter contains just an IMG, this can happen when someone selects an image in the visual-selector
+# Tests fetcher can throw a "ReplyWithContentButNoText" exception after applying filter and extracting text
+def test_filter_is_empty_help_suggestion(client, live_server):
+    #live_server_setup(live_server)
+
+    include_filters = "#blob-a"
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("""<html><body>
+         <div id="blob-a">
+           <img src="something.jpg">
+         </div>
+         </body>
+         </html>
+        """)
+
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+    wait_for_all_checks(client)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"include_filters": include_filters,
+              "url": test_url,
+              "tags": "",
+              "headers": "",
+              'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    wait_for_all_checks(client)
+
+
+    res = client.get(
+        url_for("index"),
+        follow_redirects=True
+    )
+
+    assert b'empty result or contain only an image' in res.data
+
+
+    ### Just an empty selector, no image
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("""<html><body>
+         <div id="blob-a">
+           <!-- doo doo -->
+         </div>
+         </body>
+         </html>
+        """)
+
+    res = client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    res = client.get(
+        url_for("index"),
+        follow_redirects=True
+    )
+
+    assert b'empty result or contain only an image' not in res.data
+    assert b'but contained no usable text' in res.data

changedetectionio/tests/test_extract_regex.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from .util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 from ..html_tools import *
 
@@ -55,6 +55,8 @@ def set_multiline_response():
      </p>
      
      <div>aaand something lines</div>
+     <br>
+     <div>and this should be</div>
      </body>
      </html>
     """
@@ -66,11 +68,10 @@ def set_multiline_response():
 
 
 def test_setup(client, live_server):
-
     live_server_setup(live_server)
 
 def test_check_filter_multiline(client, live_server):
-
+    #live_server_setup(live_server)
     set_multiline_response()
 
     # Add our URL to the import page
@@ -82,14 +83,15 @@ def test_check_filter_multiline(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(3)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"include_filters": '',
-              'extract_text': '/something.+?6 billion.+?lines/si',
+              # Test a regex and a plaintext
+              'extract_text': '/something.+?6 billion.+?lines/si\r\nand this should be',
               "url": test_url,
               "tags": "",
               "headers": "",
@@ -99,13 +101,19 @@ def test_check_filter_multiline(client, live_server):
     )
 
     assert b"Updated watch." in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)
+
+    res = client.get(url_for("index"))
+
+    # Issue 1828
+    assert b'not at the start of the expression' not in res.data
 
     res = client.get(
         url_for("preview_page", uuid="first"),
         follow_redirects=True
     )
-
+    # Plaintext that doesnt look like a regex should match also
+    assert b'and this should be' in res.data
 
     assert b'<div class="">Something' in res.data
     assert b'<div class="">across 6 billion multiple' in res.data
@@ -115,14 +123,11 @@ def test_check_filter_multiline(client, live_server):
     assert b'aaand something lines' not in res.data
 
 def test_check_filter_and_regex_extract(client, live_server):
-    sleep_time_for_fetch_thread = 3
+    
     include_filters = ".changetext"
 
     set_original_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -132,19 +137,15 @@ def test_check_filter_and_regex_extract(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    time.sleep(1)
-    # Trigger a check
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
         data={"include_filters": include_filters,
-              'extract_text': '\d+ online\r\n\d+ guests\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i',
+              'extract_text': '/\d+ online/\r\n/\d+ guests/\r\n/somecase insensitive \d+/i\r\n/somecase insensitive (345\d)/i\r\n/issue1828.+?2022/i',
               "url": test_url,
               "tags": "",
               "headers": "",
@@ -155,8 +156,13 @@ def test_check_filter_and_regex_extract(client, live_server):
 
     assert b"Updated watch." in res.data
 
+
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
+
+    res = client.get(url_for("index"))
+    #issue 1828
+    assert b'not at the start of the expression' not in res.data
 
     #  Make a change
     set_modified_response()
@@ -164,7 +170,7 @@ def test_check_filter_and_regex_extract(client, live_server):
     # Trigger a check
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should have 'unviewed' still
     # Because it should be looking at only that 'sametext' id

changedetectionio/tests/test_trigger_regex.py

@@ -2,7 +2,7 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from .util import live_server_setup, wait_for_all_checks
 
 
 def set_original_ignore_response():
@@ -26,13 +26,8 @@ def test_trigger_regex_functionality(client, live_server):
 
     live_server_setup(live_server)
 
-    sleep_time_for_fetch_thread = 3
-
     set_original_ignore_response()
 
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -43,7 +38,7 @@ def test_trigger_regex_functionality(client, live_server):
     assert b"1 Imported" in res.data
 
     # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (just a new one shouldnt have anything)
     res = client.get(url_for("index"))
@@ -57,7 +52,7 @@ def test_trigger_regex_functionality(client, live_server):
               "fetch_backend": "html_requests"},
         follow_redirects=True
     )
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     # so that we set the state to 'unviewed' after all the edits
     client.get(url_for("diff_history_page", uuid="first"))
 
@@ -65,7 +60,7 @@ def test_trigger_regex_functionality(client, live_server):
         f.write("some new noise")
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
 
     # It should report nothing found (nothing should match the regex)
     res = client.get(url_for("index"))
@@ -75,7 +70,7 @@ def test_trigger_regex_functionality(client, live_server):
         f.write("regex test123<br>\nsomething 123")
 
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    time.sleep(sleep_time_for_fetch_thread)
+    wait_for_all_checks(client)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 

changedetectionio/update_worker.py

@@ -3,7 +3,7 @@ import threading
 import queue
 import time
 
-from changedetectionio import content_fetcher
+from changedetectionio import content_fetcher, html_tools
 from .processors.text_json_diff import FilterNotFoundInResponse
 from .processors.restock_diff import UnableToExtractRestockData
 
@@ -251,7 +251,20 @@ class update_worker(threading.Thread):
                         # Totally fine, it's by choice - just continue on, nothing more to care about
                         # Page had elements/content but no renderable text
                         # Backend (not filters) gave zero output
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Got HTML content but no text found (With {} reply code).".format(e.status_code)})
+                        extra_help = ""
+                        if e.has_filters:
+                            # Maybe it contains an image? offer a more helpful link
+                            has_img = html_tools.include_filters(include_filters='img',
+                                                                 html_content=e.html_content)
+                            if has_img:
+                                extra_help = ", it's possible that the filters you have give an empty result or contain only an image."
+                            else:
+                                extra_help = ", it's possible that the filters were found, but contained no usable text."
+
+                        self.datastore.update_watch(uuid=uuid, update_obj={
+                            'last_error': f"Got HTML content but no text found (With {e.status_code} reply code){extra_help}"
+                        })
+
                         if e.screenshot:
                             self.datastore.save_screenshot(watch_uuid=uuid, screenshot=e.screenshot)
                         process_changedetection_results = False

requirements.txt

@@ -16,7 +16,7 @@ validators~=0.21
 # Set these versions together to avoid a RequestsDependencyWarning
 # >= 2.26 also adds Brotli support if brotli is installed
 brotli~=1.0
-requests[socks] ~=2.28
+requests[socks]
 
 urllib3>1.26
 chardet>2.3.0

runtime.txt

@@ -1 +1 @@
-python-3.9.15
+python-3.11.5

setup.py

@@ -41,7 +41,7 @@ setup(
     include_package_data=True,
     install_requires=install_requires,
     license="Apache License 2.0",
-    python_requires=">= 3.6",
+    python_requires=">= 3.7",
     classifiers=['Intended Audience :: Customer Service',
                  'Intended Audience :: Developers',
                  'Intended Audience :: Education',