BrowserSteps error handling improvements

.github/dependabot.yml

@@ -1,10 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: github-actions
-    directory: /
-    schedule:
-      interval: "weekly"
-    groups:
-      all:
-        patterns:
-        - "*"

.github/workflows/codeql-analysis.yml

@@ -34,7 +34,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -45,7 +45,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v3
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -59,4 +59,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v2

.github/workflows/containers.yml

@@ -41,7 +41,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python 3.11
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v4
         with:
           python-version: 3.11
 

.github/workflows/pypi-release.yml

@@ -1,72 +0,0 @@
-name: Publish Python 🐍distribution 📦 to PyPI and TestPyPI
-
-on: push
-jobs:
-  build:
-    name: Build distribution 📦
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.x"
-    - name: Install pypa/build
-      run: >-
-        python3 -m
-        pip install
-        build
-        --user
-    - name: Build a binary wheel and a source tarball
-      run: python3 -m build
-    - name: Store the distribution packages
-      uses: actions/upload-artifact@v4
-      with:
-        name: python-package-distributions
-        path: dist/
-
-
-  test-pypi-package:
-    name: Test the built 📦 package works basically.
-    runs-on: ubuntu-latest
-    needs:
-    - build
-    steps:
-    - name: Download all the dists
-      uses: actions/download-artifact@v4
-      with:
-        name: python-package-distributions
-        path: dist/
-    - name: Test that the basic pip built package runs without error
-      run: |
-        set -ex
-        pip3 install dist/changedetection.io*.whl
-        changedetection.io -d /tmp -p 10000 &
-        sleep 3
-        curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
-        curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null
-        killall changedetection.io
-
-
-  publish-to-pypi:
-    name: >-
-      Publish Python 🐍 distribution 📦 to PyPI
-    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
-    needs:
-    - test-pypi-package
-    runs-on: ubuntu-latest
-    environment:
-      name: release
-      url: https://pypi.org/p/changedetection.io
-    permissions:
-      id-token: write  # IMPORTANT: mandatory for trusted publishing
-
-    steps:
-    - name: Download all the dists
-      uses: actions/download-artifact@v4
-      with:
-        name: python-package-distributions
-        path: dist/
-    - name: Publish distribution 📦 to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/test-container-build.yml

@@ -26,7 +26,7 @@ jobs:
     steps:
         - uses: actions/checkout@v4
         - name: Set up Python 3.11
-          uses: actions/setup-python@v5
+          uses: actions/setup-python@v4
           with:
             python-version: 3.11
 

.github/workflows/test-only.yml

@@ -11,7 +11,7 @@ jobs:
 
       # Mainly just for link/flake8
       - name: Set up Python 3.11
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v4
         with:
           python-version: '3.11'
 
@@ -27,7 +27,7 @@ jobs:
         run: |
           
           docker network create changedet-network
-          
+
           # Selenium+browserless
           docker run --network changedet-network -d --hostname selenium  -p 4444:4444 --rm --shm-size="2g"  selenium/standalone-chrome:4
           docker run --network changedet-network -d --name browserless --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm  -p 3000:3000  --shm-size="2g"  browserless/chrome:1.60-chrome-stable
@@ -38,7 +38,7 @@ jobs:
       - name: Build changedetection.io container for testing
         run: |         
           # Build a changedetection.io container and start testing inside
-          docker build --build-arg LOGGER_LEVEL=TRACE -t test-changedetectionio .
+          docker build . -t test-changedetectionio
           # Debug info
           docker run test-changedetectionio  bash -c 'pip list'
 
@@ -47,46 +47,29 @@ jobs:
           # Debug SMTP server/echo message back server
           docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py' 
 
-      - name: Test built container with Pytest (generally as requests/plaintext fetching)
+      - name: Test built container with pytest
         run: |
           # Unit tests
-          echo "run test with unittest"
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'
           
           # All tests
-          echo "run test with pytest"
-          # The default pytest logger_level is TRACE
-          # To change logger_level for pytest(test/conftest.py),
-          # append the docker option. e.g. '-e LOGGER_LEVEL=DEBUG'
           docker run --network changedet-network  test-changedetectionio  bash -c 'cd changedetectionio && ./run_basic_tests.sh'
 
-      - name: Specific tests in built container for Selenium
+      - name: Test built container selenium+browserless/playwright
         run: |
           
           # Selenium fetch
           docker run --rm -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'
-
+          
-      - name: Specific tests in built container for Playwright
-        run: |         
           # Playwright/Browserless fetch
           docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
-
+          
-      - name: Specific tests in built container for headers and requests checks with Playwright
-        run: |                  
           # Settings headers playwright tests - Call back in from Browserless, check headers
           docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-
-      - name: Specific tests in built container for headers and requests checks with Selenium
-        run: |                  
           docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-
-      - name: Specific tests in built container with Playwright as Puppeteer experimental fetcher
-        run: |                  
           docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'          
-
+          
-      - name: Test built container restock detection via Playwright
-        run: |                            
           # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it
           docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 
@@ -118,17 +101,10 @@ jobs:
           docker run --name test-changedetectionio -p 5556:5000  -d test-changedetectionio
           sleep 3
           # Should return 0 (no error) when grep finds it
-          curl --retry-connrefused --retry 6  -s http://localhost:5556 |grep -q checkbox-uuid
+          curl -s http://localhost:5556 |grep -q checkbox-uuid
           
           # and IPv6
-          curl --retry-connrefused --retry 6  -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
+          curl -s -g -6 "http://[::1]:5556"|grep -q checkbox-uuid
-
-          # Check whether TRACE log is enabled.
-          # Also, check whether TRACE is came from STDERR
-          docker logs test-changedetectionio 2>&1 1>/dev/null | grep 'TRACE log is enabled' || exit 1
-          # Check whether DEBUG is came from STDOUT
-          docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1
-
           docker kill test-changedetectionio
 
       - name: Test changedetection.io SIGTERM and SIGINT signal shutdown
@@ -142,9 +118,8 @@ jobs:
           sleep 3
           # invert the check (it should be not 0/not running)
           docker ps
-          # check signal catch(STDERR) log. Because of
+          # check signal catch(STDOUT) log
-          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
+          docker logs sig-test | grep 'Shutdown: Got Signal - SIGINT' || exit 1
-          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1
           test -z "`docker ps|grep sig-test`"
           if [ $? -ne 0 ]
           then
@@ -164,9 +139,7 @@ jobs:
           sleep 3
           # invert the check (it should be not 0/not running)
           docker ps
-          # check signal catch(STDERR) log. Because of
+          docker logs sig-test | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
-          # changedetectionio/__init__.py: logger.add(sys.stderr, level=logger_level)
-          docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1
           test -z "`docker ps|grep sig-test`"
           if [ $? -ne 0 ]
           then

.github/workflows/test-pip-build.yml

@@ -0,0 +1,36 @@
+name: ChangeDetection.io PIP package test
+
+# Triggers the workflow on push or pull request events
+
+# This line doesnt work, even tho it is the documented one
+on: [push, pull_request]
+
+  # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
+  # @todo: some kind of path filter for requirements.txt and Dockerfile
+jobs:
+  test-pip-build-basics:
+    runs-on: ubuntu-latest
+    steps:
+        - uses: actions/checkout@v4
+
+        - name: Set up Python 3.11
+          uses: actions/setup-python@v4
+          with:
+            python-version: 3.11
+
+
+        - name: Test that the basic pip built package runs without error
+          run: |
+            set -e
+            mkdir dist
+            pip3 install wheel
+            python3 setup.py bdist_wheel            
+            pip3 install -r requirements.txt
+            rm ./changedetection.py
+            rm -rf changedetectio
+            
+            pip3 install dist/changedetection.io*.whl
+            changedetection.io -d /tmp -p 10000 &
+            sleep 3
+            curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+            killall -9 changedetection.io

Dockerfile

@@ -58,11 +58,6 @@ COPY changedetectionio /app/changedetectionio
 # Starting wrapper
 COPY changedetection.py /app/changedetection.py
 
-# Github Action test purpose(test-only.yml).
-# On production, it is effectively LOGGER_LEVEL=''.
-ARG LOGGER_LEVEL=''
-ENV LOGGER_LEVEL "$LOGGER_LEVEL"
-
 WORKDIR /app
 CMD ["python", "./changedetection.py", "-d", "/datastore"]
 

MANIFEST.in

@@ -10,8 +10,6 @@ prune changedetectionio/static/package-lock.json
 prune changedetectionio/static/styles/node_modules
 prune changedetectionio/static/styles/package-lock.json
 include changedetection.py
-include requirements.txt
-include README-pip.md
 global-exclude *.pyc
 global-exclude node_modules
 global-exclude venv

Procfile

@@ -0,0 +1 @@
+web: python3 ./changedetection.py -C -d ./datastore -p $PORT

README.md

@@ -17,7 +17,7 @@ _Live your data-life pro-actively._
 - Nothing to install, access via browser login after signup.
 - Super fast, no registration needed setup.
 - Get started watching and receiving website change notifications straight away.
-- See our [tutorials and how-to page for more inspiration](https://changedetection.io/tutorials) 
+
 
 ### Target specific parts of the webpage using the Visual Selector tool.
 
@@ -98,7 +98,7 @@ Please :star: star :star: this project and help it grow! https://github.com/dgtl
 With Docker composer, just clone this repository and..
 
 ```bash
-$ docker compose up -d
+$ docker-compose up -d
 ```
 
 Docker standalone
@@ -137,10 +137,10 @@ docker rm $(docker ps -a -f name=changedetection.io -q)
 docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```
 
-### docker compose
+### docker-compose
 
 ```bash
-docker compose pull && docker compose up -d
+docker-compose pull && docker-compose up -d
 ```
 
 See the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki
@@ -249,7 +249,7 @@ Supports managing the website watch list [via our API](https://changedetection.i
 Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 
 
-Firstly, consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
+Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
 
 Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)
 
@@ -273,9 +273,3 @@ I offer commercial support, this software is depended on by network security, ae
 ## Third-party licenses
 
 changedetectionio.html_tools.elementpath_tostring: Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati), Licensed under [MIT license](https://github.com/sissaschool/elementpath/blob/master/LICENSE)
-
-## Contributors
-
-Recognition of fantastic contributors to the project
-
-- Constantin Hong https://github.com/Constantin1489

app.json

@@ -0,0 +1,21 @@
+{
+  "name": "ChangeDetection.io",
+  "description": "The best and simplest self-hosted open source website change detection monitoring and notification service.",
+  "keywords": [
+    "changedetection",
+    "website monitoring"
+  ],
+  "repository": "https://github.com/dgtlmoon/changedetection.io",
+  "success_url": "/",
+  "scripts": {
+  },
+  "env": {
+  },
+  "formation": {
+    "web": {
+      "quantity": 1,
+      "size": "free"
+    }
+  },
+  "image": "heroku/python"
+}

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.45.13'
+__version__ = '0.45.9'
 
 from distutils.util import strtobool
 from json.decoder import JSONDecodeError
@@ -17,7 +17,6 @@ import sys
 
 from changedetectionio import store
 from changedetectionio.flask_app import changedetection_app
-from loguru import logger
 
 
 # Only global so we can access it in the signal handler
@@ -29,9 +28,9 @@ def sigshutdown_handler(_signo, _stack_frame):
     global app
     global datastore
     name = signal.Signals(_signo).name
-    logger.critical(f'Shutdown: Got Signal - {name} ({_signo}), Saving DB to disk and calling shutdown')
+    print(f'Shutdown: Got Signal - {name} ({_signo}), Saving DB to disk and calling shutdown')
     datastore.sync_to_json()
-    logger.success('Sync JSON to disk complete.')
+    print(f'Sync JSON to disk complete.')
     # This will throw a SystemExit exception, because eventlet.wsgi.server doesn't know how to deal with it.
     # Solution: move to gevent or other server in the future (#2014)
     datastore.stop_thread = True
@@ -58,22 +57,13 @@ def main():
         datastore_path = os.path.join(os.getcwd(), "../datastore")
 
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:l:", "port")
+        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:", "port")
     except getopt.GetoptError:
-        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path] -l [debug level - TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL]')
+        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
         sys.exit(2)
 
     create_datastore_dir = False
 
-    # Set a default logger level
-    logger_level = 'DEBUG'
-    # Set a logger level via shell env variable
-    # Used: Dockerfile for CICD
-    # To set logger level for pytest, see the app function in tests/conftest.py
-    if os.getenv("LOGGER_LEVEL"):
-        level = os.getenv("LOGGER_LEVEL")
-        logger_level = int(level) if level.isdigit() else level.upper()
-
     for opt, arg in opts:
         if opt == '-s':
             ssl_mode = True
@@ -88,7 +78,7 @@ def main():
             datastore_path = arg
 
         if opt == '-6':
-            logger.success("Enabling IPv6 listen support")
+            print ("Enabling IPv6 listen support")
             ipv6_enabled = True
 
         # Cleanup (remove text files that arent in the index)
@@ -99,25 +89,6 @@ def main():
         if opt == '-C':
             create_datastore_dir = True
 
-        if opt == '-l':
-            logger_level = int(arg) if arg.isdigit() else arg.upper()
-
-    # Without this, a logger will be duplicated
-    logger.remove()
-    try:
-        log_level_for_stdout = { 'DEBUG', 'SUCCESS' }
-        logger.configure(handlers=[
-            {"sink": sys.stdout, "level": logger_level,
-             "filter" : lambda record: record['level'].name in log_level_for_stdout},
-            {"sink": sys.stderr, "level": logger_level,
-             "filter": lambda record: record['level'].name not in log_level_for_stdout},
-            ])
-    # Catch negative number or wrong log level name
-    except ValueError:
-        print("Available log level names: TRACE, DEBUG(default), INFO, SUCCESS,"
-              " WARNING, ERROR, CRITICAL")
-        sys.exit(2)
-
     # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
     app_config = {'datastore_path': datastore_path}
 
@@ -125,19 +96,17 @@ def main():
         if create_datastore_dir:
             os.mkdir(app_config['datastore_path'])
         else:
-            logger.critical(
+            print(
-                f"ERROR: Directory path for the datastore '{app_config['datastore_path']}'"
+                "ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists or specify a directory with the -d option.\n"
-                f" does not exist, cannot start, please make sure the"
+                "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
-                f" directory exists or specify a directory with the -d option.\n"
-                f"Or use the -C parameter to create the directory.")
             sys.exit(2)
 
     try:
         datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
     except JSONDecodeError as e:
         # Dont' start if the JSON DB looks corrupt
-        logger.critical(f"ERROR: JSON DB or Proxy List JSON at '{app_config['datastore_path']}' appears to be corrupt, aborting.")
+        print ("ERROR: JSON DB or Proxy List JSON at '{}' appears to be corrupt, aborting".format(app_config['datastore_path']))
-        logger.critical(str(e))
+        print(str(e))
         return
 
     app = changedetection_app(app_config, datastore)
@@ -176,7 +145,7 @@ def main():
     #         proxy_set_header X-Forwarded-Prefix /app;
 
     if os.getenv('USE_X_SETTINGS'):
-        logger.info("USE_X_SETTINGS is ENABLED")
+        print ("USE_X_SETTINGS is ENABLED\n")
         from werkzeug.middleware.proxy_fix import ProxyFix
         app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
 

changedetectionio/api/api_v1.py

@@ -30,7 +30,7 @@ class Watch(Resource):
         self.update_q = kwargs['update_q']
 
     # Get information about a single watch, excluding the history list (can be large)
-    # curl http://localhost:5000/api/v1/watch/<string:uuid>
+    # curl http://localhost:4000/api/v1/watch/<string:uuid>
     # @todo - version2 - ?muted and ?paused should be able to be called together, return the watch struct not "OK"
     # ?recheck=true
     @auth.check_token
@@ -39,9 +39,9 @@ class Watch(Resource):
         @api {get} /api/v1/watch/:uuid Single watch - get data, recheck, pause, mute.
         @apiDescription Retrieve watch information and set muted/paused status
         @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=unmuted"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?muted=unmuted"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
-            curl "http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?paused=unpaused"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl "http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091?paused=unpaused"  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
         @apiName Watch
         @apiGroup Watch
         @apiParam {uuid} uuid Watch unique ID.
@@ -84,7 +84,7 @@ class Watch(Resource):
         """
         @api {delete} /api/v1/watch/:uuid Delete a watch and related history
         @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X DELETE -H"x-api-key:813031b16330fe25e3780cf0325daa45"
         @apiParam {uuid} uuid Watch unique ID.
         @apiName Delete
         @apiGroup Watch
@@ -103,7 +103,7 @@ class Watch(Resource):
         @api {put} /api/v1/watch/:uuid Update watch information
         @apiExample {curl} Example usage:
             Update (PUT)
-            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}'
+            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091 -X PUT -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "new list"}'
 
         @apiDescription Updates an existing watch using JSON, accepts the same structure as returned in <a href="#api-Watch-Watch">get single watch information</a>
         @apiParam {uuid} uuid Watch unique ID.
@@ -132,14 +132,13 @@ class WatchHistory(Resource):
         self.datastore = kwargs['datastore']
 
     # Get a list of available history for a watch by UUID
-    # curl http://localhost:5000/api/v1/watch/<string:uuid>/history
+    # curl http://localhost:4000/api/v1/watch/<string:uuid>/history
-    @auth.check_token
     def get(self, uuid):
         """
         @api {get} /api/v1/watch/<string:uuid>/history Get a list of all historical snapshots available for a watch
         @apiDescription Requires `uuid`, returns list
         @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
+            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
             {
                 "1676649279": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/cb7e9be8258368262246910e6a2a4c30.txt",
                 "1677092785": "/tmp/data/6a4b7d5c-fee4-4616-9f43-4ac97046b595/e20db368d6fc633e34f559ff67bb4044.txt",
@@ -167,7 +166,7 @@ class WatchSingleHistory(Resource):
         @api {get} /api/v1/watch/<string:uuid>/history/<int:timestamp> Get single snapshot from watch
         @apiDescription Requires watch `uuid` and `timestamp`. `timestamp` of "`latest`" for latest available snapshot, or <a href="#api-Watch_History-Get_list_of_available_stored_snapshots_for_watch">use the list returned here</a>
         @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
+            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history/1677092977 -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json"
         @apiName Get single snapshot content
         @apiGroup Watch History
         @apiSuccess (200) {String} OK
@@ -203,7 +202,7 @@ class CreateWatch(Resource):
         @api {post} /api/v1/watch Create a single watch
         @apiDescription Requires atleast `url` set, can accept the same structure as <a href="#api-Watch-Watch">get single watch information</a> to create.
         @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
+            curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45" -H "Content-Type: application/json" -d '{"url": "https://my-nice.com" , "tag": "nice list"}'
         @apiName Create
         @apiGroup Watch
         @apiSuccess (200) {String} OK Was created
@@ -246,7 +245,7 @@ class CreateWatch(Resource):
         @api {get} /api/v1/watch List watches
         @apiDescription Return concise list of available watches and some very basic info
         @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl http://localhost:4000/api/v1/watch -H"x-api-key:813031b16330fe25e3780cf0325daa45"
             {
                 "6a4b7d5c-fee4-4616-9f43-4ac97046b595": {
                     "last_changed": 1677103794,
@@ -364,7 +363,7 @@ class SystemInfo(Resource):
         @api {get} /api/v1/systeminfo Return system info
         @apiDescription Return some info about the current system state
         @apiExample {curl} Example usage:
-            curl http://localhost:5000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
+            curl http://localhost:4000/api/v1/systeminfo -H"x-api-key:813031b16330fe25e3780cf0325daa45"
             HTTP/1.0 200
             {
                 'queue_size': 10 ,

changedetectionio/blueprint/browser_steps/__init__.py

@@ -23,11 +23,11 @@
 
 from distutils.util import strtobool
 from flask import Blueprint, request, make_response
+import logging
 import os
 
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.flask_app import login_optionally_required
-from loguru import logger
 
 browsersteps_sessions = {}
 io_interface_context = None
@@ -58,7 +58,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             io_interface_context = io_interface_context.start()
 
         keepalive_ms = ((keepalive_seconds + 3) * 1000)
-        base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '').strip('"')
+        base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '')
         a = "?" if not '?' in base_url else '&'
         base_url += a + f"timeout={keepalive_ms}"
 
@@ -88,7 +88,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                 if parsed.password:
                     proxy['password'] = parsed.password
 
-                logger.debug(f"Browser Steps: UUID {watch_uuid} selected proxy {proxy_url}")
+                print("Browser Steps: UUID {} selected proxy {}".format(watch_uuid, proxy_url))
 
         # Tell Playwright to connect to Chrome and setup a new session via our stepper interface
         browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui(
@@ -115,10 +115,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
         if not watch_uuid:
             return make_response('No Watch UUID specified', 500)
 
-        logger.debug("Starting connection with playwright")
+        print("Starting connection with playwright")
-        logger.debug("browser_steps.py connecting")
+        logging.debug("browser_steps.py connecting")
         browsersteps_sessions[browsersteps_session_id] = start_browsersteps_session(watch_uuid)
-        logger.debug("Starting connection with playwright - done")
+        print("Starting connection with playwright - done")
         return {'browsersteps_session_id': browsersteps_session_id}
 
     @login_optionally_required
@@ -189,7 +189,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
                                          optional_value=step_optional_value)
 
             except Exception as e:
-                logger.error(f"Exception when calling step operation {step_operation} {str(e)}")
+                print("Exception when calling step operation", step_operation, str(e))
                 # Try to find something of value to give back to the user
                 return make_response(str(e).splitlines()[0], 401)
 

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -4,7 +4,6 @@ import os
 import time
 import re
 from random import randint
-from loguru import logger
 
 # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end
 # 0- off, 1- on
@@ -54,7 +53,7 @@ class steppable_browser_interface():
         if call_action_name == 'choose_one':
             return
 
-        logger.debug(f"> Action calling '{call_action_name}'")
+        print("> action calling", call_action_name)
         # https://playwright.dev/python/docs/selectors#xpath-selectors
         if selector and selector.startswith('/') and not selector.startswith('//'):
             selector = "xpath=" + selector
@@ -73,7 +72,7 @@ class steppable_browser_interface():
 
         action_handler(selector, optional_value)
         self.page.wait_for_timeout(1.5 * 1000)
-        logger.debug(f"Call action done in {time.time()-now:.2f}s")
+        print("Call action done in", time.time() - now)
 
     def action_goto_url(self, selector=None, value=None):
         # self.page.set_viewport_size({"width": 1280, "height": 5000})
@@ -83,7 +82,7 @@ class steppable_browser_interface():
         #and also wait for seconds ?
         #await page.waitForTimeout(1000);
         #await page.waitForTimeout(extra_wait_ms);
-        logger.debug(f"Time to goto URL {time.time()-now:.2f}s")
+        print("Time to goto URL ", time.time() - now)
         return response
 
     def action_click_element_containing_text(self, selector=None, value=''):
@@ -104,7 +103,7 @@ class steppable_browser_interface():
         return response
 
     def action_click_element(self, selector, value):
-        logger.debug("Clicking element")
+        print("Clicking element")
         if not len(selector.strip()):
             return
 
@@ -112,7 +111,7 @@ class steppable_browser_interface():
 
     def action_click_element_if_exists(self, selector, value):
         import playwright._impl._errors as _api_types
-        logger.debug("Clicking element if exists")
+        print("Clicking element if exists")
         if not len(selector.strip()):
             return
         try:
@@ -228,11 +227,11 @@ class browsersteps_live_ui(steppable_browser_interface):
         # Listen for all console events and handle errors
         self.page.on("console", lambda msg: print(f"Browser steps console - {msg.type}: {msg.text} {msg.args}"))
 
-        logger.debug(f"Time to browser setup {time.time()-now:.2f}s")
+        print("Time to browser setup", time.time() - now)
         self.page.wait_for_timeout(1 * 1000)
 
     def mark_as_closed(self):
-        logger.debug("Page closed, cleaning up..")
+        print("Page closed, cleaning up..")
 
     @property
     def has_expired(self):
@@ -258,7 +257,7 @@ class browsersteps_live_ui(steppable_browser_interface):
         xpath_data = self.page.evaluate("async () => {" + xpath_element_js + "}")
         # So the JS will find the smallest one first
         xpath_data['size_pos'] = sorted(xpath_data['size_pos'], key=lambda k: k['width'] * k['height'], reverse=True)
-        logger.debug(f"Time to complete get_current_state of browser {time.time()-now:.2f}s")
+        print("Time to complete get_current_state of browser", time.time() - now)
         # except
         # playwright._impl._api_types.Error: Browser closed.
         # @todo show some countdown timer?

changedetectionio/content_fetcher.py

@@ -4,12 +4,12 @@ from urllib.parse import urlparse
 import chardet
 import hashlib
 import json
+import logging
 import os
 import requests
 import sys
 import time
 import urllib.parse
-from loguru import logger
 
 visualselector_xpath_selectors = 'div,span,form,table,tbody,tr,td,a,p,ul,li,h1,h2,h3,h4, header, footer, section, article, aside, details, main, nav, section, summary'
 
@@ -47,11 +47,10 @@ class BrowserStepsStepException(Exception):
     def __init__(self, step_n, original_e):
         self.step_n = step_n
         self.original_e = original_e
-        logger.debug(f"Browser Steps exception at step {self.step_n} {str(original_e)}")
+        print(f"Browser Steps exception at step {self.step_n}", str(original_e))
         return
 
 
-# @todo - make base Exception class that announces via logger()
 class PageUnloadable(Exception):
     def __init__(self, status_code, url, message, screenshot=False):
         # Set this so we can use it in other parts of the app
@@ -189,7 +188,7 @@ class Fetcher():
 
             for step in valid_steps:
                 step_n += 1
-                logger.debug(f">> Iterating check - browser Step n {step_n} - {step['operation']}...")
+                print(">> Iterating check - browser Step n {} - {}...".format(step_n, step['operation']))
                 self.screenshot_step("before-" + str(step_n))
                 self.save_step_html("before-" + str(step_n))
                 try:
@@ -206,8 +205,8 @@ class Fetcher():
                                                       optional_value=optional_value)
                     self.screenshot_step(step_n)
                     self.save_step_html(step_n)
+
                 except (Error, TimeoutError) as e:
-                    logger.debug(str(e))
                     # Stop processing here
                     raise BrowserStepsStepException(step_n=step_n, original_e=e)
 
@@ -296,14 +295,14 @@ class base_html_playwright(Fetcher):
 
         if self.browser_steps_screenshot_path is not None:
             destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.jpeg'.format(step_n))
-            logger.debug(f"Saving step screenshot to {destination}")
+            logging.debug("Saving step screenshot to {}".format(destination))
             with open(destination, 'wb') as f:
                 f.write(screenshot)
 
     def save_step_html(self, step_n):
         content = self.page.content()
         destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
-        logger.debug(f"Saving step HTML to {destination}")
+        logging.debug("Saving step HTML to {}".format(destination))
         with open(destination, 'w') as f:
             f.write(content)
 
@@ -390,24 +389,10 @@ class base_html_playwright(Fetcher):
             raise PageUnloadable(url=url, status_code=None, message=f"Timed out connecting to browserless, retrying..")
         else:
             # 200 Here means that the communication to browserless worked only, not the page state
-            try:
+            if response.status_code == 200:
-                x = response.json()
-            except Exception as e:
-                raise PageUnloadable(url=url, message="Error reading JSON response from browserless")
-
-            try:
-                self.status_code = response.status_code
-            except Exception as e:
-                raise PageUnloadable(url=url, message="Error reading status_code code response from browserless")
-
-            self.headers = x.get('headers')
-
-            if self.status_code != 200 and not ignore_status_codes:
-                raise Non200ErrorCodeReceived(url=url, status_code=self.status_code, page_html=x.get('content',''))
-
-            if self.status_code == 200:
                 import base64
 
+                x = response.json()
                 if not x.get('screenshot'):
                     # https://github.com/puppeteer/puppeteer/blob/v1.0.0/docs/troubleshooting.md#tips
                     # https://github.com/puppeteer/puppeteer/issues/1834
@@ -418,10 +403,16 @@ class base_html_playwright(Fetcher):
                 if not x.get('content', '').strip():
                     raise EmptyReply(url=url, status_code=None)
 
+                if x.get('status_code', 200) != 200 and not ignore_status_codes:
+                    raise Non200ErrorCodeReceived(url=url, status_code=x.get('status_code', 200), page_html=x['content'])
+
                 self.content = x.get('content')
+                self.headers = x.get('headers')
                 self.instock_data = x.get('instock_data')
                 self.screenshot = base64.b64decode(x.get('screenshot'))
+                self.status_code = x.get('status_code')
                 self.xpath_data = x.get('xpath_data')
+
             else:
                 # Some other error from browserless
                 raise PageUnloadable(url=url, status_code=None, message=response.content.decode('utf-8'))
@@ -500,7 +491,7 @@ class base_html_playwright(Fetcher):
             if response is None:
                 context.close()
                 browser.close()
-                logger.debug("Content Fetcher > Response object was none")
+                print("Content Fetcher > Response object was none")
                 raise EmptyReply(url=url, status_code=None)
 
             try:
@@ -512,7 +503,7 @@ class base_html_playwright(Fetcher):
                 # This can be ok, we will try to grab what we could retrieve
                 pass
             except Exception as e:
-                logger.debug(f"Content Fetcher > Other exception when executing custom JS code {str(e)}")
+                print("Content Fetcher > Other exception when executing custom JS code", str(e))
                 context.close()
                 browser.close()
                 raise PageUnloadable(url=url, status_code=None, message=str(e))
@@ -520,13 +511,8 @@ class base_html_playwright(Fetcher):
             extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
             self.page.wait_for_timeout(extra_wait * 1000)
 
-            try:
+
-                self.status_code = response.status
+            self.status_code = response.status
-            except Exception as e:
-                # https://github.com/dgtlmoon/changedetection.io/discussions/2122#discussioncomment-8241962
-                logger.critical(f"Response from browserless/playwright did not have a status_code! Response follows.")
-                logger.critical(response)
-                raise PageUnloadable(url=url, status_code=None, message=str(e))
 
             if self.status_code != 200 and not ignore_status_codes:
 
@@ -538,7 +524,7 @@ class base_html_playwright(Fetcher):
             if len(self.page.content().strip()) == 0:
                 context.close()
                 browser.close()
-                logger.debug("Content Fetcher > Content was empty")
+                print("Content Fetcher > Content was empty")
                 raise EmptyReply(url=url, status_code=response.status)
 
             # Run Browser Steps here
@@ -690,7 +676,7 @@ class base_html_webdriver(Fetcher):
             try:
                 self.driver.quit()
             except Exception as e:
-                logger.debug(f"Content Fetcher > Exception in chrome shutdown/quit {str(e)}")
+                print("Content Fetcher > Exception in chrome shutdown/quit" + str(e))
 
 
 # "html_requests" is listed as the default fetcher in store.py!
@@ -751,8 +737,6 @@ class html_requests(Fetcher):
                 if encoding:
                     r.encoding = encoding
 
-        self.headers = r.headers
-
         if not r.content or not len(r.content):
             raise EmptyReply(url=url, status_code=r.status_code)
 
@@ -769,7 +753,7 @@ class html_requests(Fetcher):
         else:
             self.content = r.text
 
-
+        self.headers = r.headers
         self.raw_content = r.content
 
 

changedetectionio/flask_app.py

@@ -12,7 +12,7 @@ from functools import wraps
 from threading import Event
 import datetime
 import flask_login
-from loguru import logger
+import logging
 import os
 import pytz
 import queue
@@ -210,8 +210,6 @@ def login_optionally_required(func):
     return decorated_view
 
 def changedetection_app(config=None, datastore_o=None):
-    logger.trace("TRACE log is enabled")
-
     global datastore
     datastore = datastore_o
 
@@ -316,9 +314,6 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.route("/rss", methods=['GET'])
     def rss():
-        from jinja2 import Environment, BaseLoader
-        jinja2_env = Environment(loader=BaseLoader)
-        now = time.time()
         # Always requires token set
         app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
         rss_url_token = request.args.get('token')
@@ -382,12 +377,8 @@ def changedetection_app(config=None, datastore_o=None):
                                              include_equal=False,
                                              line_feed_sep="<br>")
 
-                # @todo Make this configurable and also consider html-colored markup
+                fe.content(content="<html><body><h4>{}</h4>{}</body></html>".format(watch_title, html_diff),
-                # @todo User could decide if <link> goes to the diff page, or to the watch link
+                           type='CDATA')
-                rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n"
-                content = jinja2_env.from_string(rss_template).render(watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
-
-                fe.content(content=content, type='CDATA')
 
                 fe.guid(guid, permalink=False)
                 dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key))
@@ -396,7 +387,6 @@ def changedetection_app(config=None, datastore_o=None):
 
         response = make_response(fg.rss_str())
         response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8')
-        logger.trace(f"RSS generated in {time.time() - now:.3f}s")
         return response
 
     @app.route("/", methods=['GET'])
@@ -1502,7 +1492,7 @@ def changedetection_app(config=None, datastore_o=None):
 
 
         except Exception as e:
-            logger.error(f"Error sharing -{str(e)}")
+            logging.error("Error sharing -{}".format(str(e)))
             flash("Could not share, something went wrong while communicating with the share server - {}".format(str(e)), 'error')
 
         # https://changedetection.io/share/VrMv05wpXyQa
@@ -1602,20 +1592,11 @@ def notification_runner():
 
             try:
                 from changedetectionio import notification
-                # Fallback to system config if not set
-                if not n_object.get('notification_body') and datastore.data['settings']['application'].get('notification_body'):
-                    n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
-
-                if not n_object.get('notification_title') and datastore.data['settings']['application'].get('notification_title'):
-                    n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
-
-                if not n_object.get('notification_format') and datastore.data['settings']['application'].get('notification_format'):
-                    n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
 
                 sent_obj = notification.process_notification(n_object, datastore)
 
             except Exception as e:
-                logger.error(f"Watch URL: {n_object['watch_url']}  Error {str(e)}")
+                logging.error("Watch URL: {}  Error {}".format(n_object['watch_url'], str(e)))
 
                 # UUID wont be present when we submit a 'test' from the global settings
                 if 'uuid' in n_object:
@@ -1638,7 +1619,7 @@ def ticker_thread_check_time_launch_checks():
     proxy_last_called_time = {}
 
     recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 20))
-    logger.debug(f"System env MINIMUM_SECONDS_RECHECK_TIME {recheck_time_minimum_seconds}")
+    print("System env MINIMUM_SECONDS_RECHECK_TIME", recheck_time_minimum_seconds)
 
     # Spin up Workers that do the fetching
     # Can be overriden by ENV or use the default settings
@@ -1683,7 +1664,7 @@ def ticker_thread_check_time_launch_checks():
             now = time.time()
             watch = datastore.data['watching'].get(uuid)
             if not watch:
-                logger.error(f"Watch: {uuid} no longer present.")
+                logging.error("Watch: {} no longer present.".format(uuid))
                 continue
 
             # No need todo further processing if it's paused
@@ -1716,10 +1697,10 @@ def ticker_thread_check_time_launch_checks():
                             time_since_proxy_used = int(time.time() - proxy_last_used_time)
                             if time_since_proxy_used < proxy_list_reuse_time_minimum:
                                 # Not enough time difference reached, skip this watch
-                                logger.debug(f"> Skipped UUID {uuid} "
+                                print("> Skipped UUID {} using proxy '{}', not enough time between proxy requests {}s/{}s".format(uuid,
-                                        f"using proxy '{watch_proxy}', not "
+                                                                                                                         watch_proxy,
-                                        f"enough time between proxy requests "
+                                                                                                                         time_since_proxy_used,
-                                        f"{time_since_proxy_used}s/{proxy_list_reuse_time_minimum}s")
+                                                                                                                         proxy_list_reuse_time_minimum))
                                 continue
                             else:
                                 # Record the last used time
@@ -1727,12 +1708,14 @@ def ticker_thread_check_time_launch_checks():
 
                     # Use Epoch time as priority, so we get a "sorted" PriorityQueue, but we can still push a priority 1 into it.
                     priority = int(time.time())
-                    logger.debug(
+                    print(
-                        f"> Queued watch UUID {uuid} "
+                        "> Queued watch UUID {} last checked at {} queued at {:0.2f} priority {} jitter {:0.2f}s, {:0.2f}s since last checked".format(
-                        f"last checked at {watch['last_checked']} "
+                            uuid,
-                        f"queued at {now:0.2f} priority {priority} "
+                            watch['last_checked'],
-                        f"jitter {watch.jitter_seconds:0.2f}s, "
+                            now,
-                        f"{now - watch['last_checked']:0.2f}s since last checked")
+                            priority,
+                            watch.jitter_seconds,
+                            now - watch['last_checked']))
 
                     # Into the queue with you
                     update_q.put(queuedWatchMetaData.PrioritizedItem(priority=priority, item={'uuid': uuid, 'skip_when_checksum_same': True}))

changedetectionio/forms.py

@@ -43,7 +43,6 @@ valid_method = {
     'PUT',
     'PATCH',
     'DELETE',
-    'OPTIONS',
 }
 
 default_method = 'GET'

changedetectionio/importer.py

@@ -2,7 +2,6 @@ from abc import ABC, abstractmethod
 import time
 import validators
 from wtforms import ValidationError
-from loguru import logger
 
 from changedetectionio.forms import validate_url
 
@@ -196,7 +195,7 @@ class import_xlsx_wachete(Importer):
                     try:
                         validate_url(data.get('url'))
                     except ValidationError as e:
-                        logger.error(f">> Import URL error {data.get('url')} {str(e)}")
+                        print(">> import URL error", data.get('url'), str(e))
                         flash(f"Error processing row number {row_id}, URL value was incorrect, row was skipped.", 'error')
                         # Don't bother processing anything else on this row
                         continue
@@ -210,7 +209,7 @@ class import_xlsx_wachete(Importer):
                         self.new_uuids.append(new_uuid)
                         good += 1
             except Exception as e:
-                logger.error(e)
+                print(e)
                 flash(f"Error processing row number {row_id}, check all cell data types are correct, row was skipped.", 'error')
             else:
                 row_id += 1
@@ -265,7 +264,7 @@ class import_xlsx_custom(Importer):
                         try:
                             validate_url(url)
                         except ValidationError as e:
-                            logger.error(f">> Import URL error {url} {str(e)}")
+                            print(">> Import URL error", url, str(e))
                             flash(f"Error processing row number {row_i}, URL value was incorrect, row was skipped.", 'error')
                             # Don't bother processing anything else on this row
                             url = None
@@ -294,7 +293,7 @@ class import_xlsx_custom(Importer):
                         self.new_uuids.append(new_uuid)
                         good += 1
         except Exception as e:
-            logger.error(e)
+            print(e)
             flash(f"Error processing row number {row_i}, check all cell data types are correct, row was skipped.", 'error')
         else:
             row_i += 1

changedetectionio/model/Watch.py

@@ -1,10 +1,10 @@
 from distutils.util import strtobool
+import logging
 import os
 import re
 import time
 import uuid
 from pathlib import Path
-from loguru import logger
 
 # Allowable protocols, protects against javascript: etc
 # file:// is further checked by ALLOW_FILE_URI
@@ -56,7 +56,6 @@ base_config = {
     'previous_md5': False,
     'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
     'proxy': None,  # Preferred proxy connection
-    'remote_server_reply': None, # From 'server' reply header
     'subtractive_selectors': [],
     'tag': '', # Old system of text name for a tag, to be removed
     'tags': [], # list of UUIDs to App.Tags
@@ -123,7 +122,7 @@ class model(dict):
 
     def ensure_data_dir_exists(self):
         if not os.path.isdir(self.watch_data_dir):
-            logger.debug(f"> Creating data dir {self.watch_data_dir}")
+            print ("> Creating data dir {}".format(self.watch_data_dir))
             os.mkdir(self.watch_data_dir)
 
     @property
@@ -212,7 +211,7 @@ class model(dict):
         # Read the history file as a dict
         fname = os.path.join(self.watch_data_dir, "history.txt")
         if os.path.isfile(fname):
-            logger.debug(f"Reading watch history index for {self.get('uuid')}")
+            logging.debug("Reading history index " + str(time.time()))
             with open(fname, "r") as f:
                 for i in f.readlines():
                     if ',' in i:
@@ -247,10 +246,10 @@ class model(dict):
     @property
     def has_browser_steps(self):
         has_browser_steps = self.get('browser_steps') and list(filter(
-            lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
+                lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),
-            self.get('browser_steps')))
+                self.get('browser_steps')))
 
-        return has_browser_steps
+        return  has_browser_steps
 
     # Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0.
     @property

changedetectionio/notification.py

@@ -1,9 +1,7 @@
 import apprise
-import time
 from jinja2 import Environment, BaseLoader
 from apprise import NotifyFormat
 import json
-from loguru import logger
 
 valid_tokens = {
     'base_url': '',
@@ -116,16 +114,13 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
 
 def process_notification(n_object, datastore):
 
-    now = time.time()
-    if n_object.get('notification_timestamp'):
-        logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s")
     # Insert variables into the notification content
     notification_parameters = create_notification_parameters(n_object, datastore)
 
     # Get the notification body from datastore
     jinja2_env = Environment(loader=BaseLoader)
-    n_body = jinja2_env.from_string(n_object.get('notification_body', '')).render(**notification_parameters)
+    n_body = jinja2_env.from_string(n_object.get('notification_body', default_notification_body)).render(**notification_parameters)
-    n_title = jinja2_env.from_string(n_object.get('notification_title', '')).render(**notification_parameters)
+    n_title = jinja2_env.from_string(n_object.get('notification_title', default_notification_title)).render(**notification_parameters)
     n_format = valid_notification_formats.get(
         n_object.get('notification_format', default_notification_format),
         valid_notification_formats[default_notification_format],
@@ -136,100 +131,90 @@ def process_notification(n_object, datastore):
         # Initially text or whatever
         n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
 
-    logger.trace(f"Complete notification body including Jinja and placeholders calculated in  {time.time() - now:.3f}s")
 
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
     # raise it as an exception
-
+    apobjs=[]
-    sent_objs = []
+    sent_objs=[]
     from .apprise_asset import asset
-    apobj = apprise.Apprise(debug=True, asset=asset)
+    for url in n_object['notification_urls']:
+        url = jinja2_env.from_string(url).render(**notification_parameters)
+        apobj = apprise.Apprise(debug=True, asset=asset)
+        url = url.strip()
+        if len(url):
+            print(">> Process Notification: AppRise notifying {}".format(url))
+            with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
+                # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
+                # Because different notifications may require different pre-processing, run each sequentially :(
+                # 2000 bytes minus -
+                #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
+                #     Length of URL - Incase they specify a longer custom avatar_url
 
-    if not n_object.get('notification_urls'):
+                # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
-        return None
+                k = '?' if not '?' in url else '&'
+                if not 'avatar_url' in url \
+                        and not url.startswith('mail') \
+                        and not url.startswith('post') \
+                        and not url.startswith('get') \
+                        and not url.startswith('delete') \
+                        and not url.startswith('put'):
+                    url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
 
-    with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
+                if url.startswith('tgram://'):
-        for url in n_object['notification_urls']:
+                    # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
-            url = url.strip()
+                    # re https://github.com/dgtlmoon/changedetection.io/issues/555
-            if not url:
+                    # @todo re-use an existing library we have already imported to strip all non-allowed tags
-                logger.warning(f"Process Notification: skipping empty notification URL.")
+                    n_body = n_body.replace('<br>', '\n')
-                continue
+                    n_body = n_body.replace('</br>', '\n')
+                    # real limit is 4096, but minus some for extra metadata
+                    payload_max_size = 3600
+                    body_limit = max(0, payload_max_size - len(n_title))
+                    n_title = n_title[0:payload_max_size]
+                    n_body = n_body[0:body_limit]
 
-            logger.info(">> Process Notification: AppRise notifying {}".format(url))
+                elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith('https://discord.com/api'):
-            url = jinja2_env.from_string(url).render(**notification_parameters)
+                    # real limit is 2000, but minus some for extra metadata
+                    payload_max_size = 1700
+                    body_limit = max(0, payload_max_size - len(n_title))
+                    n_title = n_title[0:payload_max_size]
+                    n_body = n_body[0:body_limit]
 
-            # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
+                elif url.startswith('mailto'):
-            # Because different notifications may require different pre-processing, run each sequentially :(
+                    # Apprise will default to HTML, so we need to override it
-            # 2000 bytes minus -
+                    # So that whats' generated in n_body is in line with what is going to be sent.
-            #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
+                    # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-            #     Length of URL - Incase they specify a longer custom avatar_url
+                    if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
+                        prefix = '?' if not '?' in url else '&'
+                        # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
+                        n_format = n_format.tolower()
+                        url = "{}{}format={}".format(url, prefix, n_format)
+                    # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
 
-            # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
+                apobj.add(url)
-            k = '?' if not '?' in url else '&'
-            if not 'avatar_url' in url \
-                    and not url.startswith('mail') \
-                    and not url.startswith('post') \
-                    and not url.startswith('get') \
-                    and not url.startswith('delete') \
-                    and not url.startswith('put'):
-                url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
 
-            if url.startswith('tgram://'):
+                apobj.notify(
-                # Telegram only supports a limit subset of HTML, remove the '<br>' we place in.
+                    title=n_title,
-                # re https://github.com/dgtlmoon/changedetection.io/issues/555
+                    body=n_body,
-                # @todo re-use an existing library we have already imported to strip all non-allowed tags
+                    body_format=n_format,
-                n_body = n_body.replace('<br>', '\n')
+                    # False is not an option for AppRise, must be type None
-                n_body = n_body.replace('</br>', '\n')
+                    attach=n_object.get('screenshot', None)
-                # real limit is 4096, but minus some for extra metadata
+                )
-                payload_max_size = 3600
-                body_limit = max(0, payload_max_size - len(n_title))
-                n_title = n_title[0:payload_max_size]
-                n_body = n_body[0:body_limit]
 
-            elif url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks') or url.startswith(
+                apobj.clear()
-                    'https://discord.com/api'):
-                # real limit is 2000, but minus some for extra metadata
-                payload_max_size = 1700
-                body_limit = max(0, payload_max_size - len(n_title))
-                n_title = n_title[0:payload_max_size]
-                n_body = n_body[0:body_limit]
 
-            elif url.startswith('mailto'):
+                # Incase it needs to exist in memory for a while after to process(?)
-                # Apprise will default to HTML, so we need to override it
+                apobjs.append(apobj)
-                # So that whats' generated in n_body is in line with what is going to be sent.
-                # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-                if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
-                    prefix = '?' if not '?' in url else '&'
-                    # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
-                    n_format = n_format.lower()
-                    url = f"{url}{prefix}format={n_format}"
-                # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
 
-            apobj.add(url)
+                # Returns empty string if nothing found, multi-line string otherwise
+                log_value = logs.getvalue()
+                if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
+                    raise Exception(log_value)
 
-            sent_objs.append({'title': n_title,
+                sent_objs.append({'title': n_title,
-                              'body': n_body,
+                                  'body': n_body,
-                              'url': url,
+                                  'url' : url,
-                              'body_format': n_format})
+                                  'body_format': n_format})
-
-        # Blast off the notifications tht are set in .add()
-        apobj.notify(
-            title=n_title,
-            body=n_body,
-            body_format=n_format,
-            # False is not an option for AppRise, must be type None
-            attach=n_object.get('screenshot', None)
-        )
-
-        # Give apprise time to register an error
-        time.sleep(3)
-
-        # Returns empty string if nothing found, multi-line string otherwise
-        log_value = logs.getvalue()
-
-        if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
-            raise Exception(log_value)
 
     # Return what was sent for better logging - after the for loop
     return sent_objs

changedetectionio/processors/__init__.py

@@ -5,7 +5,6 @@ import re
 from changedetectionio import content_fetcher
 from copy import deepcopy
 from distutils.util import strtobool
-from loguru import logger
 
 class difference_detection_processor():
 
@@ -70,7 +69,7 @@ class difference_detection_processor():
         proxy_url = None
         if preferred_proxy_id:
             proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
-            logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}' for {url}")
+            print(f"Using proxy Key: {preferred_proxy_id} as Proxy URL {proxy_url}")
 
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)

changedetectionio/processors/restock_diff.py

@@ -1,9 +1,8 @@
 
-from . import difference_detection_processor
-from copy import deepcopy
-from loguru import logger
 import hashlib
 import urllib3
+from . import difference_detection_processor
+from copy import deepcopy
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -44,13 +43,11 @@ class perform_site_check(difference_detection_processor):
             fetched_md5 = hashlib.md5(self.fetcher.instock_data.encode('utf-8')).hexdigest()
             # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.
             update_obj["in_stock"] = True if self.fetcher.instock_data == 'Possibly in stock' else False
-            logger.debug(f"Watch UUID {uuid} restock check returned '{self.fetcher.instock_data}' from JS scraper.")
         else:
             raise UnableToExtractRestockData(status_code=self.fetcher.status_code)
 
         # The main thing that all this at the moment comes down to :)
         changed_detected = False
-        logger.debug(f"Watch UUID {uuid} restock check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
 
         if watch.get('previous_md5') and watch.get('previous_md5') != fetched_md5:
             # Yes if we only care about it going to instock, AND we are in stock
@@ -63,4 +60,5 @@ class perform_site_check(difference_detection_processor):
 
         # Always record the new checksum
         update_obj["previous_md5"] = fetched_md5
-        return changed_detected, update_obj, self.fetcher.instock_data.encode('utf-8').strip()
+
+        return changed_detected, update_obj, self.fetcher.instock_data.encode('utf-8')

changedetectionio/processors/text_json_diff.py

@@ -2,16 +2,16 @@
 
 import hashlib
 import json
+import logging
 import os
 import re
 import urllib3
 
-from . import difference_detection_processor
-from ..html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text
 from changedetectionio import content_fetcher, html_tools
 from changedetectionio.blueprint.price_data_follower import PRICE_DATA_TRACK_ACCEPT, PRICE_DATA_TRACK_REJECT
 from copy import deepcopy
-from loguru import logger
+from . import difference_detection_processor
+from ..html_tools import PERL_STYLE_REGEX, cdata_in_document_to_text
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -335,17 +335,15 @@ class perform_site_check(difference_detection_processor):
                 if not watch['title'] or not len(watch['title']):
                     update_obj['title'] = html_tools.extract_element(find='title', html_content=self.fetcher.content)
 
-        logger.debug(f"Watch UUID {uuid} content check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}")
-
         if changed_detected:
             if watch.get('check_unique_lines', False):
                 has_unique_lines = watch.lines_contain_something_unique_compared_to_history(lines=stripped_text_from_html.splitlines())
                 # One or more lines? unsure?
                 if not has_unique_lines:
-                    logger.debug(f"check_unique_lines: UUID {uuid} didnt have anything new setting change_detected=False")
+                    logging.debug("check_unique_lines: UUID {} didnt have anything new setting change_detected=False".format(uuid))
                     changed_detected = False
                 else:
-                    logger.debug(f"check_unique_lines: UUID {uuid} had unique content")
+                    logging.debug("check_unique_lines: UUID {} had unique content".format(uuid))
 
         # Always record the new checksum
         update_obj["previous_md5"] = fetched_md5

changedetectionio/res/stock-not-in-stock.js

@@ -1,132 +1,117 @@
-// Restock Detector
-// (c) Leigh Morresi dgtlmoon@gmail.com
-//
-// Assumes the product is in stock to begin with, unless the following appears above the fold ;
-// - outOfStockTexts appears above the fold (out of stock)
-// - negateOutOfStockRegex (really is in stock)
-
 function isItemInStock() {
-    // @todo Pass these in so the same list can be used in non-JS fetchers
+  // @todo Pass these in so the same list can be used in non-JS fetchers
-    const outOfStockTexts = [
+  const outOfStockTexts = [
-        ' أخبرني عندما يتوفر',
+    ' أخبرني عندما يتوفر',
-        '0 in stock',
+    '0 in stock',
-        'agotado',
+    'agotado',
-        'article épuisé',
+    'article épuisé',
-        'artikel zurzeit vergriffen',
+    'artikel zurzeit vergriffen',
-        'as soon as stock is available',
+    'as soon as stock is available',
-        'ausverkauft', // sold out
+    'ausverkauft', // sold out
-        'available for back order',
+    'available for back order',
-        'back-order or out of stock',
+    'back-order or out of stock',
-        'backordered',
+    'backordered',
-        'benachrichtigt mich', // notify me
+    'benachrichtigt mich', // notify me
-        'brak na stanie',
+    'brak na stanie',
-        'brak w magazynie',
+    'brak w magazynie',
-        'coming soon',
+    'coming soon',
-        'currently have any tickets for this',
+    'currently have any tickets for this',
-        'currently unavailable',
+    'currently unavailable',
-        'dostępne wkrótce',
+    'dostępne wkrótce',
-        'en rupture de stock',
+    'en rupture de stock',
-        'ist derzeit nicht auf lager',
+    'ist derzeit nicht auf lager',
-        'item is no longer available',
+    'item is no longer available',
-        'let me know when it\'s available',
+    'let me know when it\'s available',
-        'message if back in stock',
+    'message if back in stock',
-        'nachricht bei',
+    'nachricht bei',
-        'nicht auf lager',
+    'nicht auf lager',
-        'nicht lieferbar',
+    'nicht lieferbar',
-        'nicht zur verfügung',
+    'nicht zur verfügung',
-        'niet beschikbaar',
+    'niet beschikbaar',
-        'niet leverbaar',
+    'niet leverbaar',
-        'no disponible temporalmente',
+    'no disponible temporalmente',
-        'no longer in stock',
+    'no longer in stock',
-        'no tickets available',
+    'no tickets available',
-        'not available',
+    'not available',
-        'not currently available',
+    'not currently available',
-        'not in stock',        
+    'not in stock',
-        'notify me when available',
+    'notify me when available',
-        'notify when available',            
+    'não estamos a aceitar encomendas',
-        'não estamos a aceitar encomendas',
+    'out of stock',
-        'out of stock',
+    'out-of-stock',
-        'out-of-stock',
+    'produkt niedostępny',
-        'produkt niedostępny',
+    'sold out',
-        'sold out',
+    'sold-out',
-        'sold-out',
+    'temporarily out of stock',
-        'temporarily out of stock',
+    'temporarily unavailable',
-        'temporarily unavailable',
+    'tickets unavailable',
-        'tickets unavailable',
+    'tijdelijk uitverkocht',
-        'tijdelijk uitverkocht',
+    'unavailable tickets',
-        'unavailable tickets',
+    'we do not currently have an estimate of when this product will be back in stock.',
-        'we do not currently have an estimate of when this product will be back in stock.',
+    'zur zeit nicht an lager',
-        'we don\'t know when or if this item will be back in stock.',
+    '品切れ',
-        'zur zeit nicht an lager',
+    '已售完',
-        '品切れ',
+    '품절'
-        '已售完',
+  ];
-        '품절'
-    ];
 
-    const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
-    function getElementBaseText(element) {
-        // .textContent can include text from children which may give the wrong results
-        // scan only immediate TEXT_NODEs, which will be a child of the element
-        var text = "";
-        for (var i = 0; i < element.childNodes.length; ++i)
-            if (element.childNodes[i].nodeType === Node.TEXT_NODE)
-                text += element.childNodes[i].textContent;
-        return text.toLowerCase().trim();
-    }
 
-    const negateOutOfStockRegex = new RegExp('([0-9] in stock|add to cart)', 'ig');
+  const negateOutOfStockRegexs = [
+      '[0-9] in stock'
+  ]
+  var negateOutOfStockRegexs_r = [];
+  for (let i = 0; i < negateOutOfStockRegexs.length; i++) {
+    negateOutOfStockRegexs_r.push(new RegExp(negateOutOfStockRegexs[0], 'g'));
+  }
 
-    // The out-of-stock or in-stock-text is generally always above-the-fold
-    // and often below-the-fold is a list of related products that may or may not contain trigger text
-    // so it's good to filter to just the 'above the fold' elements
-    // and it should be atleast 100px from the top to ignore items in the toolbar, sometimes menu items like "Coming soon" exist
-    const elementsToScan = Array.from(document.getElementsByTagName('*')).filter(element => element.getBoundingClientRect().top + window.scrollY <= vh && element.getBoundingClientRect().top + window.scrollY >= 100);
 
-    var elementText = "";
+  const elementsWithZeroChildren = Array.from(document.getElementsByTagName('*')).filter(element => element.children.length === 0);
 
-    // REGEXS THAT REALLY MEAN IT'S IN STOCK
+  // REGEXS THAT REALLY MEAN IT'S IN STOCK
-    for (let i = elementsToScan.length - 1; i >= 0; i--) {
+  for (let i = elementsWithZeroChildren.length - 1; i >= 0; i--) {
-        const element = elementsToScan[i];
+    const element = elementsWithZeroChildren[i];
-        elementText = "";
+    if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
-        if (element.tagName.toLowerCase() === "input") {
+      var elementText="";
-            elementText = element.value.toLowerCase();
+      if (element.tagName.toLowerCase() === "input") {
-        } else {
+        elementText = element.value.toLowerCase();
-            elementText = getElementBaseText(element);
+      } else {
-        }
+        elementText = element.textContent.toLowerCase();
-
+      }
-        if (elementText.length) {
+
-            // try which ones could mean its in stock
+      if (elementText.length) {
-            if (negateOutOfStockRegex.test(elementText)) {
+        // try which ones could mean its in stock
-                return 'Possibly in stock';
+        for (let i = 0; i < negateOutOfStockRegexs.length; i++) {
-            }
+          if (negateOutOfStockRegexs_r[i].test(elementText)) {
+            return 'Possibly in stock';
+          }
         }
+      }
     }
+  }
 
-    // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
+  // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
-    for (let i = elementsToScan.length - 1; i >= 0; i--) {
+  for (let i = elementsWithZeroChildren.length - 1; i >= 0; i--) {
-        const element = elementsToScan[i];
+    const element = elementsWithZeroChildren[i];
-        if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
+    if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
-            elementText = "";
+      var elementText="";
-            if (element.tagName.toLowerCase() === "input") {
+      if (element.tagName.toLowerCase() === "input") {
-                elementText = element.value.toLowerCase();
+        elementText = element.value.toLowerCase();
-            } else {
+      } else {
-                elementText = getElementBaseText(element);
+        elementText = element.textContent.toLowerCase();
-            }
+      }
 
-            if (elementText.length) {
+      if (elementText.length) {
-                // and these mean its out of stock
+        // and these mean its out of stock
-                for (const outOfStockText of outOfStockTexts) {
+        for (const outOfStockText of outOfStockTexts) {
-                    if (elementText.includes(outOfStockText)) {
+          if (elementText.includes(outOfStockText)) {
-                        return outOfStockText; // item is out of stock
+            return elementText; // item is out of stock
-                    }
+          }
-                }
-            }
         }
+      }
     }
+  }
 
-    return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
+  return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
 }
 
 // returns the element text that makes it think it's out of stock
-return isItemInStock().trim()
+return isItemInStock();
-

changedetectionio/static/images/steps.svg

@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   aria-hidden="true"
-   viewBox="0 0 19.966091 17.999964"
-   class="css-1oqmxjn"
-   version="1.1"
-   id="svg4"
-   sodipodi:docname="steps.svg"
-   width="19.966091"
-   height="17.999964"
-   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg">
-  <defs
-     id="defs8" />
-  <sodipodi:namedview
-     id="namedview6"
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1.0"
-     inkscape:pageshadow="2"
-     inkscape:pageopacity="0.0"
-     inkscape:pagecheckerboard="0"
-     showgrid="false"
-     fit-margin-top="0"
-     fit-margin-left="0"
-     fit-margin-right="0"
-     fit-margin-bottom="0"
-     inkscape:zoom="8.6354167"
-     inkscape:cx="-1.3896261"
-     inkscape:cy="6.1375151"
-     inkscape:window-width="1280"
-     inkscape:window-height="667"
-     inkscape:window-x="2419"
-     inkscape:window-y="250"
-     inkscape:window-maximized="0"
-     inkscape:current-layer="svg4" />
-  <path
-     d="m 16.95807,12.000003 c -0.7076,0.0019 -1.3917,0.2538 -1.9316,0.7113 -0.5398,0.4575 -0.9005,1.091 -1.0184,1.7887 H 5.60804 c -0.80847,0.0297 -1.60693,-0.1865 -2.29,-0.62 -0.26632,-0.1847 -0.48375,-0.4315 -0.63356,-0.7189 -0.14982,-0.2874 -0.22753,-0.607 -0.22644,-0.9311 -0.02843,-0.3931 0.03646,-0.7873 0.1894,-1.1505 0.15293,-0.3632 0.38957,-0.6851 0.6906,-0.9395 0.66628,-0.4559004 1.4637,-0.6807004 2.27,-0.6400004 h 8.35003 c 0.8515,-0.0223 1.6727,-0.3206 2.34,-0.85 0.3971,-0.3622 0.7076,-0.8091 0.9084,-1.3077 0.2008,-0.49857 0.2868,-1.03596 0.2516,-1.57229 0.0113,-0.47161 -0.0887,-0.93924 -0.292,-1.36493 -0.2033,-0.4257 -0.5041,-0.79745 -0.878,-1.08507 -0.7801,-0.55815 -1.7212,-0.84609 -2.68,-0.82 H 5.95804 c -0.12537,-0.7417 -0.5248,-1.40924 -1.11913,-1.87032996 -0.59434,-0.46108 -1.3402,-0.68207 -2.08979,-0.61917 -0.74958,0.06291 -1.44818,0.40512 -1.95736,0.95881 C 0.28259,1.5230126 0,2.2477926 0,3.0000126 c 0,0.75222 0.28259,1.47699 0.79176,2.03068 0.50918,0.55369 1.20778,0.8959 1.95736,0.95881 0.74959,0.0629 1.49545,-0.15808 2.08979,-0.61917 0.59433,-0.46109 0.99376,-1.12863 1.11913,-1.87032 h 7.70003 c 0.7353,-0.03061 1.4599,0.18397 2.06,0.61 0.2548,0.19335 0.4595,0.445 0.597,0.73385 0.1375,0.28884 0.2036,0.60644 0.193,0.92615 0.0316,0.38842 -0.0247,0.77898 -0.165,1.14258 -0.1402,0.36361 -0.3607,0.69091 -0.645,0.95741 -0.5713,0.4398 -1.2799,0.663 -2,0.63 H 5.69804 c -1.03259,-0.0462 -2.05065,0.2568 -2.89,0.86 -0.43755,0.3361 -0.78838,0.7720004 -1.02322,1.2712004 -0.23484,0.4993 -0.34688,1.0474 -0.32678,1.5988 -0.00726,0.484 0.10591,0.9622 0.32934,1.3916 0.22344,0.4295 0.55012,0.7966 0.95066,1.0684 0.85039,0.5592 1.85274,0.8421 2.87,0.81 h 8.40003 c 0.0954,0.5643 0.3502,1.0896 0.7343,1.5138 0.3842,0.4242 0.8817,0.7297 1.4338,0.8803 0.5521,0.1507 1.1358,0.1403 1.6822,-0.0299 0.5464,-0.1702 1.0328,-0.4932 1.4016,-0.9308 0.3688,-0.4376 0.6048,-0.9716 0.6801,-1.5389 0.0752,-0.5673 -0.0134,-1.1444 -0.2554,-1.663 -0.242,-0.5186 -0.6273,-0.9572 -1.1104,-1.264 -0.4831,-0.3068 -1.0439,-0.469 -1.6162,-0.4675 z m 0,5 c -0.3956,0 -0.7823,-0.1173 -1.1112,-0.3371 -0.3289,-0.2197 -0.5852,-0.5321 -0.7366,-0.8975 -0.1514,-0.3655 -0.191,-0.7676 -0.1138,-1.1556 0.0772,-0.3879 0.2677,-0.7443 0.5474,-1.024 0.2797,-0.2797 0.636,-0.4702 1.024,-0.5474 0.388,-0.0771 0.7901,-0.0375 1.1555,0.1138 0.3655,0.1514 0.6778,0.4078 0.8976,0.7367 0.2198,0.3289 0.3371,0.7155 0.3371,1.1111 0,0.5304 -0.2107,1.0391 -0.5858,1.4142 -0.3751,0.3751 -0.8838,0.5858 -1.4142,0.5858 z"
-     id="path2"
-     style="fill:#777777;fill-opacity:1" />
-</svg>

changedetectionio/static/styles/diff.css

@@ -126,8 +126,6 @@ html[data-darkmode="true"] {
   html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
   html[data-darkmode="true"] .watch-table .current-diff-url::after {
     filter: invert(0.5) hue-rotate(10deg) brightness(2); }
-  html[data-darkmode="true"] .watch-table .status-browsersteps {
-    filter: invert(0.5) hue-rotate(10deg) brightness(1.5); }
   html[data-darkmode="true"] .watch-table .watch-controls .state-off img {
     opacity: 0.3; }
   html[data-darkmode="true"] .watch-table .watch-controls .state-on img {

changedetectionio/static/styles/scss/parts/_variables.scss

@@ -152,10 +152,6 @@ html[data-darkmode="true"] {
       filter: invert(.5) hue-rotate(10deg) brightness(2);
     }
 
-    .status-browsersteps {
-      filter: invert(.5) hue-rotate(10deg) brightness(1.5);
-    }
-
     .watch-controls {
       .state-off {
         img {

changedetectionio/static/styles/styles.css

@@ -342,8 +342,6 @@ html[data-darkmode="true"] {
   html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
   html[data-darkmode="true"] .watch-table .current-diff-url::after {
     filter: invert(0.5) hue-rotate(10deg) brightness(2); }
-  html[data-darkmode="true"] .watch-table .status-browsersteps {
-    filter: invert(0.5) hue-rotate(10deg) brightness(1.5); }
   html[data-darkmode="true"] .watch-table .watch-controls .state-off img {
     opacity: 0.3; }
   html[data-darkmode="true"] .watch-table .watch-controls .state-on img {

changedetectionio/store.py

@@ -9,6 +9,7 @@ from copy import deepcopy, copy
 from os import path, unlink
 from threading import Lock
 import json
+import logging
 import os
 import re
 import requests
@@ -16,7 +17,6 @@ import secrets
 import threading
 import time
 import uuid as uuid_builder
-from loguru import logger
 
 # Because the server will run as a daemon and wont know the URL for notification links when firing off a notification
 BASE_URL_NOT_SET_TEXT = '("Base URL" not set - see settings - notifications)'
@@ -42,7 +42,7 @@ class ChangeDetectionStore:
         self.__data = App.model()
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
-        logger.info(f"Datastore path is '{self.json_store_path}'")
+        print(">>> Datastore path is ", self.json_store_path)
         self.needs_write = False
         self.start_time = time.time()
         self.stop_thread = False
@@ -83,12 +83,12 @@ class ChangeDetectionStore:
                 for uuid, watch in self.__data['watching'].items():
                     watch['uuid']=uuid
                     self.__data['watching'][uuid] = Watch.model(datastore_path=self.datastore_path, default=watch)
-                    logger.info(f"Watching: {uuid} {self.__data['watching'][uuid]['url']}")
+                    print("Watching:", uuid, self.__data['watching'][uuid]['url'])
 
         # First time ran, Create the datastore.
         except (FileNotFoundError):
             if include_default_watches:
-                logger.critical(f"No JSON DB found at {self.json_store_path}, creating JSON store at {self.datastore_path}")
+                print("No JSON DB found at {}, creating JSON store at {}".format(self.json_store_path, self.datastore_path))
                 self.add_watch(url='https://news.ycombinator.com/',
                                tag='Tech news',
                                extras={'fetch_backend': 'html_requests'})
@@ -139,7 +139,7 @@ class ChangeDetectionStore:
         save_data_thread = threading.Thread(target=self.save_datastore).start()
 
     def set_last_viewed(self, uuid, timestamp):
-        logger.debug(f"Setting watch UUID: {uuid} last viewed to {int(timestamp)}")
+        logging.debug("Setting watch UUID: {} last viewed to {}".format(uuid, int(timestamp)))
         self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
         self.needs_write = True
 
@@ -255,7 +255,6 @@ class ChangeDetectionStore:
                 'last_viewed': 0,
                 'previous_md5': False,
                 'previous_md5_before_filters': False,
-                'remote_server_reply': None,
                 'track_ldjson_price_data': None,
             })
 
@@ -317,7 +316,7 @@ class ChangeDetectionStore:
                             apply_extras['include_filters'] = [res['css_filter']]
 
             except Exception as e:
-                logger.error(f"Error fetching metadata for shared watch link {url} {str(e)}")
+                logging.error("Error fetching metadata for shared watch link", url, str(e))
                 flash("Error fetching metadata for {}".format(url), 'error')
                 return False
         from .model.Watch import is_safe_url
@@ -346,7 +345,7 @@ class ChangeDetectionStore:
 
         new_uuid = new_watch.get('uuid')
 
-        logger.debug(f"Adding URL {url} - {new_uuid}")
+        logging.debug("Added URL {} - {}".format(url, new_uuid))
 
         for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
             if k in apply_extras:
@@ -363,7 +362,7 @@ class ChangeDetectionStore:
         if write_to_disk_now:
             self.sync_to_json()
 
-        logger.debug(f"Added '{url}'")
+        print("added ", url)
 
         return new_uuid
 
@@ -417,13 +416,14 @@ class ChangeDetectionStore:
 
 
     def sync_to_json(self):
-        logger.info("Saving JSON..")
+        logging.info("Saving JSON..")
+        print("Saving JSON..")
         try:
             data = deepcopy(self.__data)
         except RuntimeError as e:
             # Try again in 15 seconds
             time.sleep(15)
-            logger.error(f"! Data changed when writing to JSON, trying again.. {str(e)}")
+            logging.error ("! Data changed when writing to JSON, trying again.. %s", str(e))
             self.sync_to_json()
             return
         else:
@@ -436,7 +436,7 @@ class ChangeDetectionStore:
                     json.dump(data, json_file, indent=4)
                 os.replace(self.json_store_path+".tmp", self.json_store_path)
             except Exception as e:
-                logger.error(f"Error writing JSON!! (Main JSON file save was skipped) : {str(e)}")
+                logging.error("Error writing JSON!! (Main JSON file save was skipped) : %s", str(e))
 
             self.needs_write = False
             self.needs_write_urgent = False
@@ -447,16 +447,7 @@ class ChangeDetectionStore:
 
         while True:
             if self.stop_thread:
-                # Suppressing "Logging error in Loguru Handler #0" during CICD.
+                print("Shutting down datastore thread")
-                # Not a meaningful difference for a real use-case just for CICD.
-                # the side effect is a "Shutting down datastore thread" message
-                # at the end of each test.
-                # But still more looking better.
-                import sys
-                logger.remove()
-                logger.add(sys.stderr)
-
-                logger.critical("Shutting down datastore thread")
                 return
 
             if self.needs_write or self.needs_write_urgent:
@@ -472,7 +463,7 @@ class ChangeDetectionStore:
     # Go through the datastore path and remove any snapshots that are not mentioned in the index
     # This usually is not used, but can be handy.
     def remove_unused_snapshots(self):
-        logger.info("Removing snapshots from datastore that are not in the index..")
+        print ("Removing snapshots from datastore that are not in the index..")
 
         index=[]
         for uuid in self.data['watching']:
@@ -485,7 +476,7 @@ class ChangeDetectionStore:
         for uuid in self.data['watching']:
             for item in pathlib.Path(self.datastore_path).rglob(uuid+"/*.txt"):
                 if not str(item) in index:
-                    logger.info(f"Removing {item}")
+                    print ("Removing",item)
                     unlink(item)
 
     @property
@@ -571,7 +562,7 @@ class ChangeDetectionStore:
             if os.path.isfile(filepath):
                 headers.update(parse_headers_from_text_file(filepath))
         except Exception as e:
-            logger.error(f"ERROR reading headers.txt at {filepath} {str(e)}")
+            print(f"ERROR reading headers.txt at {filepath}", str(e))
 
         watch = self.data['watching'].get(uuid)
         if watch:
@@ -582,7 +573,7 @@ class ChangeDetectionStore:
                 if os.path.isfile(filepath):
                     headers.update(parse_headers_from_text_file(filepath))
             except Exception as e:
-                logger.error(f"ERROR reading headers.txt at {filepath} {str(e)}")
+                print(f"ERROR reading headers.txt at {filepath}", str(e))
 
             # In /datastore/tag-name.txt
             tags = self.get_all_tags_for_watch(uuid=uuid)
@@ -593,7 +584,7 @@ class ChangeDetectionStore:
                     if os.path.isfile(filepath):
                         headers.update(parse_headers_from_text_file(filepath))
                 except Exception as e:
-                    logger.error(f"ERROR reading headers.txt at {filepath} {str(e)}")
+                    print(f"ERROR reading headers.txt at {filepath}", str(e))
 
         return headers
 
@@ -611,13 +602,13 @@ class ChangeDetectionStore:
     def add_tag(self, name):
         # If name exists, return that
         n = name.strip().lower()
-        logger.debug(f">>> Adding new tag - '{n}'")
+        print (f">>> Adding new tag - '{n}'")
         if not n:
             return False
 
         for uuid, tag in self.__data['settings']['application'].get('tags', {}).items():
             if n == tag.get('title', '').lower().strip():
-                logger.warning(f"Tag '{name}' already exists, skipping creation.")
+                print (f">>> Tag {name} already exists")
                 return uuid
 
         # Eventually almost everything todo with a watch will apply as a Tag
@@ -679,7 +670,7 @@ class ChangeDetectionStore:
         updates_available = self.get_updates_available()
         for update_n in updates_available:
             if update_n > self.__data['settings']['application']['schema_version']:
-                logger.critical(f"Applying update_{update_n}")
+                print ("Applying update_{}".format((update_n)))
                 # Wont exist on fresh installs
                 if os.path.exists(self.json_store_path):
                     shutil.copyfile(self.json_store_path, self.datastore_path+"/url-watches-before-{}.json".format(update_n))
@@ -687,8 +678,8 @@ class ChangeDetectionStore:
                 try:
                     update_method = getattr(self, "update_{}".format(update_n))()
                 except Exception as e:
-                    logger.error(f"Error while trying update_{update_n}")
+                    print("Error while trying update_{}".format((update_n)))
-                    logger.error(e)
+                    print(e)
                     # Don't run any more updates
                     return
                 else:
@@ -726,7 +717,7 @@ class ChangeDetectionStore:
                         with open(os.path.join(target_path, "history.txt"), "w") as f:
                             f.writelines(history)
                     else:
-                        logger.warning(f"Datastore history directory {target_path} does not exist, skipping history import.")
+                        logging.warning("Datastore history directory {} does not exist, skipping history import.".format(target_path))
 
                 # No longer needed, dynamically pulled from the disk when needed.
                 # But we should set it back to a empty dict so we don't break if this schema runs on an earlier version.

changedetectionio/templates/_common_fields.jinja

@@ -115,12 +115,6 @@
 									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
                                     For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
                                     </p>
-                                    <p>
-                                        For JSON payloads, use <strong>|tojson</strong> without quotes for automatic escaping, for example - <code>{ "name": {{ '{{ watch_title|tojson }}' }} }</code>
-                                    </p>
-                                    <p>
-                                        URL encoding, use <strong>|urlencode</strong>, for example - <code>gets://hook-website.com/test.php?title={{ '{{ watch_title|urlencode }}' }}</code>
-                                    </p>
                                 </div>
                             </div>
                             <div class="pure-control-group">

changedetectionio/templates/edit.html

@@ -401,7 +401,6 @@ Unavailable") }}
                                 <li>Use <code>//(?aiLmsux))</code> type flags (more <a href="https://docs.python.org/3/library/re.html#index-15">information here</a>)<br></li>
                                 <li>Keyword example &dash; example <code>Out of stock</code></li>
                                 <li>Use groups to extract just that text &dash; example <code>/reports.+?(\d+)/i</code> returns a list of years only</li>
-                                <li>Example - match lines containing a keyword <code>/.*icecream.*/</code></li>
                             </ul>
                         </li>
                         <li>One line per regular-expression/string match</li>

changedetectionio/templates/watch-overview.html

@@ -110,7 +110,6 @@
                     {% endif %}
 
                     {%if watch.is_pdf  %}<img class="status-icon" src="{{url_for('static_content', group='images', filename='pdf-icon.svg')}}" title="Converting PDF to text" >{% endif %}
-                    {% if watch.has_browser_steps %}<img class="status-icon status-browsersteps" src="{{url_for('static_content', group='images', filename='steps.svg')}}" title="Browser Steps is enabled" >{% endif %}
                     {% if watch.last_error is defined and watch.last_error != False %}
                     <div class="fetch-error">{{ watch.last_error }}
 

changedetectionio/tests/conftest.py

@@ -4,8 +4,6 @@ import pytest
 from changedetectionio import changedetection_app
 from changedetectionio import store
 import os
-import sys
-from loguru import logger
 
 # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
 # Much better boilerplate than the docs
@@ -13,15 +11,6 @@ from loguru import logger
 
 global app
 
-# https://loguru.readthedocs.io/en/latest/resources/migration.html#replacing-caplog-fixture-from-pytest-library
-# Show loguru logs only if CICD pytest fails.
-from loguru import logger
-@pytest.fixture
-def reportlog(pytestconfig):
-    logging_plugin = pytestconfig.pluginmanager.getplugin("logging-plugin")
-    handler_id = logger.add(logging_plugin.report_handler, format="{message}")
-    yield
-    logger.remove(handler_id)
 
 def cleanup(datastore_path):
     import glob
@@ -52,18 +41,6 @@ def app(request):
 
     app_config = {'datastore_path': datastore_path, 'disable_checkver' : True}
     cleanup(app_config['datastore_path'])
-
-    logger_level = 'TRACE'
-
-    logger.remove()
-    log_level_for_stdout = { 'DEBUG', 'SUCCESS' }
-    logger.configure(handlers=[
-        {"sink": sys.stdout, "level": logger_level,
-         "filter" : lambda record: record['level'].name in log_level_for_stdout},
-        {"sink": sys.stderr, "level": logger_level,
-         "filter": lambda record: record['level'].name not in log_level_for_stdout},
-        ])
-
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
     app = changedetection_app(app_config, datastore)
 

changedetectionio/tests/fetchers/test_content.py

@@ -37,4 +37,4 @@ def test_fetch_webdriver_content(client, live_server):
     )
     logging.getLogger().info("Looking for correct fetched HTML (text) from server")
 
-    assert b'cool it works' in res.data
+    assert b'cool it works' in res.data

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -97,17 +97,6 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     set_original_response()
     global smtp_test_server
     notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-    notification_body = f"""<!DOCTYPE html>
-<html lang="en">
-<head>
-    <title>My Webpage</title>
-</head>
-<body>
-    <h1>Test</h1>
-    {default_notification_body}
-</body>
-</html>
-"""
 
     #####################
     # Set this up for when we remove the notification from the watch, it should fallback with these details
@@ -115,7 +104,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
         url_for("settings_page"),
         data={"application-notification_urls": notification_url,
               "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": notification_body,
+              "application-notification_body": default_notification_body,
               "application-notification_format": 'Text',
               "requests-time_between_check-minutes": 180,
               'application-fetch_backend': "html_requests"},
@@ -172,10 +161,5 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv
     assert 'Content-Type: text/html' in msg
     assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
 
-    # https://github.com/dgtlmoon/changedetection.io/issues/2103
-    assert '<h1>Test</h1>' in msg
-    assert '&lt;' not in msg
-    assert 'Content-Type: text/html' in msg
-
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_api.py

@@ -163,7 +163,6 @@ def test_api_simple(client, live_server):
     # Loading the most recent snapshot should force viewed to become true
     client.get(url_for("diff_history_page", uuid="first"), follow_redirects=True)
 
-    time.sleep(3)
     # Fetch the whole watch again, viewed should be true
     res = client.get(
         url_for("watch", uuid=watch_uuid),

changedetectionio/tests/test_backup.py

@@ -1,8 +1,8 @@
 #!/usr/bin/python3
 
-from .util import set_original_response, live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 from flask import url_for
-import io
+from urllib.request import urlopen
 from zipfile import ZipFile
 import re
 import time
@@ -37,10 +37,15 @@ def test_backup(client, live_server):
     # Should be PK/ZIP stream
     assert res.data.count(b'PK') >= 2
 
-    backup = ZipFile(io.BytesIO(res.data))
+    # ZipFile from buffer seems non-obvious, just save it instead
-    l = backup.namelist()
+    with open("download.zip", 'wb') as f:
+        f.write(res.data)
+
+    zip = ZipFile('download.zip')
+    l = zip.namelist()
     uuid4hex = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.*txt', re.I)
     newlist = list(filter(uuid4hex.match, l))  # Read Note below
 
     # Should be two txt files in the archive (history and the snapshot)
     assert len(newlist) == 2
+

changedetectionio/tests/test_notification_errors.py

@@ -1,7 +1,8 @@
 import os
 import time
+import re
 from flask import url_for
-from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
+from . util import set_original_response, set_modified_response, live_server_setup
 import logging
 
 def test_check_notification_error_handling(client, live_server):
@@ -10,7 +11,7 @@ def test_check_notification_error_handling(client, live_server):
     set_original_response()
 
     # Give the endpoint time to spin up
-    time.sleep(1)
+    time.sleep(2)
 
     # Set a URL and fetch it, then set a notification URL which is going to give errors
     test_url = url_for('test_endpoint', _external=True)
@@ -21,16 +22,12 @@ def test_check_notification_error_handling(client, live_server):
     )
     assert b"Watch added" in res.data
 
-    wait_for_all_checks(client)
+    time.sleep(2)
     set_modified_response()
 
-    working_notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
-    broken_notification_url = "jsons://broken-url-xxxxxxxx123/test"
-
     res = client.post(
         url_for("edit_page", uuid="first"),
-        # A URL with errors should not block the one that is working
+        data={"notification_urls": "jsons://broken-url-xxxxxxxx123/test",
-        data={"notification_urls": f"{broken_notification_url}\r\n{working_notification_url}",
               "notification_title": "xxx",
               "notification_body": "xxxxx",
               "notification_format": "Text",
@@ -66,10 +63,4 @@ def test_check_notification_error_handling(client, live_server):
     found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
     assert found_name_resolution_error
 
-    # And the working one, which is after the 'broken' one should still have fired
-    with open("test-datastore/notification.txt", "r") as f:
-        notification_submission = f.read()
-    os.unlink("test-datastore/notification.txt")
-    assert 'xxxxx' in notification_submission
-
     client.get(url_for("form_delete", uuid="all"), follow_redirects=True)

changedetectionio/tests/test_request.py

@@ -10,7 +10,7 @@ def test_setup(live_server):
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_headers_in_request(client, live_server):
-    #ve_server_setup(live_server)
+    #live_server_setup(live_server)
     # Add our URL to the import page
     test_url = url_for('test_headers', _external=True)
     if os.getenv('PLAYWRIGHT_DRIVER_URL'):
@@ -70,17 +70,16 @@ def test_headers_in_request(client, live_server):
 
     wait_for_all_checks(client)
 
-    # Re #137 -  It should have only one set of headers entered
+    # Re #137 -  Examine the JSON index file, it should have only one set of headers entered
     watches_with_headers = 0
-    for k, watch in client.application.config.get('DATASTORE').data.get('watching').items():
+    with open('test-datastore/url-watches.json') as f:
-            if (len(watch['headers'])):
+        app_struct = json.load(f)
+        for uuid in app_struct['watching']:
+            if (len(app_struct['watching'][uuid]['headers'])):
                 watches_with_headers += 1
-    assert watches_with_headers == 1
-
-    # 'server' http header was automatically recorded
-    for k, watch in client.application.config.get('DATASTORE').data.get('watching').items():
-        assert 'custom' in watch.get('remote_server_reply') # added in util.py
 
+    # Should be only one with headers set
+    assert watches_with_headers==1
     res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 

changedetectionio/tests/util.py

@@ -175,16 +175,12 @@ def live_server_setup(live_server):
     @live_server.app.route('/test-headers')
     def test_headers():
 
-        output = []
+        output= []
 
         for header in request.headers:
-            output.append("{}:{}".format(str(header[0]), str(header[1])))
+             output.append("{}:{}".format(str(header[0]),str(header[1])   ))
 
-        content = "\n".join(output)
+        return "\n".join(output)
-
-        resp = make_response(content, 200)
-        resp.headers['server'] = 'custom'
-        return resp
 
     # Just return the body in the request
     @live_server.app.route('/test-body', methods=['POST', 'GET'])

changedetectionio/update_worker.py

@@ -12,13 +12,14 @@ from .processors.restock_diff import UnableToExtractRestockData
 # Requests for checking on a single site(watch) from a queue of watches
 # (another process inserts watches into the queue that are time-ready for checking)
 
+import logging
 import sys
-from loguru import logger
 
 class update_worker(threading.Thread):
     current_uuid = None
 
     def __init__(self, q, notification_q, app, datastore, *args, **kwargs):
+        logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
         self.q = q
         self.app = app
         self.notification_q = notification_q
@@ -31,8 +32,6 @@ class update_worker(threading.Thread):
         dates = []
         trigger_text = ''
 
-        now = time.time()
-
         if watch:
             watch_history = watch.history
             dates = list(watch_history.keys())
@@ -74,14 +73,13 @@ class update_worker(threading.Thread):
             'diff_full': diff.render_diff(prev_snapshot, current_snapshot, include_equal=True, line_feed_sep=line_feed_sep),
             'diff_patch': diff.render_diff(prev_snapshot, current_snapshot, line_feed_sep=line_feed_sep, patch_format=True),
             'diff_removed': diff.render_diff(prev_snapshot, current_snapshot, include_added=False, line_feed_sep=line_feed_sep),
-            'notification_timestamp': now,
             'screenshot': watch.get_screenshot() if watch and watch.get('notification_screenshot') else None,
             'triggered_text': triggered_text,
             'uuid': watch.get('uuid') if watch else None,
             'watch_url': watch.get('url') if watch else None,
         })
-        logger.trace(f"Main rendered notification placeholders (diff_added etc) calculated in {time.time()-now:.3f}s")
+        logging.info(">> SENDING NOTIFICATION")
-        logger.debug("Queued notification for sending")
+
         notification_q.put(n_object)
 
     # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
@@ -182,7 +180,7 @@ class update_worker(threading.Thread):
                 'screenshot': None
             })
             self.notification_q.put(n_object)
-            logger.error(f"Sent filter not found notification for {watch_uuid}")
+            print("Sent filter not found notification for {}".format(watch_uuid))
 
     def send_step_failure_notification(self, watch_uuid, step_n):
         watch = self.datastore.data['watching'].get(watch_uuid, False)
@@ -190,9 +188,9 @@ class update_worker(threading.Thread):
             return
         threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
         n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1),
-                    'notification_body': "Your configured browser step at position {} for {{{{watch_url}}}} "
+                    'notification_body': "Your configured browser step at position {} for {{watch['url']}} "
                                          "did not appear on the page after {} attempts, did the page change layout? "
-                                         "Does it need a delay added?\n\nLink: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\n"
+                                         "Does it need a delay added?\n\nLink: {{base_url}}/edit/{{watch_uuid}}\n\n"
                                          "Thanks - Your omniscient changedetection.io installation :)\n".format(step_n+1, threshold),
                     'notification_format': 'text'}
 
@@ -209,7 +207,7 @@ class update_worker(threading.Thread):
                 'uuid': watch_uuid
             })
             self.notification_q.put(n_object)
-            logger.error(f"Sent step not found notification for {watch_uuid}")
+            print("Sent step not found notification for {}".format(watch_uuid))
 
 
     def cleanup_error_artifacts(self, uuid):
@@ -223,8 +221,7 @@ class update_worker(threading.Thread):
     def run(self):
 
         from .processors import text_json_diff, restock_diff
-        now = time.time()
+
-        
         while not self.app.config.exit.is_set():
             update_handler = None
 
@@ -236,14 +233,14 @@ class update_worker(threading.Thread):
             else:
                 uuid = queued_item_data.item.get('uuid')
                 self.current_uuid = uuid
+
                 if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
                     changed_detected = False
                     contents = b''
                     process_changedetection_results = True
                     update_obj = {}
-                    logger.info(f"Processing watch UUID {uuid} "
+                    print("> Processing UUID {} Priority {} URL {}".format(uuid, queued_item_data.priority,
-                            f"Priority {queued_item_data.priority} "
+                                                                           self.datastore.data['watching'][uuid]['url']))
-                            f"URL {self.datastore.data['watching'][uuid]['url']}")
                     now = time.time()
 
                     try:
@@ -283,8 +280,7 @@ class update_worker(threading.Thread):
                         if not isinstance(contents, (bytes, bytearray)):
                             raise Exception("Error - returned data from the fetch handler SHOULD be bytes")
                     except PermissionError as e:
-                        logger.critical(f"File permission error updating file, watch: {uuid}")
+                        self.app.logger.error("File permission error updating", uuid, str(e))
-                        logger.critical(str(e))
                         process_changedetection_results = False
                     except content_fetcher.ReplyWithContentButNoText as e:
                         # Totally fine, it's by choice - just continue on, nothing more to care about
@@ -342,7 +338,7 @@ class update_worker(threading.Thread):
                             # Send notification if we reached the threshold?
                             threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
                                                                                            0)
-                            logger.error(f"Filter for {uuid} not found, consecutive_filter_failures: {c}")
+                            print("Filter for {} not found, consecutive_filter_failures: {}".format(uuid, c))
                             if threshold > 0 and c >= threshold:
                                 if not self.datastore.data['watching'][uuid].get('notification_muted'):
                                     self.send_filter_failure_notification(uuid)
@@ -376,7 +372,7 @@ class update_worker(threading.Thread):
                             # Other Error, more info is good.
                             err_text += " " + str(e.original_e).splitlines()[0]
 
-                        logger.debug(f"BrowserSteps exception at step {error_step} {str(e.original_e)}")
+                        print(f"BrowserSteps exception at step {error_step}", str(e.original_e))
 
                         self.datastore.update_watch(uuid=uuid,
                                                     update_obj={'last_error': err_text,
@@ -390,7 +386,7 @@ class update_worker(threading.Thread):
                             # Send notification if we reached the threshold?
                             threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
                                                                                            0)
-                            logger.error(f"Step for {uuid} not found, consecutive_filter_failures: {c}")
+                            print("Step for {} not found, consecutive_filter_failures: {}".format(uuid, c))
                             if threshold > 0 and c >= threshold:
                                 if not self.datastore.data['watching'][uuid].get('notification_muted'):
                                     self.send_step_failure_notification(watch_uuid=uuid, step_n=e.step_n)
@@ -432,13 +428,11 @@ class update_worker(threading.Thread):
                         process_changedetection_results = False
                     except UnableToExtractRestockData as e:
                         # Usually when fetcher.instock_data returns empty
-                        logger.error(f"Exception (UnableToExtractRestockData) reached processing watch UUID: {uuid}")
+                        self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
-                        logger.error(str(e))
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': f"Unable to extract restock data for this page unfortunately. (Got code {e.status_code} from server)"})
                         process_changedetection_results = False
                     except Exception as e:
-                        logger.error(f"Exception reached processing watch UUID: {uuid}")
+                        self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
-                        logger.error(str(e))
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
                         # Other serious error
                         process_changedetection_results = False
@@ -474,33 +468,23 @@ class update_worker(threading.Thread):
 
                             # A change was detected
                             if changed_detected:
+                                print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))
+
                                 # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
                                 if watch.history_n >= 2:
-                                    logger.info(f"Change detected in UUID {uuid} - {watch['url']}")
                                     if not self.datastore.data['watching'][uuid].get('notification_muted'):
                                         self.send_content_changed_notification(watch_uuid=uuid)
-                                else:
+
-                                    logger.info(f"Change triggered in UUID {uuid} due to first history saving (no notifications sent) - {watch['url']}")
 
                         except Exception as e:
                             # Catch everything possible here, so that if a worker crashes, we don't lose it until restart!
-                            logger.critical("!!!! Exception in update_worker while processing process_changedetection_results !!!")
+                            print("!!!! Exception in update_worker !!!\n", e)
-                            logger.critical(str(e))
+                            self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
                             self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
 
                     if self.datastore.data['watching'].get(uuid):
                         # Always record that we atleast tried
                         count = self.datastore.data['watching'][uuid].get('check_count', 0) + 1
-
-                        # Record the 'server' header reply, can be used for actions in the future like cloudflare/akamai workarounds
-                        try:
-                            server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
-                            self.datastore.update_watch(uuid=uuid,
-                                                        update_obj={'remote_server_reply': server_header}
-                                                        )
-                        except Exception as e:
-                            pass
-
                         self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
                                                                            'last_checked': round(time.time()),
                                                                            'check_count': count
@@ -515,7 +499,6 @@ class update_worker(threading.Thread):
 
                 self.current_uuid = None  # Done
                 self.q.task_done()
-                logger.debug(f"Watch {uuid} done in {time.time()-now:.2f}s")
 
                 # Give the CPU time to interrupt
                 time.sleep(0.1)

docker-compose.yml

@@ -16,10 +16,6 @@ services:
   #      - PUID=1000
   #      - PGID=1000
   #
-  #        Log levels are in descending order. (TRACE is the most detailed one)
-  #        Log output levels: TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL
-  #      - LOGGER_LEVEL=DEBUG
-  #
   #       Alternative WebDriver/selenium URL, do not use "'s or 's!
   #      - WEBDRIVER_URL=http://browser-chrome:4444/wd/hub
   #
@@ -94,9 +90,7 @@ services:
 #
 
      # Used for fetching pages via Playwright+Chrome where you need Javascript support.
-     # Note: Works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector)
+     # Note: works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector) and other issues
-     #       Does not report status codes (200, 404, 403) and other issues
-     # More information about the advantages of playwright/browserless https://www.browserless.io/blog/2023/12/13/migrating-selenium-to-playwright/
 #    browser-chrome:
 #        hostname: browser-chrome
 #        image: selenium/standalone-chrome:4

heroku.yml

@@ -0,0 +1,2 @@
+run:
+  changedetection: python3 ./changedetection.py -C -d ./datastore -p $PORT

requirements.txt

@@ -72,5 +72,3 @@ pytest-flask ~=1.2
 
 # Pin jsonschema version to prevent build errors on armv6 while rpds-py wheels aren't available (1708)
 jsonschema==4.17.3
-
-loguru

setup.py

@@ -27,7 +27,7 @@ install_requires = open('requirements.txt').readlines()
 setup(
     name='changedetection.io',
     version=find_version("changedetectionio", "__init__.py"),
-    description='Website change detection and monitoring service, detect changes to web pages and send alerts/notifications.',
+    description='Website change detection and monitoring service',
     long_description=open('README-pip.md').read(),
     long_description_content_type='text/markdown',
     keywords='website change monitor for changes notification change detection '
@@ -41,7 +41,7 @@ setup(
     include_package_data=True,
     install_requires=install_requires,
     license="Apache License 2.0",
-    python_requires=">= 3.10",
+    python_requires=">= 3.7",
     classifiers=['Intended Audience :: Customer Service',
                  'Intended Audience :: Developers',
                  'Intended Audience :: Education',