Adds a [Add email] button to notification settings with a prefix set from NOTIFICATION_MAIL_BUTTON_PREFIX env var when defined.

Co-authored-by: Ara Hayrabedian <ara.hayrabedian@gmail.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,41 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Version**
+In the top right area: 0....
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS] 
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,23 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+**Version and OS**
+For example, 0.123 on linux/docker
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe the use-case and give concrete real-world examples**
+Attach any HTML/JSON, give links to sites, screenshots etc, we are not mind readers
+
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/containers.yml

@@ -0,0 +1,129 @@
+name: Build and push containers
+
+on:
+  # Automatically triggered by a testing workflow passing, but this is only checked when it lands in the `master`/default branch
+#  workflow_run:
+#    workflows: ["ChangeDetection.io Test"]
+#    branches: [master]
+#    tags: ['0.*']
+#    types: [completed]
+
+  # Or a new tagged release
+  release:
+    types: [published, edited]
+
+  push:
+    branches:
+      - master
+
+jobs:
+  metadata:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Show metadata
+      run: |
+        echo SHA ${{ github.sha }}
+        echo github.ref:  ${{ github.ref }}
+        echo github_ref: $GITHUB_REF
+        echo Event name: ${{ github.event_name }}
+        echo Ref ${{ github.ref }}
+        echo c: ${{ github.event.workflow_run.conclusion }}
+        echo r: ${{ github.event.workflow_run }}
+        echo tname: "${{ github.event.release.tag_name }}"
+        echo headbranch: -${{ github.event.workflow_run.head_branch }}-
+        set
+
+  build-push-containers:
+    runs-on: ubuntu-latest
+    # If the testing workflow has a success, then we build to :latest
+    # Or if we are in a tagged release scenario.
+    if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+      - name: Create release metadata
+        run: |
+          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > changedetectionio/source.txt
+          echo ${{ github.ref }} > changedetectionio/tag.txt
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: all
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to Docker Hub Container Registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+          version: latest
+          driver-opts: image=moby/buildkit:master
+
+      # master always builds :latest
+      - name: Build and push :latest
+        id: docker_build
+        if: ${{ github.ref }} == "refs/heads/master"
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest,ghcr.io/${{ github.repository }}:latest
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+
+      # A new tagged release is required, which builds :tag
+      - name: Build and push :tag
+        id: docker_build_tag_release
+        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }},ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+
+      - name: Image digest
+        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+

.github/workflows/image-javascript.yml

@@ -1,88 +0,0 @@
-name: Javascript/Webdriver support - Test, build and push to Docker Hub :javascript tag
-
-on:
-  push:
-    branches: [  javascript-browser ]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install flake8 pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-
-      - name: Create release metadata
-        run: |
-          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
-          echo ${{ github.sha }} > changedetectionio/source.txt
-          echo ${{ github.ref }} > changedetectionio/tag.txt
-
-      - name: Test with pytest
-        run: |
-          # Each test is totally isolated and performs its own cleanup/reset
-          cd changedetectionio; ./run_all_tests.sh
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          image: tonistiigi/binfmt:latest
-          platforms: all
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-          version: latest
-          driver-opts: image=moby/buildkit:master
-
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./
-          file: ./Dockerfile
-          push: true
-          tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:javascript-dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
-
-      - name: Image digest
-        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
-
-# failed: Cache service responded with 503
-#      - name: Cache Docker layers
-#        uses: actions/cache@v2
-#        with:
-#          path: /tmp/.buildx-cache
-#          key: ${{ runner.os }}-buildx-${{ github.sha }}
-#          restore-keys: |
-#            ${{ runner.os }}-buildx-
-
-

.github/workflows/image-tag.yml

@@ -1,92 +0,0 @@
-name: Test, build and push tagged release to Docker Hub
-
-on:
-  push:
-    tags:
-      - '*.*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - uses: olegtarasov/get-tag@v2.1
-        id: tagName
-
-#        with:
-#          tagRegex: "foobar-(.*)"  # Optional. Returns specified group text as tag name. Full tag string is returned if regex is not defined.
-#          tagRegexGroup: 1 # Optional. Default is 1.
-
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install flake8 pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-
-      - name: Create release metadata
-        run: |
-          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
-          echo ${{ github.sha }} > changedetectionio/source.txt
-          echo ${{ github.ref }} > changedetectionio/tag.txt
-
-      - name: Test with pytest
-        run: |
-          # Each test is totally isolated and performs its own cleanup/reset
-          cd changedetectionio; ./run_all_tests.sh
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          image: tonistiigi/binfmt:latest
-          platforms: all
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-          version: latest
-          driver-opts: image=moby/buildkit:master
-
-      - name: tag
-        run : echo ${{ github.event.release.tag_name }}
-
-      - name: Build and push tagged version
-        id: docker_build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./
-          file: ./Dockerfile
-          push: true
-          tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ steps.tagName.outputs.tag }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
-        env:
-            SOURCE_NAME: ${{ steps.branch_name.outputs.SOURCE_NAME }}
-            SOURCE_BRANCH: ${{ steps.branch_name.outputs.SOURCE_BRANCH }}
-            SOURCE_TAG: ${{ steps.branch_name.outputs.SOURCE_TAG }}
-
-      - name: Image digest
-        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}

.github/workflows/image.yml

@@ -1,88 +0,0 @@
-name: Test, build and push to Docker Hub
-
-on:
-  push:
-    branches: [ master, arm-build ]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install flake8 pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-
-      - name: Create release metadata
-        run: |
-          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
-          echo ${{ github.sha }} > changedetectionio/source.txt
-          echo ${{ github.ref }} > changedetectionio/tag.txt
-
-      - name: Test with pytest
-        run: |
-          # Each test is totally isolated and performs its own cleanup/reset
-          cd changedetectionio; ./run_all_tests.sh
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          image: tonistiigi/binfmt:latest
-          platforms: all
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-          version: latest
-          driver-opts: image=moby/buildkit:master
-
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./
-          file: ./Dockerfile
-          push: true
-          tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
-
-      - name: Image digest
-        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
-
-# failed: Cache service responded with 503
-#      - name: Cache Docker layers
-#        uses: actions/cache@v2
-#        with:
-#          path: /tmp/.buildx-cache
-#          key: ${{ runner.os }}-buildx-${{ github.sha }}
-#          restore-keys: |
-#            ${{ runner.os }}-buildx-
-
-

.github/workflows/pypi.yml

@@ -0,0 +1,44 @@
+name: PyPi Test and Push tagged release
+
+# Triggers the workflow on push or pull request events
+on:
+  workflow_run:
+    workflows: ["ChangeDetection.io Test"]
+    tags: '*.*'
+    types: [completed]
+
+
+jobs:
+  test-build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+#      - name: Install dependencies
+#        run: |
+#          python -m pip install --upgrade pip
+#          pip install flake8 pytest
+#          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+#          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+      - name: Test that pip builds without error
+        run: |
+          pip3 --version
+          python3 -m pip install wheel
+          python3 setup.py bdist_wheel
+          python3 -m pip install dist/changedetection.io-*-none-any.whl --force
+          changedetection.io -d /tmp -p 10000 &
+          sleep 3
+          curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+          killall -9 changedetection.io
+
+      # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
+      # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
+
+      # https://github.com/docker/buildx/issues/495#issuecomment-918925854
+#if: ${{ github.event_name == 'release'}}

.github/workflows/test-only.yml

@@ -4,7 +4,7 @@ name: ChangeDetection.io Test
 on: [push, pull_request]
 
 jobs:
-  build:
+  test-build:
     runs-on: ubuntu-latest
     steps:
 
@@ -14,6 +14,9 @@ jobs:
         with:
           python-version: 3.9
 
+      - name: Show env vars
+        run: set
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -27,20 +30,16 @@ jobs:
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
+      - name: Unit tests
+        run: |
+          python3 -m unittest changedetectionio.tests.unit.test_notification_diff
+
       - name: Test with pytest
         run: |
           # Each test is totally isolated and performs its own cleanup/reset
           cd changedetectionio; ./run_all_tests.sh
 
-      - name: Test that pip builds without error
-        run: |
-          pip3 --version
-          python3 -m pip install wheel
-          python3 setup.py bdist_wheel
-          python3 -m pip install dist/changedetection.io-*-none-any.whl --force
-          changedetection.io -d /tmp -p 10000 &
-          sleep 3
-          curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
-          killall -9 changedetection.io
-
+      # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
+      # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
 
+      # https://github.com/docker/buildx/issues/495#issuecomment-918925854

.gitignore

@@ -7,4 +7,6 @@ __pycache__
 .pytest_cache
 build
 dist
+venv
+*.egg-info*
 .vscode/settings.json

CONTRIBUTING.md

@@ -0,0 +1,15 @@
+Contributing is always welcome!
+
+I am no professional flask developer, if you know a better way that something can be done, please let me know!
+
+Otherwise, it's always best to PR into the `dev` branch.
+
+Please be sure that all new functionality has a matching test!
+
+Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notifications.py` for example
+
+```
+pip3 install -r requirements-dev
+```
+
+this is from https://github.com/dgtlmoon/changedetection.io/blob/master/requirements-dev.txt

Dockerfile

@@ -42,10 +42,15 @@ ENV PYTHONUNBUFFERED=1
 
 RUN [ ! -d "/datastore" ] && mkdir /datastore
 
+# Re #80, sets SECLEVEL=1 in openssl.conf to allow monitoring sites with weak/old cipher suites
+RUN sed -i 's/^CipherString = .*/CipherString = DEFAULT@SECLEVEL=1/' /etc/ssl/openssl.cnf
+
 # Copy modules over to the final image and add their dir to PYTHONPATH
 COPY --from=builder /dependencies /usr/local
 ENV PYTHONPATH=/usr/local
 
+EXPOSE 5000
+
 # The actual flask app
 COPY changedetectionio /app/changedetectionio
 # The eventlet server wrapper

MANIFEST.in

@@ -1,4 +1,6 @@
 recursive-include changedetectionio/templates *
 recursive-include changedetectionio/static *
 include changedetection.py
-global-exclude *.pyc
+global-exclude *.pyc
+global-exclude node_modules
+global-exclude venv

Procfile

@@ -0,0 +1 @@
+web: python3 ./changedetection.py -C -d ./datastore -p $PORT

README.md

@@ -1,49 +1,67 @@
 #  changedetection.io
-![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
-<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
-  <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
-</a>
-<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
-  <img src="https://img.shields.io/github/v/release/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
-</a>
+[![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)
 
-## Self-hosted open source change monitoring of web pages.
+![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
+
+## Self-Hosted, Open Source, Change Monitoring of Web Pages
 
 _Know when web pages change! Stay ontop of new information!_ 
 
-Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
+Live your data-life *pro-actively* instead of *re-actively*.
 
-Open source web page monitoring, notification and change detection.
+Free, Open-source web page monitoring, notification and change detection. Don't have time? [Try our $6.99/month plan - unlimited checks and watches!](https://lemonade.changedetection.io/start)
 
 
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start)
+
+
+**Get your own private instance now! Let us host it for you!**
+
+[![Deploy to Lemonade](https://lemonade.changedetection.io/static/images/lemonade.svg)](https://lemonade.changedetection.io/start)
+
+
+[_Let us host your own private instance - We accept PayPal and Bitcoin, Support the further development of changedetection.io!_](https://lemonade.changedetection.io/start)
+
+
+
+- Automatic Updates, Automatic Backups, No Heroku "paused application", don't miss a change!
+- Javascript browser included
+- Unlimited checks and watches!
+
 
 #### Example use cases
 
-Know when ...
-
-- Government department updates (changes are often only on their websites)
-- Local government news (changes are often only on their websites)
+- Products and services have a change in pricing
+- Governmental department updates (changes are often only on their websites)
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
 - Realestate listing changes
 - COVID related news from government websites
+- University/organisation news from their website
 - Detect and monitor changes in JSON API responses 
 - API monitoring and alerting
-
+- Changes in legal and other documents
+- Trigger API calls via notifications when text appears on a website
+- Glue together APIs using the JSON filter and JSON notifications
+- Create RSS feeds based on changes in web content
+- You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
+ 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver!</a>_
 
-**Get monitoring now! super simple, one command!**
+## Screenshots
+
+Examining differences in content.
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
+
+Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
-Run the python code on your own machine by cloning this repository, or with <a href="https://docs.docker.com/get-docker/">docker</a> and/or <a href="https://www.digitalocean.com/community/tutorial_collections/how-to-install-docker-compose">docker-compose</a>
 
 ## Installation
 
 ### Docker
 
-Check out our Docker hub page https://hub.docker.com/r/dgtlmoon/changedetection.io
-
-With Docker composer, just clone this repository and
+With Docker composer, just clone this repository and..
 ```bash
 $ docker-compose up -d
 ```
@@ -52,6 +70,10 @@ Docker standalone
 $ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```
 
+### Windows
+
+See the install instructions at the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Microsoft-Windows
+
 ### Python Pip
 
 Check out our pypi page https://pypi.org/project/changedetection.io/
@@ -65,16 +87,31 @@ Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
 
 _Now with per-site configurable support for using a fast built in HTTP fetcher or use a Chrome based fetcher for monitoring of JavaScript websites!_
 
- 
-### Screenshots
+## Updating changedetection.io
 
-Examining differences in content.
+### Docker
+```
+docker pull dgtlmoon/changedetection.io
+docker kill $(docker ps -a|grep changedetection.io|awk '{print $1}')
+docker rm $(docker ps -a|grep changedetection.io|awk '{print $1}')
+docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
+```
 
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
+### docker-compose
 
-Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
+```bash
+docker-compose pull && docker-compose up -d
+```
 
-### Notifications
+See the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki
+
+
+## Filters
+XPath, JSONPath and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools.
+
+(We support LXML re:test, re:math and re:replace.)
+
+## Notifications
 
 ChangeDetection.io supports a massive amount of notifications (including email, office365, custom APIs, etc) when a web-page has a change detected thanks to the <a href="https://github.com/caronc/apprise">apprise</a> library.
 Simply set one or more notification URL's in the _[edit]_ tab of that watch.
@@ -92,13 +129,13 @@ Just some examples
     json://someserver.com/custom-api
     syslog://
  
-<a href="https://github.com/caronc/apprise">And everything else in this list!</a>
+<a href="https://github.com/caronc/apprise#popular-notification-services">And everything else in this list!</a>
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
 
 Now you can also customise your notification content!
 
-### JSON API Monitoring
+## JSON API Monitoring
 
 Detect changes and monitor data in JSON API's by using the built-in JSONPath selectors as a filter / selector.
 
@@ -108,7 +145,7 @@ This will re-parse the JSON and apply formatting to the text, making it super ea
 
 ![image](https://user-images.githubusercontent.com/275001/125165995-d9ea5580-e1dc-11eb-8030-f0deced2661a.png)
 
-#### Parse JSON embedded in HTML!
+### Parse JSON embedded in HTML!
 
 When you enable a `json:` filter, you can even automatically extract and parse embedded JSON inside a HTML page! Amazingly handy for sites that build content based on JSON, such as many e-commerce websites. 
 
@@ -122,35 +159,16 @@ When you enable a `json:` filter, you can even automatically extract and parse e
 
 `json:$.price` would give `23.50`, or you can extract the whole structure
 
-### Proxy
+## Proxy configuration
 
-A proxy for ChangeDetection.io can be configured by setting environment the 
-`HTTP_PROXY`, `HTTPS_PROXY` variables, examples are also in the `docker-compose.yml`
+See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration
 
-`NO_PROXY` exclude list can be specified by following `"localhost,192.168.0.0/24"`
+## Raspberry Pi support?
 
-as `docker run` with `-e`
-
-```
-docker run -d --restart always -e HTTPS_PROXY="socks5h://10.10.1.10:1080" -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
-```
-
-With `docker-compose`, see the `Proxy support example` in <a href="https://github.com/dgtlmoon/changedetection.io/blob/master/docker-compose.yml">docker-compose.yml</a>.
-
-For more information see https://docs.python-requests.org/en/master/user/advanced/#proxies
-
-This proxy support also extends to the notifications https://github.com/caronc/apprise/issues/387#issuecomment-841718867
+Raspberry Pi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! See the wiki for [details](https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver)
 
 
-### RaspberriPi support?
-
-RaspberriPi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! 
-
-### Windows native support?
-
-Sorry not yet :( https://github.com/dgtlmoon/changedetection.io/labels/windows
-
-### Support us
+## Support us
 
 Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 
@@ -159,3 +177,16 @@ Please support us, even small amounts help a LOT.
 BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />
+
+## Commercial Support
+
+I offer commercial support, this software is depended on by network security, aerospace , data-science and data-journalist professionals just to name a few, please reach out at dgtlmoon@gmail.com for any enquiries, I am more than glad to work with your organisation to further the possibilities of what can be done with changedetection.io
+
+
+[release-shield]: https://img.shields.io:/github/v/release/dgtlmoon/changedetection.io?style=for-the-badge
+[docker-pulls]: https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io?style=for-the-badge
+[test-shield]: https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master
+
+[license-shield]: https://img.shields.io/github/license/dgtlmoon/changedetection.io.svg?style=for-the-badge
+[release-link]: https://github.com/dgtlmoon.com/changedetection.io/releases
+[docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io

app.json

@@ -0,0 +1,21 @@
+{
+  "name": "ChangeDetection.io",
+  "description": "The best and simplest self-hosted open source website change detection monitoring and notification service.",
+  "keywords": [
+    "changedetection",
+    "website monitoring"
+  ],
+  "repository": "https://github.com/dgtlmoon/changedetection.io",
+  "success_url": "/",
+  "scripts": {
+  },
+  "env": {
+  },
+  "formation": {
+    "web": {
+      "quantity": 1,
+      "size": "free"
+    }
+  },
+  "image": "heroku/python"
+}

changedetection.py

@@ -1,97 +1,11 @@
 #!/usr/bin/python3
 
-# Launch as a eventlet.wsgi server instance.
-
-import getopt
-import os
-import sys
-
-import eventlet
-import eventlet.wsgi
-import changedetectionio
-
-from changedetectionio import store
-
-def main():
-    ssl_mode = False
-    port = os.environ.get('PORT') or 5000
-    do_cleanup = False
-
-    # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
-    datastore_path = os.path.join(os.getcwd(), "datastore")
-
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "csd:p:", "port")
-    except getopt.GetoptError:
-        print('backend.py -s SSL enable -p [port] -d [datastore path]')
-        sys.exit(2)
-
-    for opt, arg in opts:
-        #        if opt == '--purge':
-        # Remove history, the actual files you need to delete manually.
-        #            for uuid, watch in datastore.data['watching'].items():
-        #                watch.update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': None})
-
-        if opt == '-s':
-            ssl_mode = True
-
-        if opt == '-p':
-            port = int(arg)
-
-        if opt == '-d':
-            datastore_path = arg
-
-        # Cleanup (remove text files that arent in the index)
-        if opt == '-c':
-            do_cleanup = True
-
-    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
-    app_config = {'datastore_path': datastore_path}
-
-    if not os.path.isdir(app_config['datastore_path']):
-        print ("ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists.\n"
-               "Alternatively, use the -d parameter.".format(app_config['datastore_path']),file=sys.stderr)
-        sys.exit(2)
-
-    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=changedetectionio.__version__)
-    app = changedetectionio.changedetection_app(app_config, datastore)
-
-    # Go into cleanup mode
-    if do_cleanup:
-        datastore.remove_unused_snapshots()
-
-    app.config['datastore_path'] = datastore_path
-
-
-    @app.context_processor
-    def inject_version():
-        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
-                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
-                    has_password=datastore.data['settings']['application']['password'] != False
-                    )
-
-    # Proxy sub-directory support
-    # Set environment var USE_X_SETTINGS=1 on this script
-    # And then in your proxy_pass settings
-    #
-    #         proxy_set_header Host "localhost";
-    #         proxy_set_header X-Forwarded-Prefix /app;
-
-    if os.getenv('USE_X_SETTINGS'):
-        print ("USE_X_SETTINGS is ENABLED\n")
-        from werkzeug.middleware.proxy_fix import ProxyFix
-        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
-
-    if ssl_mode:
-        # @todo finalise SSL config, but this should get you in the right direction if you need it.
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen(('', port)),
-                                               certfile='cert.pem',
-                                               keyfile='privkey.pem',
-                                               server_side=True), app)
-
-    else:
-        eventlet.wsgi.server(eventlet.listen(('', int(port))), app)
+# Entry-point for running from the CLI when not installed via Pip, Pip will handle the console_scripts entry_points's from setup.py
+# It's recommended to use `pip3 install changedetection.io` and start with `changedetection.py` instead, it will be linkd to your global path.
+# or Docker.
+# Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
+from changedetectionio import changedetection
 
 if __name__ == '__main__':
-    main()
+    changedetection.main()

changedetectionio/.gitignore

@@ -0,0 +1 @@
+test-datastore

changedetectionio/__init__.py

@@ -11,25 +11,35 @@
 # proxy per check
 #  - flask_cors, itsdangerous,MarkupSafe
 
-import time
+import datetime
 import os
-import timeago
-import flask_login
-from flask_login import login_required
-
+import queue
 import threading
+import time
+from copy import deepcopy
 from threading import Event
 
-import queue
-
-from flask import Flask, render_template, request, send_from_directory, abort, redirect, url_for, flash
-
-from feedgen.feed import FeedGenerator
-from flask import make_response
-import datetime
+import flask_login
 import pytz
+import timeago
+from feedgen.feed import FeedGenerator
+from flask import (
+    Flask,
+    abort,
+    flash,
+    make_response,
+    redirect,
+    render_template,
+    request,
+    send_from_directory,
+    url_for,
+)
+from flask_login import login_required
+from flask_wtf import CSRFProtect
 
-__version__ = '0.39.1'
+from changedetectionio import html_tools
+
+__version__ = '0.39.11'
 
 datastore = None
 
@@ -43,11 +53,10 @@ update_q = queue.Queue()
 
 notification_q = queue.Queue()
 
-# Needs to be set this way because we also build and publish via pip
-base_path = os.path.dirname(os.path.realpath(__file__))
 app = Flask(__name__,
-            static_url_path="{}/static".format(base_path),
-            template_folder="{}/templates".format(base_path))
+            static_url_path="",
+            static_folder="static",
+            template_folder="templates")
 
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
@@ -63,6 +72,10 @@ app.config['LOGIN_DISABLED'] = False
 # Disables caching of the templates
 app.config['TEMPLATES_AUTO_RELOAD'] = True
 
+csrf = CSRFProtect()
+csrf.init_app(app)
+
+notification_debug_log=[]
 
 def init_app_secret(datastore_path):
     secret = ""
@@ -88,8 +101,7 @@ def populate_form_from_watch(form, watch):
         if i[0] != '_':
             p = getattr(form, i)
             if hasattr(p, 'data') and i in watch:
-                if not p.data:
-                    setattr(p, "data", watch[i])
+                setattr(p, "data", watch[i])
 
 
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
@@ -119,7 +131,7 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
     # return timeago.format(timestamp, time.time())
     # return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
 
-
+# When nobody is logged in Flask-Login's current_user is set to an AnonymousUser object.
 class User(flask_login.UserMixin):
     id=None
 
@@ -128,7 +140,6 @@ class User(flask_login.UserMixin):
     def get_user(self, email="defaultuser@changedetection.io"):
         return self
     def is_authenticated(self):
-
         return True
     def is_active(self):
         return True
@@ -137,13 +148,21 @@ class User(flask_login.UserMixin):
     def get_id(self):
         return str(self.id)
 
+    # Compare given password against JSON store or Env var
     def check_password(self, password):
 
-        import hashlib
         import base64
+        import hashlib
+
+        # Can be stored in env (for deployments) or in the general configs
+        raw_salt_pass = os.getenv("SALTED_PASS", False)
+
+        if not raw_salt_pass:
+            raw_salt_pass = datastore.data['settings']['application']['password']
+
+        raw_salt_pass = base64.b64decode(raw_salt_pass)
+
 
-        # Getting the values back out
-        raw_salt_pass = base64.b64decode(datastore.data['settings']['application']['password'])
         salt_from_storage = raw_salt_pass[:32]  # 32 is the length of the salt
 
         # Use the exact same setup you used to generate the key, but this time put in the password to check
@@ -194,11 +213,15 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route('/login', methods=['GET', 'POST'])
     def login():
 
-        if not datastore.data['settings']['application']['password']:
+        if not datastore.data['settings']['application']['password'] and not os.getenv("SALTED_PASS", False):
             flash("Login not required, no password enabled.", "notice")
             return redirect(url_for('index'))
 
         if request.method == 'GET':
+            if flask_login.current_user.is_authenticated:
+                flash("Already logged in")
+                return redirect(url_for("index"))
+
             output = render_template("login.html")
             return output
 
@@ -209,10 +232,18 @@ def changedetection_app(config=None, datastore_o=None):
 
         if (user.check_password(password)):
             flask_login.login_user(user, remember=True)
-            next = request.args.get('next')
+
+            # For now there's nothing else interesting here other than the index/list page
+            # It's more reliable and safe to ignore the 'next' redirect
+            # When we used...
+            # next = request.args.get('next')
+            # return redirect(next or url_for('index'))
+            # We would sometimes get login loop errors on sites hosted in sub-paths
+
+            # note for the future:
             #            if not is_safe_url(next):
             #                return flask.abort(400)
-            return redirect(next or url_for('index'))
+            return redirect(url_for('index'))
 
         else:
             flash('Incorrect password', 'error')
@@ -221,8 +252,101 @@ def changedetection_app(config=None, datastore_o=None):
 
     @app.before_request
     def do_something_whenever_a_request_comes_in():
-        # Disable password  loginif there is not one set
-        app.config['LOGIN_DISABLED'] = datastore.data['settings']['application']['password'] == False
+
+        # Disable password login if there is not one set
+        # (No password in settings or env var)
+        app.config['LOGIN_DISABLED'] = datastore.data['settings']['application']['password'] == False and os.getenv("SALTED_PASS", False) == False
+
+        # Set the auth cookie path if we're running as X-settings/X-Forwarded-Prefix
+        if os.getenv('USE_X_SETTINGS') and 'X-Forwarded-Prefix' in request.headers:
+            app.config['REMEMBER_COOKIE_PATH'] = request.headers['X-Forwarded-Prefix']
+            app.config['SESSION_COOKIE_PATH'] = request.headers['X-Forwarded-Prefix']
+
+        # For the RSS path, allow access via a token
+        if request.path == '/rss' and request.args.get('token'):
+            app_rss_token = datastore.data['settings']['application']['rss_access_token']
+            rss_url_token = request.args.get('token')
+            if app_rss_token == rss_url_token:
+                app.config['LOGIN_DISABLED'] = True
+
+    @app.route("/rss", methods=['GET'])
+    @login_required
+    def rss():
+        from . import diff
+        limit_tag = request.args.get('tag')
+
+        # Sort by last_changed and add the uuid which is usually the key..
+        sorted_watches = []
+
+        # @todo needs a .itemsWithTag() or something
+        for uuid, watch in datastore.data['watching'].items():
+
+            if limit_tag != None:
+                # Support for comma separated list of tags.
+                for tag_in_watch in watch['tag'].split(','):
+                    tag_in_watch = tag_in_watch.strip()
+                    if tag_in_watch == limit_tag:
+                        watch['uuid'] = uuid
+                        sorted_watches.append(watch)
+
+            else:
+                watch['uuid'] = uuid
+                sorted_watches.append(watch)
+
+        sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)
+
+        fg = FeedGenerator()
+        fg.title('changedetection.io')
+        fg.description('Feed description')
+        fg.link(href='https://changedetection.io')
+
+        for watch in sorted_watches:
+
+            dates = list(watch['history'].keys())
+            # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected.
+            if len(dates) < 2:
+                continue
+
+            # Convert to int, sort and back to str again
+            # @todo replace datastore getter that does this automatically
+            dates = [int(i) for i in dates]
+            dates.sort(reverse=True)
+            dates = [str(i) for i in dates]
+            prev_fname = watch['history'][dates[1]]
+
+            if not watch['viewed']:
+                # Re #239 - GUID needs to be individual for each event
+                # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228)
+                guid = "{}/{}".format(watch['uuid'], watch['last_changed'])
+                fe = fg.add_entry()
+
+                # Include a link to the diff page, they will have to login here to see if password protection is enabled.
+                # Description is the page you watch, link takes you to the diff JS UI page
+                base_url = datastore.data['settings']['application']['base_url']
+                if base_url == '':
+                    base_url = "<base-url-env-var-not-set>"
+
+                diff_link = {'href': "{}{}".format(base_url, url_for('diff_history_page', uuid=watch['uuid']))}
+
+                fe.link(link=diff_link)
+
+                # @todo watch should be a getter - watch.get('title') (internally if URL else..)
+
+                watch_title = watch.get('title') if watch.get('title') else watch.get('url')
+                fe.title(title=watch_title)
+                latest_fname = watch['history'][dates[0]]
+
+                html_diff = diff.render_diff(prev_fname, latest_fname, include_equal=False, line_feed_sep="</br>")
+                fe.description(description="<![CDATA[<html><body><h4>{}</h4>{}</body></html>".format(watch_title, html_diff))
+
+                fe.guid(guid, permalink=False)
+                dt = datetime.datetime.fromtimestamp(int(watch['newest_history_key']))
+                dt = dt.replace(tzinfo=pytz.UTC)
+                fe.pubDate(dt)
+
+        response = make_response(fg.rss_str())
+        response.headers.set('Content-Type', 'application/rss+xml')
+        return response
 
     @app.route("/", methods=['GET'])
     @login_required
@@ -230,6 +354,10 @@ def changedetection_app(config=None, datastore_o=None):
         limit_tag = request.args.get('tag')
         pause_uuid = request.args.get('pause')
 
+        # Redirect for the old rss path which used the /?rss=true
+        if request.args.get('rss'):
+            return redirect(url_for('rss', tag=limit_tag))
+
         if pause_uuid:
             try:
                 datastore.data['watching'][pause_uuid]['paused'] ^= True
@@ -239,7 +367,6 @@ def changedetection_app(config=None, datastore_o=None):
             except KeyError:
                 pass
 
-
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
         for uuid, watch in datastore.data['watching'].items():
@@ -259,42 +386,55 @@ def changedetection_app(config=None, datastore_o=None):
         sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)
 
         existing_tags = datastore.get_all_tags()
-        rss = request.args.get('rss')
 
-        if rss:
-            fg = FeedGenerator()
-            fg.title('changedetection.io')
-            fg.description('Feed description')
-            fg.link(href='https://changedetection.io')
+        from changedetectionio import forms
+        form = forms.quickWatchForm(request.form)
 
-            for watch in sorted_watches:
-                if not watch['viewed']:
-                    fe = fg.add_entry()
-                    fe.title(watch['url'])
-                    fe.link(href=watch['url'])
-                    fe.description(watch['url'])
-                    fe.guid(watch['uuid'], permalink=False)
-                    dt = datetime.datetime.fromtimestamp(int(watch['newest_history_key']))
-                    dt = dt.replace(tzinfo=pytz.UTC)
-                    fe.pubDate(dt)
-
-            response = make_response(fg.rss_str())
-            response.headers.set('Content-Type', 'application/rss+xml')
-            return response
-
-        else:
-            from changedetectionio import forms
-            form = forms.quickWatchForm(request.form)
-
-            output = render_template("watch-overview.html",
-                                     form=form,
-                                     watches=sorted_watches,
-                                     tags=existing_tags,
-                                     active_tag=limit_tag,
-                                     has_unviewed=datastore.data['has_unviewed'])
+        output = render_template("watch-overview.html",
+                                 form=form,
+                                 watches=sorted_watches,
+                                 tags=existing_tags,
+                                 active_tag=limit_tag,
+                                 app_rss_token=datastore.data['settings']['application']['rss_access_token'],
+                                 has_unviewed=datastore.data['has_unviewed'],
+                                 # Don't link to hosting when we're on the hosting environment
+                                 hosted_sticky=os.getenv("SALTED_PASS", False) == False,
+                                 guid=datastore.data['app_guid'])
 
         return output
 
+
+    # AJAX endpoint for sending a test
+    @app.route("/notification/send-test", methods=['POST'])
+    @login_required
+    def ajax_callback_send_notification_test():
+
+        import apprise
+        apobj = apprise.Apprise()
+
+        # validate URLS
+        if not len(request.form['notification_urls'].strip()):
+            return make_response({'error': 'No Notification URLs set'}, 400)
+
+        for server_url in request.form['notification_urls'].splitlines():
+            if len(server_url.strip()):
+                if not apobj.add(server_url):
+                    message = '{} is not a valid AppRise URL.'.format(server_url)
+                    return make_response({'error': message}, 400)
+
+        try:
+            n_object = {'watch_url': request.form['window_url'],
+                        'notification_urls': request.form['notification_urls'].splitlines(),
+                        'notification_title': request.form['notification_title'].strip(),
+                        'notification_body': request.form['notification_body'].strip(),
+                        'notification_format': request.form['notification_format'].strip()
+                        }
+            notification_q.put(n_object)
+        except Exception as e:
+            return make_response({'error': str(e)}, 400)
+
+        return 'OK'
+
     @app.route("/scrub", methods=['GET', 'POST'])
     @login_required
     def scrub_page():
@@ -313,7 +453,7 @@ def changedetection_app(config=None, datastore_o=None):
                     # I noticed chrome will show '/' but actually submit '-'
                     limit_date = limit_date.replace('-', '/')
                     # In the case that :ss seconds are supplied
-                    limit_date = re.sub('(\d\d:\d\d)(:\d\d)', '\\1', limit_date)
+                    limit_date = re.sub(r'(\d\d:\d\d)(:\d\d)', '\\1', limit_date)
 
                     str_to_dt = datetime.datetime.strptime(limit_date, '%Y/%m/%d %H:%M')
                     limit_timestamp = int(str_to_dt.timestamp())
@@ -349,12 +489,13 @@ def changedetection_app(config=None, datastore_o=None):
     def get_current_checksum_include_ignore_text(uuid):
 
         import hashlib
+
         from changedetectionio import fetch_site_status
 
         # Get the most recent one
         newest_history_key = datastore.get_val(uuid, 'newest_history_key')
 
-        # 0 means that theres only one, so that there should be no 'unviewed' history availabe
+        # 0 means that theres only one, so that there should be no 'unviewed' history available
         if newest_history_key == 0:
             newest_history_key = list(datastore.data['watching'][uuid]['history'].keys())[0]
 
@@ -364,10 +505,14 @@ def changedetection_app(config=None, datastore_o=None):
                 raw_content = file.read()
 
                 handler = fetch_site_status.perform_site_check(datastore=datastore)
-                stripped_content = handler.strip_ignore_text(raw_content,
+                stripped_content = html_tools.strip_ignore_text(raw_content,
                                                              datastore.data['watching'][uuid]['ignore_text'])
 
-                checksum = hashlib.md5(stripped_content).hexdigest()
+                if datastore.data['settings']['application'].get('ignore_whitespace', False):
+                    checksum = hashlib.md5(stripped_content.translate(None, b'\r\n\t ')).hexdigest()
+                else:
+                    checksum = hashlib.md5(stripped_content).hexdigest()
+
                 return checksum
 
         return datastore.data['watching'][uuid]['previous_md5']
@@ -408,10 +553,15 @@ def changedetection_app(config=None, datastore_o=None):
                           'tag': form.tag.data.strip(),
                           'title': form.title.data.strip(),
                           'headers': form.headers.data,
+                          'body': form.body.data,
+                          'method': form.method.data,
+                          'ignore_status_codes': form.ignore_status_codes.data,
                           'fetch_backend': form.fetch_backend.data,
                           'trigger_text': form.trigger_text.data,
                           'notification_title': form.notification_title.data,
-                          'notification_body': form.notification_body.data
+                          'notification_body': form.notification_body.data,
+                          'notification_format': form.notification_format.data,
+                          'extract_title_as_title': form.extract_title_as_title.data,
                           }
 
             # Notification URLs
@@ -428,35 +578,33 @@ def changedetection_app(config=None, datastore_o=None):
 
 
             datastore.data['watching'][uuid]['css_filter'] = form.css_filter.data.strip()
+            datastore.data['watching'][uuid]['subtractive_selectors'] = form.subtractive_selectors.data
 
             # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
             if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']:
                 if len(datastore.data['watching'][uuid]['history']):
                     update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
-
             datastore.data['watching'][uuid].update(update_obj)
-            datastore.needs_write = True
+
             flash("Updated watch.")
 
+            # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
+            # But in the case something is added we should save straight away
+            datastore.sync_to_json()
+
             # Queue the watch for immediate recheck
             update_q.put(uuid)
 
-            if form.trigger_check.data:
-                n_object = {'watch_url': form.url.data.strip(),
-                            'notification_urls': form.notification_urls.data,
-                            'notification_title': form.notification_title.data,
-                            'notification_body' :  form.notification_body.data
-                }
-                notification_q.put(n_object)
-
-                flash('Notifications queued.')
-
             # Diff page [edit] link should go back to diff page
-            if request.args.get("next") and request.args.get("next") == 'diff':
+            if request.args.get("next") and request.args.get("next") == 'diff' and not form.save_and_preview_button.data:
                 return redirect(url_for('diff_history_page', uuid=uuid))
             else:
-                return redirect(url_for('index'))
+                if form.save_and_preview_button.data:
+                    flash('You may need to reload this page to see the new content.')
+                    return redirect(url_for('preview_page', uuid=uuid))
+                else:
+                    return redirect(url_for('index'))
 
         else:
             if request.method == 'POST' and not form.validate():
@@ -472,7 +620,9 @@ def changedetection_app(config=None, datastore_o=None):
                                      uuid=uuid,
                                      watch=datastore.data['watching'][uuid],
                                      form=form,
-                                     using_default_minutes=using_default_minutes
+                                     using_default_minutes=using_default_minutes,
+                                     current_base_url=datastore.data['settings']['application']['base_url'],
+                                     emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False)
                                      )
 
         return output
@@ -481,62 +631,64 @@ def changedetection_app(config=None, datastore_o=None):
     @login_required
     def settings_page():
 
-        from changedetectionio import forms
-        from changedetectionio import content_fetcher
+        from changedetectionio import content_fetcher, forms
 
         form = forms.globalSettingsForm(request.form)
 
         if request.method == 'GET':
             form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check'])
             form.notification_urls.data = datastore.data['settings']['application']['notification_urls']
+            form.global_subtractive_selectors.data = datastore.data['settings']['application']['global_subtractive_selectors']
+            form.global_ignore_text.data = datastore.data['settings']['application']['global_ignore_text']
+            form.ignore_whitespace.data = datastore.data['settings']['application']['ignore_whitespace']
             form.extract_title_as_title.data = datastore.data['settings']['application']['extract_title_as_title']
             form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
             form.notification_title.data = datastore.data['settings']['application']['notification_title']
             form.notification_body.data = datastore.data['settings']['application']['notification_body']
+            form.notification_format.data = datastore.data['settings']['application']['notification_format']
+            form.base_url.data = datastore.data['settings']['application']['base_url']
+            form.real_browser_save_screenshot.data = datastore.data['settings']['application']['real_browser_save_screenshot']
 
-            # Password unset is a GET
-            if request.values.get('removepassword') == 'yes':
-                from pathlib import Path
+        if request.method == 'POST' and form.data.get('removepassword_button') == True:
+            # Password unset is a GET, but we can lock the session to a salted env password to always need the password
+            if not os.getenv("SALTED_PASS", False):
                 datastore.data['settings']['application']['password'] = False
                 flash("Password protection removed.", 'notice')
                 flask_login.logout_user()
                 return redirect(url_for('settings_page'))
 
         if request.method == 'POST' and form.validate():
-
             datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
             datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data
             datastore.data['settings']['application']['extract_title_as_title'] = form.extract_title_as_title.data
             datastore.data['settings']['application']['fetch_backend'] = form.fetch_backend.data
             datastore.data['settings']['application']['notification_title'] = form.notification_title.data
             datastore.data['settings']['application']['notification_body'] = form.notification_body.data
-
+            datastore.data['settings']['application']['notification_format'] = form.notification_format.data
             datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
-            datastore.needs_write = True
+            datastore.data['settings']['application']['base_url'] = form.base_url.data
+            datastore.data['settings']['application']['global_subtractive_selectors'] = form.global_subtractive_selectors.data
+            datastore.data['settings']['application']['global_ignore_text'] =  form.global_ignore_text.data
+            datastore.data['settings']['application']['ignore_whitespace'] = form.ignore_whitespace.data
+            datastore.data['settings']['application']['real_browser_save_screenshot'] = form.real_browser_save_screenshot.data
 
-            if form.trigger_check.data and len(form.notification_urls.data):
-                n_object = {'watch_url': "Test from changedetection.io!",
-                            'notification_urls': form.notification_urls.data,
-                            'notification_title': form.notification_title.data,
-                            'notification_body': form.notification_body.data
-                            }
-                notification_q.put(n_object)
-                flash('Notifications queued.')
-
-            if form.password.encrypted_password:
+            if not os.getenv("SALTED_PASS", False) and form.password.encrypted_password:
                 datastore.data['settings']['application']['password'] = form.password.encrypted_password
                 flash("Password protection enabled.", 'notice')
                 flask_login.logout_user()
                 return redirect(url_for('index'))
 
+            datastore.needs_write = True
             flash("Settings updated.")
 
         if request.method == 'POST' and not form.validate():
             flash("An error occurred, please see below.", "error")
 
-        # Same as notification.py
-        base_url = os.getenv('BASE_URL', '').strip('"')
-        output = render_template("settings.html", form=form, base_url=base_url)
+        output = render_template("settings.html",
+                                 form=form,
+                                 current_base_url = datastore.data['settings']['application']['base_url'],
+                                 hide_remove_pass=os.getenv("SALTED_PASS", False),
+                                 emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False))
 
         return output
 
@@ -552,8 +704,10 @@ def changedetection_app(config=None, datastore_o=None):
             urls = request.values.get('urls').split("\n")
             for url in urls:
                 url = url.strip()
+                url, *tags = url.split(" ")
+                # Flask wtform validators wont work with basic auth, use validators package
                 if len(url) and validators.url(url):
-                    new_uuid = datastore.add_watch(url=url.strip(), tag="")
+                    new_uuid = datastore.add_watch(url=url.strip(), tag=" ".join(tags))
                     # Straight into the queue.
                     update_q.put(new_uuid)
                     good += 1
@@ -602,6 +756,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         dates = list(watch['history'].keys())
         # Convert to int, sort and back to str again
+        # @todo replace datastore getter that does this automatically
         dates = [int(i) for i in dates]
         dates.sort(reverse=True)
         dates = [str(i) for i in dates]
@@ -612,21 +767,29 @@ def changedetection_app(config=None, datastore_o=None):
 
         # Save the current newest history as the most recently viewed
         datastore.set_last_viewed(uuid, dates[0])
-
         newest_file = watch['history'][dates[0]]
-        with open(newest_file, 'r') as f:
-            newest_version_file_contents = f.read()
+
+        try:
+            with open(newest_file, 'r') as f:
+                newest_version_file_contents = f.read()
+        except Exception as e:
+            newest_version_file_contents = "Unable to read {}.\n".format(newest_file)
 
         previous_version = request.args.get('previous_version')
-
         try:
             previous_file = watch['history'][previous_version]
         except KeyError:
             # Not present, use a default value, the second one in the sorted list.
             previous_file = watch['history'][dates[1]]
 
-        with open(previous_file, 'r') as f:
-            previous_version_file_contents = f.read()
+        try:
+            with open(previous_file, 'r') as f:
+                previous_version_file_contents = f.read()
+        except Exception as e:
+            previous_version_file_contents = "Unable to read {}.\n".format(previous_file)
+
+
+        screenshot_url = datastore.get_screenshot(uuid)
 
         output = render_template("diff.html", watch_a=watch,
                                  newest=newest_version_file_contents,
@@ -638,13 +801,17 @@ def changedetection_app(config=None, datastore_o=None):
                                  current_previous_version=str(previous_version),
                                  current_diff_url=watch['url'],
                                  extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
-                                 left_sticky= True )
+                                 left_sticky=True,
+                                 screenshot=screenshot_url)
 
         return output
 
     @app.route("/preview/<string:uuid>", methods=['GET'])
     @login_required
     def preview_page(uuid):
+        content = []
+        ignored_line_numbers = []
+        trigger_line_numbers = []
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
@@ -658,21 +825,89 @@ def changedetection_app(config=None, datastore_o=None):
             flash("No history found for the specified link, bad link?", "error")
             return redirect(url_for('index'))
 
-        newest = list(watch['history'].keys())[-1]
-        with open(watch['history'][newest], 'r') as f:
-            content = f.readlines()
+        if len(watch['history']):
+            timestamps = sorted(watch['history'].keys(), key=lambda x: int(x))
+            filename = watch['history'][timestamps[-1]]
+            try:
+                with open(filename, 'r') as f:
+                    tmp = f.readlines()
 
+                    # Get what needs to be highlighted
+                    ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text']
+
+                    # .readlines will keep the \n, but we will parse it here again, in the future tidy this up
+                    ignored_line_numbers = html_tools.strip_ignore_text(content="".join(tmp),
+                                                                        wordlist=ignore_rules,
+                                                                        mode='line numbers'
+                                                                        )
+
+                    trigger_line_numbers = html_tools.strip_ignore_text(content="".join(tmp),
+                                                                        wordlist=watch['trigger_text'],
+                                                                        mode='line numbers'
+                                                                        )
+                    # Prepare the classes and lines used in the template
+                    i=0
+                    for l in tmp:
+                        classes=[]
+                        i+=1
+                        if i in ignored_line_numbers:
+                            classes.append('ignored')
+                        if i in trigger_line_numbers:
+                            classes.append('triggered')
+                        content.append({'line': l, 'classes': ' '.join(classes)})
+
+
+            except Exception as e:
+                content.append({'line': "File doesnt exist or unable to read file {}".format(filename), 'classes': ''})
+        else:
+            content.append({'line': "No history found", 'classes': ''})
+
+        screenshot_url = datastore.get_screenshot(uuid)
         output = render_template("preview.html",
                                  content=content,
                                  extra_stylesheets=extra_stylesheets,
+                                 ignored_line_numbers=ignored_line_numbers,
+                                 triggered_line_numbers=trigger_line_numbers,
                                  current_diff_url=watch['url'],
+                                 screenshot=screenshot_url,
+                                 watch=watch,
                                  uuid=uuid)
+        
         return output
 
+    @app.route("/settings/notification-logs", methods=['GET'])
+    @login_required
+    def notification_logs():
+        global notification_debug_log
+        output = render_template("notification-log.html",
+                                 logs=notification_debug_log if len(notification_debug_log) else ["No errors or warnings detected"])
+
+        return output
+
+    @app.route("/api/<string:uuid>/snapshot/current", methods=['GET'])
+    @login_required
+    def api_snapshot(uuid):
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            return abort(400, "No history found for the specified link, bad link?")
+
+        newest = list(watch['history'].keys())[-1]
+        with open(watch['history'][newest], 'r') as f:
+            content = f.read()
+
+        resp = make_response(content)
+        resp.headers['Content-Type'] = 'text/plain'
+        return resp
 
     @app.route("/favicon.ico", methods=['GET'])
     def favicon():
-        return send_from_directory("/app/static/images", filename="favicon.ico")
+        return send_from_directory("static/images", path="favicon.ico")
 
     # We're good but backups are even better!
     @app.route("/backup", methods=['GET'])
@@ -683,7 +918,8 @@ def changedetection_app(config=None, datastore_o=None):
         from pathlib import Path
 
         # Remove any existing backup file, for now we just keep one file
-        for previous_backup_filename in Path(app.config['datastore_path']).rglob('changedetection-backup-*.zip'):
+
+        for previous_backup_filename in Path(datastore_o.datastore_path).rglob('changedetection-backup-*.zip'):
             os.unlink(previous_backup_filename)
 
         # create a ZipFile object
@@ -691,7 +927,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         # We only care about UUIDS from the current index file
         uuids = list(datastore.data['watching'].keys())
-        backup_filepath = os.path.join(app.config['datastore_path'], backupname)
+        backup_filepath = os.path.join(datastore_o.datastore_path, backupname)
 
         with zipfile.ZipFile(backup_filepath, "w",
                              compression=zipfile.ZIP_DEFLATED,
@@ -701,40 +937,79 @@ def changedetection_app(config=None, datastore_o=None):
             datastore.sync_to_json()
 
             # Add the index
-            zipObj.write(os.path.join(app.config['datastore_path'], "url-watches.json"), arcname="url-watches.json")
+            zipObj.write(os.path.join(datastore_o.datastore_path, "url-watches.json"), arcname="url-watches.json")
 
             # Add the flask app secret
-            zipObj.write(os.path.join(app.config['datastore_path'], "secret.txt"), arcname="secret.txt")
+            zipObj.write(os.path.join(datastore_o.datastore_path, "secret.txt"), arcname="secret.txt")
 
             # Add any snapshot data we find, use the full path to access the file, but make the file 'relative' in the Zip.
-            for txt_file_path in Path(app.config['datastore_path']).rglob('*.txt'):
+            for txt_file_path in Path(datastore_o.datastore_path).rglob('*.txt'):
                 parent_p = txt_file_path.parent
                 if parent_p.name in uuids:
                     zipObj.write(txt_file_path,
-                                 arcname=str(txt_file_path).replace(app.config['datastore_path'], ''),
+                                 arcname=str(txt_file_path).replace(datastore_o.datastore_path, ''),
                                  compress_type=zipfile.ZIP_DEFLATED,
                                  compresslevel=8)
 
             # Create a list file with just the URLs, so it's easier to port somewhere else in the future
-            list_file = os.path.join(app.config['datastore_path'], "url-list.txt")
-            with open(list_file, "w") as f:
-                for uuid in datastore.data['watching']:
-                    url = datastore.data['watching'][uuid]['url']
+            list_file = "url-list.txt"
+            with open(os.path.join(datastore_o.datastore_path, list_file), "w") as f:
+                for uuid in datastore.data["watching"]:
+                    url = datastore.data["watching"][uuid]["url"]
                     f.write("{}\r\n".format(url))
+            list_with_tags_file = "url-list-with-tags.txt"
+            with open(
+                os.path.join(datastore_o.datastore_path, list_with_tags_file), "w"
+            ) as f:
+                for uuid in datastore.data["watching"]:
+                    url = datastore.data["watching"][uuid]["url"]
+                    tag = datastore.data["watching"][uuid]["tag"]
+                    f.write("{} {}\r\n".format(url, tag))
 
             # Add it to the Zip
-            zipObj.write(list_file,
-                         arcname="url-list.txt",
-                         compress_type=zipfile.ZIP_DEFLATED,
-                         compresslevel=8)
+            zipObj.write(
+                os.path.join(datastore_o.datastore_path, list_file),
+                arcname=list_file,
+                compress_type=zipfile.ZIP_DEFLATED,
+                compresslevel=8,
+            )
+            zipObj.write(
+                os.path.join(datastore_o.datastore_path, list_with_tags_file),
+                arcname=list_with_tags_file,
+                compress_type=zipfile.ZIP_DEFLATED,
+                compresslevel=8,
+            )
 
-        return send_from_directory(app.config['datastore_path'], backupname, as_attachment=True)
+        # Send_from_directory needs to be the full absolute path
+        return send_from_directory(os.path.abspath(datastore_o.datastore_path), backupname, as_attachment=True)
 
     @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
     def static_content(group, filename):
+        if group == 'screenshot':
+
+            from flask import make_response
+
+            # Could be sensitive, follow password requirements
+            if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
+                abort(403)
+
+            # These files should be in our subdirectory
+            try:
+                # set nocache, set content-type
+                watch_dir = datastore_o.datastore_path + "/" + filename
+                response = make_response(send_from_directory(filename="last-screenshot.png", directory=watch_dir, path=watch_dir + "/last-screenshot.png"))
+                response.headers['Content-type'] = 'image/png'
+                response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+                response.headers['Pragma'] = 'no-cache'
+                response.headers['Expires'] = 0
+                return response
+
+            except FileNotFoundError:
+                abort(404)
+
         # These files should be in our subdirectory
         try:
-            return send_from_directory("static/{}".format(group), filename=filename)
+            return send_from_directory("static/{}".format(group), path=filename)
         except FileNotFoundError:
             abort(404)
 
@@ -762,11 +1037,14 @@ def changedetection_app(config=None, datastore_o=None):
             flash("Error")
             return redirect(url_for('index'))
 
+
     @app.route("/api/delete", methods=['GET'])
     @login_required
     def api_delete():
-
         uuid = request.args.get('uuid')
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
         datastore.delete(uuid)
         flash('Deleted.')
 
@@ -775,9 +1053,13 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/api/clone", methods=['GET'])
     @login_required
     def api_clone():
-
         uuid = request.args.get('uuid')
-        datastore.clone(uuid)
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        new_uuid = datastore.clone(uuid)
+        update_q.put(new_uuid)
         flash('Cloned.')
 
         return redirect(url_for('index'))
@@ -816,7 +1098,7 @@ def changedetection_app(config=None, datastore_o=None):
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                     update_q.put(watch_uuid)
                     i += 1
-        flash("{} watches are rechecking.".format(i))
+        flash("{} watches are queued for rechecking.".format(i))
         return redirect(url_for('index', tag=tag))
 
     # @todo handle ctrl break
@@ -824,15 +1106,16 @@ def changedetection_app(config=None, datastore_o=None):
 
     threading.Thread(target=notification_runner).start()
 
-    # Check for new release version
-    threading.Thread(target=check_for_new_version).start()
+    # Check for new release version, but not when running in test/build
+    if not os.getenv("GITHUB_REF", False):
+        threading.Thread(target=check_for_new_version).start()
+
     return app
 
 
 # Check for new version and anonymous stats
 def check_for_new_version():
     import requests
-
     import urllib3
     urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -858,6 +1141,7 @@ def check_for_new_version():
         app.config.exit.wait(86400)
 
 def notification_runner():
+    global notification_debug_log
     while not app.config.exit.is_set():
         try:
             # At the moment only one thread runs (single runner)
@@ -872,7 +1156,19 @@ def notification_runner():
                 notification.process_notification(n_object, datastore)
 
             except Exception as e:
-                print("Watch URL: {}  Error {}".format(n_object['watch_url'], e))
+                print("Watch URL: {}  Error {}".format(n_object['watch_url'], str(e)))
+
+                # UUID wont be present when we submit a 'test' from the global settings
+                if 'uuid' in n_object:
+                    datastore.update_watch(uuid=n_object['uuid'],
+                                           update_obj={'last_notification_error': "Notification error detected, please see logs."})
+
+                log_lines = str(e).splitlines()
+                notification_debug_log += log_lines
+
+                # Trim the log length
+                notification_debug_log = notification_debug_log[-100:]
+
 
 
 
@@ -880,8 +1176,10 @@ def notification_runner():
 def ticker_thread_check_time_launch_checks():
     from changedetectionio import update_worker
 
-    # Spin up Workers.
-    for _ in range(datastore.data['settings']['requests']['workers']):
+    # Spin up Workers that do the fetching
+    # Can be overriden by ENV or use the default settings
+    n_workers = int(os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers']))
+    for _ in range(n_workers):
         new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
         running_update_threads.append(new_worker)
         new_worker.start()
@@ -894,20 +1192,43 @@ def ticker_thread_check_time_launch_checks():
             if t.current_uuid:
                 running_uuids.append(t.current_uuid)
 
+        # Re #232 - Deepcopy the data incase it changes while we're iterating through it all
+        while True:
+            try:
+                copied_datastore = deepcopy(datastore)
+            except RuntimeError as e:
+                # RuntimeError: dictionary changed size during iteration
+                time.sleep(0.1)
+            else:
+                break
+
+        # Re #438 - Don't place more watches in the queue to be checked if the queue is already large
+        while update_q.qsize() >= 2000:
+            time.sleep(1)
+
         # Check for watches outside of the time threshold to put in the thread queue.
-        for uuid, watch in datastore.data['watching'].items():
+        now = time.time()
+        max_system_wide = int(copied_datastore.data['settings']['requests']['minutes_between_check']) * 60
+
+        for uuid, watch in copied_datastore.data['watching'].items():
+
+            # No need todo further processing if it's paused
+            if watch['paused']:
+                continue
 
             # If they supplied an individual entry minutes to threshold.
-            if 'minutes_between_check' in watch and watch['minutes_between_check'] is not None:
-                max_time = watch['minutes_between_check'] * 60
+            watch_minutes_between_check = watch.get('minutes_between_check', None)
+            if watch_minutes_between_check is not None:
+                # Cast to int just incase
+                max_time = int(watch_minutes_between_check) * 60
             else:
                 # Default system wide.
-                max_time = datastore.data['settings']['requests']['minutes_between_check'] * 60
+                max_time = max_system_wide
 
-            threshold = time.time() - max_time
+            threshold = now - max_time
 
-            # Yeah, put it in the queue, it's more than time.
-            if not watch['paused'] and watch['last_checked'] <= threshold:
+            # Yeah, put it in the queue, it's more than time
+            if watch['last_checked'] <= threshold:
                 if not uuid in running_uuids and uuid not in update_q.queue:
                     update_q.put(uuid)
 

changedetectionio/changedetection.py

@@ -0,0 +1,114 @@
+#!/usr/bin/python3
+
+# Launch as a eventlet.wsgi server instance.
+
+import getopt
+import os
+import sys
+
+import eventlet
+import eventlet.wsgi
+from . import store, changedetection_app
+from . import __version__
+
+def main():
+    ssl_mode = False
+    host = ''
+    port = os.environ.get('PORT') or 5000
+    do_cleanup = False
+    datastore_path = None
+
+    # On Windows, create and use a default path.
+    if os.name == 'nt':
+        datastore_path = os.path.expandvars(r'%APPDATA%\changedetection.io')
+        os.makedirs(datastore_path, exist_ok=True)
+    else:
+        # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
+        datastore_path = os.path.join(os.getcwd(), "../datastore")
+
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "Ccsd:h:p:", "port")
+    except getopt.GetoptError:
+        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
+        sys.exit(2)
+
+    create_datastore_dir = False
+
+    for opt, arg in opts:
+        #        if opt == '--purge':
+        # Remove history, the actual files you need to delete manually.
+        #            for uuid, watch in datastore.data['watching'].items():
+        #                watch.update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': None})
+
+        if opt == '-s':
+            ssl_mode = True
+
+        if opt == '-h':
+            host = arg
+
+        if opt == '-p':
+            port = int(arg)
+
+        if opt == '-d':
+            datastore_path = arg
+
+        # Cleanup (remove text files that arent in the index)
+        if opt == '-c':
+            do_cleanup = True
+
+        # Create the datadir if it doesnt exist
+        if opt == '-C':
+            create_datastore_dir = True
+
+    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
+    app_config = {'datastore_path': datastore_path}
+
+    if not os.path.isdir(app_config['datastore_path']):
+        if create_datastore_dir:
+            os.mkdir(app_config['datastore_path'])
+        else:
+            print(
+                "ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists or specify a directory with the -d option.\n"
+                "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
+            sys.exit(2)
+
+    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
+    app = changedetection_app(app_config, datastore)
+
+    # Go into cleanup mode
+    if do_cleanup:
+        datastore.remove_unused_snapshots()
+
+    app.config['datastore_path'] = datastore_path
+
+
+    @app.context_processor
+    def inject_version():
+        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
+                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
+                    has_password=datastore.data['settings']['application']['password'] != False
+                    )
+
+    # Proxy sub-directory support
+    # Set environment var USE_X_SETTINGS=1 on this script
+    # And then in your proxy_pass settings
+    #
+    #         proxy_set_header Host "localhost";
+    #         proxy_set_header X-Forwarded-Prefix /app;
+
+    if os.getenv('USE_X_SETTINGS'):
+        print ("USE_X_SETTINGS is ENABLED\n")
+        from werkzeug.middleware.proxy_fix import ProxyFix
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
+
+    if ssl_mode:
+        # @todo finalise SSL config, but this should get you in the right direction if you need it.
+        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port)),
+                                               certfile='cert.pem',
+                                               keyfile='privkey.pem',
+                                               server_side=True), app)
+
+    else:
+        eventlet.wsgi.server(eventlet.listen((host, int(port))), app)
+
+

changedetectionio/content_fetcher.py

@@ -1,19 +1,29 @@
-import os
-import time
 from abc import ABC, abstractmethod
+import chardet
+import os
 from selenium import webdriver
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
 from selenium.common.exceptions import WebDriverException
+import requests
+import time
 import urllib3.exceptions
 
 
 class EmptyReply(Exception):
+    def __init__(self, status_code, url):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        return
+
     pass
 
 class Fetcher():
     error = None
     status_code = None
-    content = None # Should be bytes?
+    content = None
+    headers = None
 
     fetcher_description ="No description"
 
@@ -22,10 +32,24 @@ class Fetcher():
         return self.error
 
     @abstractmethod
-    def run(self, url, timeout, request_headers):
+    def run(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False):
         # Should set self.error, self.status_code and self.content
         pass
 
+    @abstractmethod
+    def quit(self):
+        return
+
+    @abstractmethod
+    def screenshot(self):
+        return
+
     @abstractmethod
     def get_last_status_code(self):
         return self.status_code
@@ -65,67 +89,121 @@ class html_webdriver(Fetcher):
 
     command_executor = ''
 
-    def __init__(self):
-        self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub')
+    # Configs for Proxy setup
+    # In the ENV vars, is prefixed with "webdriver_", so it is for example "webdriver_sslProxy"
+    selenium_proxy_settings_mappings = ['proxyType', 'ftpProxy', 'httpProxy', 'noProxy',
+                                        'proxyAutoconfigUrl', 'sslProxy', 'autodetect',
+                                        'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
 
-    def run(self, url, timeout, request_headers):
+
+
+    proxy=None
+
+    def __init__(self):
+        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+        self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
+
+        # If any proxy settings are enabled, then we should setup the proxy object
+        proxy_args = {}
+        for k in self.selenium_proxy_settings_mappings:
+            v = os.getenv('webdriver_' + k, False)
+            if v:
+                proxy_args[k] = v.strip('"')
+
+        if proxy_args:
+            self.proxy = SeleniumProxy(raw=proxy_args)
+
+    def run(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False):
+
+        # request_body, request_method unused for now, until some magic in the future happens.
 
         # check env for WEBDRIVER_URL
-        driver = webdriver.Remote(
+        self.driver = webdriver.Remote(
             command_executor=self.command_executor,
-            desired_capabilities=DesiredCapabilities.CHROME)
+            desired_capabilities=DesiredCapabilities.CHROME,
+            proxy=self.proxy)
 
         try:
-            driver.get(url)
+            self.driver.get(url)
         except WebDriverException as e:
             # Be sure we close the session window
-            driver.quit()
+            self.quit()
             raise
 
         # @todo - how to check this? is it possible?
         self.status_code = 200
+        # @todo somehow we should try to get this working for WebDriver
+        # raise EmptyReply(url=url, status_code=r.status_code)
 
         # @todo - dom wait loaded?
-        time.sleep(5)
-        self.content = driver.page_source
-
-        driver.quit()
+        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+        self.content = self.driver.page_source
+        self.headers = {}
 
+    def screenshot(self):
+        return self.driver.get_screenshot_as_png()
 
+    # Does the connection to the webdriver work? run a test connection.
     def is_ready(self):
         from selenium import webdriver
         from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
         from selenium.common.exceptions import WebDriverException
 
-        driver = webdriver.Remote(
+        self.driver = webdriver.Remote(
             command_executor=self.command_executor,
             desired_capabilities=DesiredCapabilities.CHROME)
 
         # driver.quit() seems to cause better exceptions
-        driver.quit()
-
-
+        self.quit()
         return True
 
+    def quit(self):
+        if self.driver:
+            try:
+                self.driver.quit()
+            except Exception as e:
+                print("Exception in chrome shutdown/quit" + str(e))
+
 # "html_requests" is listed as the default fetcher in store.py!
 class html_requests(Fetcher):
     fetcher_description = "Basic fast Plaintext/HTTP Client"
 
-    def run(self, url, timeout, request_headers):
-        import requests
+    def run(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False):
 
-        r = requests.get(url,
+        r = requests.request(method=request_method,
+                         data=request_body,
+                         url=url,
                          headers=request_headers,
                          timeout=timeout,
                          verify=False)
 
-        html = r.text
-
+        # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
+        # For example - some sites don't tell us it's utf-8, but return utf-8 content
+        # This seems to not occur when using webdriver/selenium, it seems to detect the text encoding more reliably.
+        # https://github.com/psf/requests/issues/1604 good info about requests encoding detection
+        if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
+            encoding = chardet.detect(r.content)['encoding']
+            if encoding:
+                r.encoding = encoding
 
         # @todo test this
-        if not r or not html or not len(html):
-            raise EmptyReply(url)
+        # @todo maybe you really want to test zero-byte return pages?
+        if (not ignore_status_codes and not r) or not r.content or not len(r.content):
+            raise EmptyReply(url=url, status_code=r.status_code)
 
         self.status_code = r.status_code
-        self.content = html
+        self.content = r.text
+        self.headers = r.headers
 

changedetectionio/diff.py

@@ -0,0 +1,52 @@
+# used for the notifications, the front-end is using a JS library
+
+import difflib
+
+
+def same_slicer(l, a, b):
+    if a == b:
+        return [l[a]]
+    else:
+        return l[a:b]
+
+# like .compare but a little different output
+def customSequenceMatcher(before, after, include_equal=False):
+    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
+
+    # @todo Line-by-line mode instead of buncghed, including `after` that is not in `before` (maybe unset?)
+    for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
+        if include_equal and tag == 'equal':
+            g = before[alo:ahi]
+            yield g
+        elif tag == 'delete':
+            g = ["(removed) " + i for i in same_slicer(before, alo, ahi)]
+            yield g
+        elif tag == 'replace':
+            g = ["(changed) " + i for i in same_slicer(before, alo, ahi)]
+            g += ["(into   ) " + i for i in same_slicer(after, blo, bhi)]
+            yield g
+        elif tag == 'insert':
+            g = ["(added  ) " + i for i in same_slicer(after, blo, bhi)]
+            yield g
+
+# only_differences - only return info about the differences, no context
+# line_feed_sep could be "<br/>" or "<li>" or "\n" etc
+def render_diff(previous_file, newest_file, include_equal=False, line_feed_sep="\n"):
+    with open(newest_file, 'r') as f:
+        newest_version_file_contents = f.read()
+        newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
+
+    if previous_file:
+        with open(previous_file, 'r') as f:
+            previous_version_file_contents = f.read()
+            previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
+    else:
+        previous_version_file_contents = ""
+
+    rendered_diff = customSequenceMatcher(previous_version_file_contents,
+                                          newest_version_file_contents,
+                                          include_equal)
+
+    # Recursively join lists
+    f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
+    return f(rendered_diff)

changedetectionio/fetch_site_status.py

@@ -1,10 +1,11 @@
-import time
-from changedetectionio import content_fetcher
 import hashlib
-from inscriptis import get_text
-import urllib3
-from . import html_tools
+import os
 import re
+import time
+import urllib3
+
+from inscriptis import get_text
+from changedetectionio import content_fetcher, html_tools
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -16,52 +17,23 @@ class perform_site_check():
         super().__init__(*args, **kwargs)
         self.datastore = datastore
 
-    def strip_ignore_text(self, content, list_ignore_text):
-        import re
-        ignore = []
-        ignore_regex = []
-        for k in list_ignore_text:
-
-            # Is it a regex?
-            if k[0] == '/':
-                ignore_regex.append(k.strip(" /"))
-            else:
-                ignore.append(k)
-
-        output = []
-        for line in content.splitlines():
-
-            # Always ignore blank lines in this mode. (when this function gets called)
-            if len(line.strip()):
-                regex_matches = False
-
-                # if any of these match, skip
-                for regex in ignore_regex:
-                    try:
-                        if re.search(regex, line, re.IGNORECASE):
-                            regex_matches = True
-                    except Exception as e:
-                        continue
-
-                if not regex_matches and not any(skip_text in line for skip_text in ignore):
-                    output.append(line.encode('utf8'))
-
-        return "\n".encode('utf8').join(output)
-
-
-
     def run(self, uuid):
         timestamp = int(time.time())  # used for storage etc too
 
         changed_detected = False
+        screenshot = False # as bytes
         stripped_text_from_html = ""
 
         watch = self.datastore.data['watching'][uuid]
 
-        update_obj = {'previous_md5': self.datastore.data['watching'][uuid]['previous_md5'],
-                      'history': {},
-                      "last_checked": timestamp
-                      }
+        # Protect against file:// access
+        if re.search(r'^file', watch['url'], re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
+            raise Exception(
+                "file:// type access is denied for security reasons."
+            )
+
+        # Unset any existing notification error
+        update_obj = {'last_notification_error': False, 'last_error': False}
 
         extra_headers = self.datastore.get_val(uuid, 'headers')
 
@@ -80,6 +52,9 @@ class perform_site_check():
         else:
             timeout = self.datastore.data['settings']['requests']['timeout']
             url = self.datastore.get_val(uuid, 'url')
+            request_body = self.datastore.get_val(uuid, 'body')
+            request_method = self.datastore.get_val(uuid, 'method')
+            ignore_status_code = self.datastore.get_val(uuid, 'ignore_status_codes')
 
             # Pluggable content fetcher
             prefer_backend = watch['fetch_backend']
@@ -91,7 +66,7 @@ class perform_site_check():
 
 
             fetcher = klass()
-            fetcher.run(url, timeout, request_headers)
+            fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code)
             # Fetching complete, now filters
             # @todo move to class / maybe inside of fetcher abstract base?
 
@@ -102,77 +77,104 @@ class perform_site_check():
             # https://stackoverflow.com/questions/41817578/basic-method-chaining ?
             # return content().textfilter().jsonextract().checksumcompare() ?
 
-            is_html = True
+            is_json = 'application/json' in fetcher.headers.get('Content-Type', '')
+            is_html = not is_json
             css_filter_rule = watch['css_filter']
-            if css_filter_rule and len(css_filter_rule.strip()):
+            subtractive_selectors = watch.get(
+                "subtractive_selectors", []
+            ) + self.datastore.data["settings"]["application"].get(
+                "global_subtractive_selectors", []
+            )
+
+            has_filter_rule = css_filter_rule and len(css_filter_rule.strip())
+            has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
+            
+            if is_json and not has_filter_rule:
+                css_filter_rule = "json:$"
+                has_filter_rule = True
+
+            if has_filter_rule:
                 if 'json:' in css_filter_rule:
                     stripped_text_from_html = html_tools.extract_json_as_string(content=fetcher.content, jsonpath_filter=css_filter_rule)
                     is_html = False
-                else:
-                    # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
-                    stripped_text_from_html = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
 
             if is_html:
                 # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
                 html_content = fetcher.content
-                if css_filter_rule and len(css_filter_rule.strip()):
-                    html_content = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
 
-                # get_text() via inscriptis
-                stripped_text_from_html = get_text(html_content)
+                # If not JSON,  and if it's not text/plain..
+                if 'text/plain' in fetcher.headers.get('Content-Type', '').lower():
+                    # Don't run get_text or xpath/css filters on plaintext
+                    stripped_text_from_html = html_content
+                else:
+                    # Then we assume HTML
+                    if has_filter_rule:
+                        # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
+                        if css_filter_rule[0] == '/':
+                            html_content = html_tools.xpath_filter(xpath_filter=css_filter_rule, html_content=fetcher.content)
+                        else:
+                            # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+                            html_content = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
+                    if has_subtractive_selectors:
+                        html_content = html_tools.element_removal(subtractive_selectors, html_content)
+                    # get_text() via inscriptis
+                    stripped_text_from_html = get_text(html_content)
+
+            # Re #340 - return the content before the 'ignore text' was applied
+            text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
 
             # We rely on the actual text in the html output.. many sites have random script vars etc,
             # in the future we'll implement other mechanisms.
 
             update_obj["last_check_status"] = fetcher.get_last_status_code()
-            update_obj["last_error"] = False
-
 
             # If there's text to skip
             # @todo we could abstract out the get_text() to handle this cleaner
-            if len(watch['ignore_text']):
-                stripped_text_from_html = self.strip_ignore_text(stripped_text_from_html, watch['ignore_text'])
+            text_to_ignore = watch.get('ignore_text', []) + self.datastore.data['settings']['application'].get('global_ignore_text', [])
+            if len(text_to_ignore):
+                stripped_text_from_html = html_tools.strip_ignore_text(stripped_text_from_html, text_to_ignore)
             else:
                 stripped_text_from_html = stripped_text_from_html.encode('utf8')
 
+            # Re #133 - if we should strip whitespaces from triggering the change detected comparison
+            if self.datastore.data['settings']['application'].get('ignore_whitespace', False):
+                fetched_md5 = hashlib.md5(stripped_text_from_html.translate(None, b'\r\n\t ')).hexdigest()
+            else:
+                fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest()
 
-            fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest()
+            # On the first run of a site, watch['previous_md5'] will be an empty string, set it the current one.
+            if not len(watch['previous_md5']):
+                watch['previous_md5'] = fetched_md5
+                update_obj["previous_md5"] = fetched_md5
 
             blocked_by_not_found_trigger_text = False
 
             if len(watch['trigger_text']):
+                # Yeah, lets block first until something matches
                 blocked_by_not_found_trigger_text = True
-                for line in watch['trigger_text']:
-                    # Because JSON wont serialize a re.compile object
-                    if line[0] == '/' and line[-1] == '/':
-                        regex = re.compile(line.strip('/'), re.IGNORECASE)
-                        # Found it? so we don't wait for it anymore
-                        r = re.search(regex, str(stripped_text_from_html))
-                        if r:
-                            blocked_by_not_found_trigger_text = False
-                            break
-
-                    elif line.lower() in str(stripped_text_from_html).lower():
-                        # We found it don't wait for it.
-                        blocked_by_not_found_trigger_text = False
-                        break
+                # Filter and trigger works the same, so reuse it
+                result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),
+                                                      wordlist=watch['trigger_text'],
+                                                      mode="line numbers")
+                if result:
+                    blocked_by_not_found_trigger_text = False
 
 
-            # could be None or False depending on JSON type
-            # On the first run of a site, watch['previous_md5'] will be an empty string
             if not blocked_by_not_found_trigger_text and watch['previous_md5'] != fetched_md5:
                 changed_detected = True
-
-                # Don't confuse people by updating as last-changed, when it actually just changed from None..
-                if self.datastore.get_val(uuid, 'previous_md5'):
-                    update_obj["last_changed"] = timestamp
-
                 update_obj["previous_md5"] = fetched_md5
+                update_obj["last_changed"] = timestamp
+
 
             # Extract title as title
-            if is_html and self.datastore.data['settings']['application']['extract_title_as_title']:
-                if not watch['title'] or not len(watch['title']):
-                    update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
+            if is_html:
+                if self.datastore.data['settings']['application']['extract_title_as_title'] or watch['extract_title_as_title']:
+                    if not watch['title'] or not len(watch['title']):
+                        update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
 
+            if self.datastore.data['settings']['application'].get('real_browser_save_screenshot', True):
+                screenshot = fetcher.screenshot()
 
-        return changed_detected, update_obj, stripped_text_from_html
+            fetcher.quit()
+
+        return changed_detected, update_obj, text_content_before_ignored_filter, screenshot

changedetectionio/forms.py

@@ -1,11 +1,41 @@
-from wtforms import Form, SelectField, RadioField, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \
-    Field
-from wtforms import widgets
-from wtforms.validators import ValidationError
-from wtforms.fields import html5
-from changedetectionio import content_fetcher
 import re
 
+from wtforms import (
+    BooleanField,
+    Field,
+    Form,
+    IntegerField,
+    PasswordField,
+    RadioField,
+    SelectField,
+    StringField,
+    SubmitField,
+    TextAreaField,
+    fields,
+    validators,
+    widgets,
+)
+from wtforms.fields import html5
+from wtforms.validators import ValidationError
+
+from changedetectionio import content_fetcher
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+    valid_notification_formats,
+)
+
+valid_method = {
+    'GET',
+    'POST',
+    'PUT',
+    'PATCH',
+    'DELETE',
+}
+
+default_method = 'GET'
+
 class StringListField(StringField):
     widget = widgets.TextArea()
 
@@ -32,8 +62,8 @@ class SaltyPasswordField(StringField):
     encrypted_password = ""
 
     def build_password(self, password):
-        import hashlib
         import base64
+        import hashlib
         import secrets
 
         # Make a new salt on every new password and store it with the password
@@ -91,9 +121,10 @@ class ValidateContentFetcherIsReady(object):
         self.message = message
 
     def __call__(self, form, field):
-        from changedetectionio import content_fetcher
         import urllib3.exceptions
 
+        from changedetectionio import content_fetcher
+
         # Better would be a radiohandler that keeps a reference to each class
         if field.data is not None:
             klass = getattr(content_fetcher, field.data)
@@ -104,10 +135,12 @@ class ValidateContentFetcherIsReady(object):
             except urllib3.exceptions.MaxRetryError as e:
                 driver_url = some_object.command_executor
                 message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data))
-                message += '<br/>'+field.gettext('Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.')
+                message += '<br/>' + field.gettext(
+                    'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.')
                 message += '<br/>' + field.gettext('Did you follow the instructions in the wiki?')
                 message += '<br/><br/>' + field.gettext('WebDriver Host: %s' % (driver_url))
                 message += '<br/><a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">Go here for more information</a>'
+                message += '<br/>'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e)))
 
                 raise ValidationError(message)
 
@@ -116,6 +149,21 @@ class ValidateContentFetcherIsReady(object):
                 raise ValidationError(message % (field.data, e))
 
 
+class ValidateNotificationBodyAndTitleWhenURLisSet(object):
+    """
+       Validates that they entered something in both notification title+body when the URL is set
+       Due to https://github.com/dgtlmoon/changedetection.io/issues/360
+       """
+
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        if len(field.data):
+            if not len(form.notification_title.data) or not len(form.notification_body.data):
+                message = field.gettext('Notification Body and Title is required when a Notification URL is used')
+                raise ValidationError(message)
+
 class ValidateAppRiseServers(object):
     """
        Validates that each URL given is compatible with AppRise
@@ -147,7 +195,24 @@ class ValidateTokensList(object):
             if not p.strip('{}') in notification.valid_tokens:
                 message = field.gettext('Token \'%s\' is not a valid token.')
                 raise ValidationError(message % (p))
+            
+class validateURL(object):
+    
+    """
+       Flask wtform validators wont work with basic auth
+    """
 
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        import validators
+        try:
+            validators.url(field.data.strip())
+        except validators.ValidationFailure:
+            message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
+            raise ValidationError(message)
+        
 class ValidateListRegex(object):
     """
     Validates that anything that looks like a regex passes as a regex
@@ -166,63 +231,127 @@ class ValidateListRegex(object):
                 except re.error:
                     message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
                     raise ValidationError(message % (line))
-
-class ValidateCSSJSONInput(object):
+              
+class ValidateCSSJSONXPATHInput(object):
     """
     Filter validation
     @todo CSS validator ;)
     """
 
-    def __init__(self, message=None):
+    def __init__(self, message=None, allow_xpath=True, allow_json=True):
         self.message = message
+        self.allow_xpath = allow_xpath
+        self.allow_json = allow_json
 
     def __call__(self, form, field):
-        if 'json:' in field.data:
-            from jsonpath_ng.exceptions import JsonPathParserError
-            from jsonpath_ng import jsonpath, parse
 
-            input = field.data.replace('json:', '')
+        if isinstance(field.data, str):
+            data = [field.data]
+        else:
+            data = field.data
 
-            try:
-                parse(input)
-            except JsonPathParserError as e:
-                message = field.gettext('\'%s\' is not a valid JSONPath expression. (%s)')
-                raise ValidationError(message % (input, str(e)))
+        for line in data:
+        # Nothing to see here
+            if not len(line.strip()):
+                return
 
+            # Does it look like XPath?
+            if line.strip()[0] == '/':
+                if not self.allow_xpath:
+                    raise ValidationError("XPath not permitted in this field!")
+                from lxml import etree, html
+                tree = html.fromstring("<html></html>")
 
+                try:
+                    tree.xpath(line.strip())
+                except etree.XPathEvalError as e:
+                    message = field.gettext('\'%s\' is not a valid XPath expression. (%s)')
+                    raise ValidationError(message % (line, str(e)))
+                except:
+                    raise ValidationError("A system-error occurred when validating your XPath expression")
+
+            if 'json:' in line:
+                if not self.allow_json:
+                    raise ValidationError("JSONPath not permitted in this field!")
+
+                from jsonpath_ng.exceptions import (
+                    JsonPathLexerError,
+                    JsonPathParserError,
+                )
+                from jsonpath_ng.ext import parse
+
+                input = line.replace('json:', '')
+
+                try:
+                    parse(input)
+                except (JsonPathParserError, JsonPathLexerError) as e:
+                    message = field.gettext('\'%s\' is not a valid JSONPath expression. (%s)')
+                    raise ValidationError(message % (input, str(e)))
+                except:
+                    raise ValidationError("A system-error occurred when validating your JSONPath expression")
+
+                # Re #265 - maybe in the future fetch the page and offer a
+                # warning/notice that its possible the rule doesnt yet match anything?
+
+            
 class quickWatchForm(Form):
     # https://wtforms.readthedocs.io/en/2.3.x/fields/#module-wtforms.fields.html5
     # `require_tld` = False is needed even for the test harness "http://localhost:5005.." to run
-    url = html5.URLField('URL', [validators.URL(require_tld=False)])
+    url = html5.URLField('URL', validators=[validateURL()])
     tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)])
 
 class commonSettingsForm(Form):
 
-    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
-    notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {watch_url}', validators=[validators.Optional(), ValidateTokensList()])
-    notification_body = TextAreaField('Notification Body', default='{watch_url} had a change.', validators=[validators.Optional(), ValidateTokensList()])
-    trigger_check = BooleanField('Send test notification on save')
+    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateNotificationBodyAndTitleWhenURLisSet(), ValidateAppRiseServers()])
+    notification_title = StringField('Notification Title', default=default_notification_title, validators=[validators.Optional(), ValidateTokensList()])
+    notification_body = TextAreaField('Notification Body', default=default_notification_body, validators=[validators.Optional(), ValidateTokensList()])
+    notification_format = SelectField('Notification Format', choices=valid_notification_formats.keys(), default=default_notification_format)
     fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
 
 class watchForm(commonSettingsForm):
 
-    url = html5.URLField('URL', [validators.URL(require_tld=False)])
+    url = html5.URLField('URL', validators=[validateURL()])
     tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)])
 
     minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
                                                [validators.Optional(), validators.NumberRange(min=1)])
-    css_filter = StringField('CSS/JSON Filter', [ValidateCSSJSONInput()])
+    css_filter = StringField('CSS/JSON/XPATH Filter', [ValidateCSSJSONXPATHInput()])
+    subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
     title = StringField('Title')
 
     ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
     headers = StringDictKeyValue('Request Headers')
+    body = TextAreaField('Request Body', [validators.Optional()])
+    method = SelectField('Request Method', choices=valid_method, default=default_method)
+    ignore_status_codes = BooleanField('Ignore Status Codes (process non-2xx status codes as normal)', default=False)
     trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
 
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+    save_and_preview_button = SubmitField('Save & Preview', render_kw={"class": "pure-button pure-button-primary"})
+
+    def validate(self, **kwargs):
+        if not super().validate():
+            return False
+
+        result = True
+
+        # Fail form validation when a body is set for a GET
+        if self.method.data == 'GET' and self.body.data:
+            self.body.errors.append('Body must be empty when Request Method is set to GET')
+            result = False
+
+        return result
 
 class globalSettingsForm(commonSettingsForm):
-
     password = SaltyPasswordField()
-
     minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
                                                [validators.NumberRange(min=1)])
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title')
+    base_url = StringField('Base URL', validators=[validators.Optional()])
+    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
+    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
+    ignore_whitespace = BooleanField('Ignore whitespace')
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+    real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome?')
+    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})

changedetectionio/html_tools.py

@@ -1,6 +1,9 @@
 import json
+import re
+from typing import List
+
 from bs4 import BeautifulSoup
-from jsonpath_ng import parse
+from jsonpath_ng.ext import parse
 
 
 class JSONNotFound(ValueError):
@@ -16,6 +19,31 @@ def css_filter(css_filter, html_content):
 
     return html_block + "\n"
 
+def subtractive_css_selector(css_selector, html_content):
+    soup = BeautifulSoup(html_content, "html.parser")
+    for item in soup.select(css_selector):
+        item.decompose()
+    return str(soup)
+
+    
+def element_removal(selectors: List[str], html_content):
+    """Joins individual filters into one css filter."""
+    selector = ",".join(selectors)
+    return subtractive_css_selector(selector, html_content)
+    
+
+# Return str Utf-8 of matched rules
+def xpath_filter(xpath_filter, html_content):
+    from lxml import etree, html
+
+    tree = html.fromstring(html_content)
+    html_block = ""
+
+    for item in tree.xpath(xpath_filter.strip(), namespaces={'re':'http://exslt.org/regular-expressions'}):
+        html_block+= etree.tostring(item, pretty_print=True).decode('utf-8')+"<br/>"
+
+    return html_block
+
 
 # Extract/find element
 def extract_element(find='title', html_content=''):
@@ -45,10 +73,13 @@ def _parse_json(json_data, jsonpath_filter):
     if len(match) == 1:
         s = match[0].value
 
-    if not s:
-        raise JSONNotFound("No Matching JSON could be found for the rule {}".format(jsonpath_filter.replace('json:', '')))
+    # Re #257 - Better handling where it does not exist, in the case the original 's' value was False..
+    if not match:
+        # Re 265 - Just return an empty string when filter not found
+        return ''
 
-    stripped_text_from_html = json.dumps(s, indent=4)
+    # Ticket #462 - allow the original encoding through, usually it's UTF-8 or similar
+    stripped_text_from_html = json.dumps(s, indent=4, ensure_ascii=False)
 
     return stripped_text_from_html
 
@@ -85,6 +116,54 @@ def extract_json_as_string(content, jsonpath_filter):
                     break
 
     if not stripped_text_from_html:
-        raise JSONNotFound("No JSON matching the rule '%s' found" % jsonpath_filter.replace('json:',''))
+        # Re 265 - Just return an empty string when filter not found
+        return ''
 
     return stripped_text_from_html
+
+# Mode     - "content" return the content without the matches (default)
+#          - "line numbers" return a list of line numbers that match (int list)
+#
+# wordlist - list of regex's (str) or words (str)
+def strip_ignore_text(content, wordlist, mode="content"):
+    ignore = []
+    ignore_regex = []
+
+    # @todo check this runs case insensitive
+    for k in wordlist:
+
+        # Is it a regex?
+        if k[0] == '/':
+            ignore_regex.append(k.strip(" /"))
+        else:
+            ignore.append(k)
+
+    i = 0
+    output = []
+    ignored_line_numbers = []
+    for line in content.splitlines():
+        i += 1
+        # Always ignore blank lines in this mode. (when this function gets called)
+        if len(line.strip()):
+            regex_matches = False
+
+            # if any of these match, skip
+            for regex in ignore_regex:
+                try:
+                    if re.search(regex, line, re.IGNORECASE):
+                        regex_matches = True
+                except Exception as e:
+                    continue
+
+            if not regex_matches and not any(skip_text.lower() in line.lower() for skip_text in ignore):
+                output.append(line.encode('utf8'))
+            else:
+                ignored_line_numbers.append(i)
+
+
+
+    # Used for finding out what to highlight
+    if mode == "line numbers":
+        return ignored_line_numbers
+
+    return "\n".encode('utf8').join(output)

changedetectionio/notification.py

@@ -1,5 +1,5 @@
-import os
 import apprise
+from apprise import NotifyFormat
 
 valid_tokens = {
     'base_url': '',
@@ -7,16 +7,25 @@ valid_tokens = {
     'watch_uuid': '',
     'watch_title': '',
     'watch_tag': '',
+    'diff': '',
+    'diff_full': '',
     'diff_url': '',
     'preview_url': '',
     'current_snapshot': ''
 }
 
+valid_notification_formats = {
+    'Text': NotifyFormat.TEXT,
+    'Markdown': NotifyFormat.MARKDOWN,
+    'HTML': NotifyFormat.HTML,
+}
+
+default_notification_format = 'Text'
+default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n'
+default_notification_title = 'ChangeDetection.io Notification - {watch_url}'
 
 def process_notification(n_object, datastore):
-    import logging
-    log = logging.getLogger('apprise')
-    log.setLevel('TRACE')
+
     apobj = apprise.Apprise(debug=True)
 
     for url in n_object['notification_urls']:
@@ -25,8 +34,13 @@ def process_notification(n_object, datastore):
         apobj.add(url)
 
     # Get the notification body from datastore
-    n_body = n_object['notification_body']
-    n_title = n_object['notification_title']
+    n_body = n_object.get('notification_body', default_notification_body)
+    n_title = n_object.get('notification_title', default_notification_title)
+    n_format = valid_notification_formats.get(
+        n_object['notification_format'],
+        valid_notification_formats[default_notification_format],
+    )
+
 
     # Insert variables into the notification content
     notification_parameters = create_notification_parameters(n_object, datastore)
@@ -37,10 +51,22 @@ def process_notification(n_object, datastore):
         n_title = n_title.replace(token, val)
         n_body = n_body.replace(token, val)
 
-    apobj.notify(
+    # https://github.com/caronc/apprise/wiki/Development_LogCapture
+    # Anything higher than or equal to WARNING (which covers things like Connection errors)
+    # raise it as an exception
+
+    with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
+        apobj.notify(
         body=n_body,
-        title=n_title
-    )
+        title=n_title,
+        body_format=n_format)
+
+        # Returns empty string if nothing found, multi-line string otherwise
+        log_value = logs.getvalue()
+        if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
+            raise Exception(log_value)
+
+
 
 # Notification title + body content parameters get created here.
 def create_notification_parameters(n_object, datastore):
@@ -57,7 +83,8 @@ def create_notification_parameters(n_object, datastore):
         watch_tag = ''
 
     # Create URLs to customise the notification with
-    base_url = os.getenv('BASE_URL', '').strip('"')
+    base_url = datastore.data['settings']['application']['base_url']
+
     watch_url = n_object['watch_url']
 
     # Re #148 - Some people have just {base_url} in the body or title, but this may break some notification services
@@ -73,15 +100,17 @@ def create_notification_parameters(n_object, datastore):
 
     # Valid_tokens also used as a field validator
     tokens.update(
-    {
-        'base_url': base_url if base_url is not None else '',
-        'watch_url': watch_url,
-        'watch_uuid': uuid,
-        'watch_title': watch_title if watch_title is not None else '',
-        'watch_tag': watch_tag if watch_tag is not None else '',
-        'diff_url': diff_url,
-        'preview_url': preview_url,
-        'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
-    })
+        {
+            'base_url': base_url if base_url is not None else '',
+            'watch_url': watch_url,
+            'watch_uuid': uuid,
+            'watch_title': watch_title if watch_title is not None else '',
+            'watch_tag': watch_tag if watch_tag is not None else '',
+            'diff_url': diff_url,
+            'diff': n_object.get('diff', ''),  # Null default in the case we use a test
+            'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
+            'preview_url': preview_url,
+            'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
+        })
 
-    return tokens
+    return tokens

changedetectionio/run_all_tests.sh

@@ -9,11 +9,16 @@
 # exit when any command fails
 set -e
 
-# Re #65 - Ability to include a link back to the installation, in the notification.
-export BASE_URL="https://foobar.com"
-
 find tests/test_*py -type f|while read test_name
 do
   echo "TEST RUNNING $test_name"
   pytest $test_name
 done
+
+echo "RUNNING WITH BASE_URL SET"
+
+# Now re-run some tests with BASE_URL enabled
+# Re #65 - Ability to include a link back to the installation, in the notification.
+export BASE_URL="https://really-unique-domain.io"
+pytest tests/test_notification.py
+

changedetectionio/static/js/notifications.js

@@ -0,0 +1,53 @@
+$(document).ready(function() {
+
+  $('#add-email-helper').click(function (e) {
+    e.preventDefault();
+    email = prompt("Destination email");
+    if(email) {
+      var n = $("#notification_urls");
+      var p=email_notification_prefix;
+      $(n).val( $.trim( $(n).val() )+"\n"+email_notification_prefix+email );
+    }
+  });
+
+  $('#send-test-notification').click(function (e) {
+    e.preventDefault();
+
+    // this can be global
+    var csrftoken = $('input[name=csrf_token]').val();
+    $.ajaxSetup({
+        beforeSend: function(xhr, settings) {
+            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken)
+            }
+        }
+    })
+
+    data = {
+        window_url : window.location.href,
+        notification_urls : $('#notification_urls').val(),
+        notification_title : $('#notification_title').val(),
+        notification_body : $('#notification_body').val(),
+        notification_format : $('#notification_format').val(),
+    }
+    for (key in data) {
+      if (!data[key].length) {
+        alert(key+" is empty, cannot send test.")
+        return;
+      }
+    }
+
+    $.ajax({
+      type: "POST",
+      url: notification_base_url,
+      data : data
+    }).done(function(data){
+      console.log(data);
+      alert('Sent');
+    }).fail(function(data){
+      console.log(data);
+      alert('Error: '+data.responseJSON.error);
+    })
+  });
+});
+

changedetectionio/static/js/tabs.js

@@ -1,6 +1,11 @@
 // Rewrite this is a plugin.. is all this JS really 'worth it?'
 
 
+if(!window.location.hash) {
+  var tab=document.querySelectorAll("#default-tab a");
+  tab[0].click();
+}
+
 window.addEventListener('hashchange', function() {
   var tabs = document.getElementsByClassName('active');
   while (tabs[0]) {
@@ -21,7 +26,6 @@ if (!has_errors.length) {
   focus_error_tab();
 }
 
-
 function set_active_tab() {
   var tab=document.querySelectorAll("a[href='"+location.hash+"']");
   if (tab.length) {

changedetectionio/static/js/watch-overview.js

@@ -0,0 +1,6 @@
+$(function () {
+  // Remove unviewed status when normally clicked
+  $('.diff-link').click(function () {
+    $(this).closest('.unviewed').removeClass('unviewed');
+  });
+});

changedetectionio/static/styles/diff.css

@@ -1,9 +1,10 @@
 #diff-ui {
   background: #fff;
   padding: 2em;
-  margin: 1em;
+  margin-left: 1em;
+  margin-right: 1em;
   border-radius: 5px;
-  font-size: 9px; }
+  font-size: 11px; }
   #diff-ui table {
     table-layout: fixed;
     width: 100%; }
@@ -54,3 +55,24 @@ ins {
   body {
     height: 99%;
     /* Hide scroll bar in Firefox */ } }
+
+td#diff-col div {
+  text-align: justify;
+  white-space: pre-wrap; }
+
+.ignored {
+  background-color: #ccc;
+  /*  border: #0d91fa 1px solid; */
+  opacity: 0.7; }
+
+.triggered {
+  background-color: #1b98f8; }
+
+/* ignored and triggered? make it obvious error */
+.ignored.triggered {
+  background-color: #ff0000; }
+
+.tab-pane-inner#screenshot {
+  text-align: center; }
+  .tab-pane-inner#screenshot img {
+    max-width: 99%; }

changedetectionio/static/styles/diff.scss

@@ -2,9 +2,10 @@
 
     background: #fff;
     padding: 2em;
-    margin: 1em;
+    margin-left: 1em;
+    margin-right: 1em;
     border-radius: 5px;
-    font-size: 9px;
+    font-size: 11px;
 
     table {
         table-layout: fixed;
@@ -66,3 +67,30 @@ ins {
 		height: 99%; /* Hide scroll bar in Firefox */
 	}
 }
+
+td#diff-col div {
+    text-align: justify;
+    white-space: pre-wrap;
+}
+
+.ignored {
+    background-color: #ccc;
+   /*  border: #0d91fa 1px solid; */
+    opacity: 0.7;
+}
+
+.triggered {
+    background-color: #1b98f8;
+}
+
+/* ignored and triggered? make it obvious error */
+.ignored.triggered {
+  background-color: #ff0000;
+}
+
+.tab-pane-inner#screenshot {
+  text-align: center;
+  img {
+    max-width: 99%;
+  }
+}

changedetectionio/static/styles/package-lock.json

@@ -9,7 +9,7 @@
       "version": "0.0.3",
       "license": "ISC",
       "dependencies": {
-        "node-sass": "^6.0.1",
+        "node-sass": "^7.0.0",
         "tar": "^6.1.9",
         "trim-newlines": "^3.0.1"
       }
@@ -128,13 +128,35 @@
       }
     },
     "node_modules/ansi-styles": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
-      "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
+    "node_modules/ansi-styles/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/ansi-styles/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
+    },
     "node_modules/aproba": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
@@ -251,18 +273,18 @@
       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
     },
     "node_modules/chalk": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-      "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dependencies": {
-        "ansi-styles": "^2.2.1",
-        "escape-string-regexp": "^1.0.2",
-        "has-ansi": "^2.0.0",
-        "strip-ansi": "^3.0.0",
-        "supports-color": "^2.0.0"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
       },
       "engines": {
-        "node": ">=0.10.0"
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
     "node_modules/chownr": {
@@ -344,6 +366,14 @@
       "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
       "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
     },
+    "node_modules/color-support": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz",
+      "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==",
+      "bin": {
+        "color-support": "bin.js"
+      }
+    },
     "node_modules/combined-stream": {
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -677,17 +707,6 @@
         "node": ">= 0.4.0"
       }
     },
-    "node_modules/has-ansi": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
-      "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
-      "dependencies": {
-        "ansi-regex": "^2.0.0"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/has-flag": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -1042,13 +1061,13 @@
       }
     },
     "node_modules/node-sass": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-6.0.1.tgz",
-      "integrity": "sha512-f+Rbqt92Ful9gX0cGtdYwjTrWAaGURgaK5rZCWOgCNyGWusFYHhbqCCBoFBeat+HKETOU02AyTxNhJV0YZf2jQ==",
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-7.0.0.tgz",
+      "integrity": "sha512-6yUnsD3L8fVbgMX6nKQqZkjRcG7a/PpmF0pEyeWf+BgbTj2ToJlCYrnUifL2KbjV5gIY22I3oppahBWA3B+jUg==",
       "hasInstallScript": true,
       "dependencies": {
         "async-foreach": "^0.1.3",
-        "chalk": "^1.1.1",
+        "chalk": "^4.1.2",
         "cross-spawn": "^7.0.3",
         "gaze": "^1.0.0",
         "get-stdin": "^4.0.1",
@@ -1057,7 +1076,7 @@
         "meow": "^9.0.0",
         "nan": "^2.13.2",
         "node-gyp": "^7.1.0",
-        "npmlog": "^4.0.0",
+        "npmlog": "^5.0.0",
         "request": "^2.88.0",
         "sass-graph": "2.2.5",
         "stdout-stream": "^1.4.0",
@@ -1070,6 +1089,106 @@
         "node": ">=12"
       }
     },
+    "node_modules/node-sass/node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/node-sass/node_modules/are-we-there-yet": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz",
+      "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==",
+      "dependencies": {
+        "delegates": "^1.0.0",
+        "readable-stream": "^3.6.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/node-sass/node_modules/emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+    },
+    "node_modules/node-sass/node_modules/gauge": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz",
+      "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==",
+      "dependencies": {
+        "aproba": "^1.0.3 || ^2.0.0",
+        "color-support": "^1.1.2",
+        "console-control-strings": "^1.0.0",
+        "has-unicode": "^2.0.1",
+        "object-assign": "^4.1.1",
+        "signal-exit": "^3.0.0",
+        "string-width": "^4.2.3",
+        "strip-ansi": "^6.0.1",
+        "wide-align": "^1.1.2"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/node-sass/node_modules/is-fullwidth-code-point": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/node-sass/node_modules/npmlog": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz",
+      "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==",
+      "dependencies": {
+        "are-we-there-yet": "^2.0.0",
+        "console-control-strings": "^1.1.0",
+        "gauge": "^3.0.0",
+        "set-blocking": "^2.0.0"
+      }
+    },
+    "node_modules/node-sass/node_modules/readable-stream": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
+      "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
+      "dependencies": {
+        "inherits": "^2.0.3",
+        "string_decoder": "^1.1.1",
+        "util-deprecate": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/node-sass/node_modules/string-width": {
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "dependencies": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/node-sass/node_modules/strip-ansi": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/nopt": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz",
@@ -1616,11 +1735,22 @@
       }
     },
     "node_modules/supports-color": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
-      "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
       "engines": {
-        "node": ">=0.8.0"
+        "node": ">=8"
+      }
+    },
+    "node_modules/supports-color/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "engines": {
+        "node": ">=8"
       }
     },
     "node_modules/tar": {
@@ -2050,9 +2180,27 @@
       "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
     },
     "ansi-styles": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
-      "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4="
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "requires": {
+        "color-convert": "^2.0.1"
+      },
+      "dependencies": {
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
+        }
+      }
     },
     "aproba": {
       "version": "1.2.0",
@@ -2149,15 +2297,12 @@
       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
     },
     "chalk": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-      "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "requires": {
-        "ansi-styles": "^2.2.1",
-        "escape-string-regexp": "^1.0.2",
-        "has-ansi": "^2.0.0",
-        "strip-ansi": "^3.0.0",
-        "supports-color": "^2.0.0"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
       }
     },
     "chownr": {
@@ -2223,6 +2368,11 @@
       "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
       "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
     },
+    "color-support": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz",
+      "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg=="
+    },
     "combined-stream": {
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -2485,14 +2635,6 @@
         "function-bind": "^1.1.1"
       }
     },
-    "has-ansi": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
-      "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
-      "requires": {
-        "ansi-regex": "^2.0.0"
-      }
-    },
     "has-flag": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -2768,12 +2910,12 @@
       }
     },
     "node-sass": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-6.0.1.tgz",
-      "integrity": "sha512-f+Rbqt92Ful9gX0cGtdYwjTrWAaGURgaK5rZCWOgCNyGWusFYHhbqCCBoFBeat+HKETOU02AyTxNhJV0YZf2jQ==",
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-7.0.0.tgz",
+      "integrity": "sha512-6yUnsD3L8fVbgMX6nKQqZkjRcG7a/PpmF0pEyeWf+BgbTj2ToJlCYrnUifL2KbjV5gIY22I3oppahBWA3B+jUg==",
       "requires": {
         "async-foreach": "^0.1.3",
-        "chalk": "^1.1.1",
+        "chalk": "^4.1.2",
         "cross-spawn": "^7.0.3",
         "gaze": "^1.0.0",
         "get-stdin": "^4.0.1",
@@ -2782,11 +2924,92 @@
         "meow": "^9.0.0",
         "nan": "^2.13.2",
         "node-gyp": "^7.1.0",
-        "npmlog": "^4.0.0",
+        "npmlog": "^5.0.0",
         "request": "^2.88.0",
         "sass-graph": "2.2.5",
         "stdout-stream": "^1.4.0",
         "true-case-path": "^1.0.2"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+          "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="
+        },
+        "are-we-there-yet": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz",
+          "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==",
+          "requires": {
+            "delegates": "^1.0.0",
+            "readable-stream": "^3.6.0"
+          }
+        },
+        "emoji-regex": {
+          "version": "8.0.0",
+          "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+          "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+        },
+        "gauge": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz",
+          "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==",
+          "requires": {
+            "aproba": "^1.0.3 || ^2.0.0",
+            "color-support": "^1.1.2",
+            "console-control-strings": "^1.0.0",
+            "has-unicode": "^2.0.1",
+            "object-assign": "^4.1.1",
+            "signal-exit": "^3.0.0",
+            "string-width": "^4.2.3",
+            "strip-ansi": "^6.0.1",
+            "wide-align": "^1.1.2"
+          }
+        },
+        "is-fullwidth-code-point": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+          "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
+        },
+        "npmlog": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz",
+          "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==",
+          "requires": {
+            "are-we-there-yet": "^2.0.0",
+            "console-control-strings": "^1.1.0",
+            "gauge": "^3.0.0",
+            "set-blocking": "^2.0.0"
+          }
+        },
+        "readable-stream": {
+          "version": "3.6.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
+          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
+          "requires": {
+            "inherits": "^2.0.3",
+            "string_decoder": "^1.1.1",
+            "util-deprecate": "^1.0.1"
+          }
+        },
+        "string-width": {
+          "version": "4.2.3",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+          "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+          "requires": {
+            "emoji-regex": "^8.0.0",
+            "is-fullwidth-code-point": "^3.0.0",
+            "strip-ansi": "^6.0.1"
+          }
+        },
+        "strip-ansi": {
+          "version": "6.0.1",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+          "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+          "requires": {
+            "ansi-regex": "^5.0.1"
+          }
+        }
       }
     },
     "nopt": {
@@ -3213,9 +3436,19 @@
       }
     },
     "supports-color": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
-      "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc="
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "requires": {
+        "has-flag": "^4.0.0"
+      },
+      "dependencies": {
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="
+        }
+      }
     },
     "tar": {
       "version": "6.1.9",

changedetectionio/static/styles/package.json

@@ -4,13 +4,12 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
-    "scss": "node-sass --watch styles.scss diff.scss -o ."
+    "build": "node-sass styles.scss -o .;node-sass diff.scss -o ."
   },
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "node-sass": "^6.0.1",
+    "node-sass": "^7.0.0",
     "tar": "^6.1.9",
     "trim-newlines": "^3.0.1"
   }

changedetectionio/static/styles/styles.scss

@@ -1,7 +1,10 @@
 /*
  * -- BASE STYLES --
  * Most of these are inherited from Base, but I want to change a few.
- * npm run scss
+ * nvm use v14.18.1
+ * npm install
+ * npm run build
+ * or npm run watch
  */
 body {
   color: #333;
@@ -32,16 +35,21 @@ a.github-link {
 
 section.content {
   padding-top: 5em;
-  padding-bottom: 5em;
+  padding-bottom: 1em;
   flex-direction: column;
   display: flex;
   align-items: center;
   justify-content: center;
 }
 
+code {
+  background: #eee;
+}
+
 /* table related */
 .watch-table {
   width: 100%;
+  font-size: 80%;
 
   tr.unviewed {
     font-weight: bold;
@@ -52,7 +60,6 @@ section.content {
   }
 
   td {
-    font-size: 80%;
     white-space: nowrap;
   }
 
@@ -104,12 +111,12 @@ section.content {
 
 body:after {
   content: "";
-  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)
+  background: linear-gradient(130deg, #5ad8f7, #2f50af 41.07%, #9150bf 84.05%);
 }
 
 body:after, body:before {
   display: block;
-  height: 600px;
+  height: 650px;
   position: absolute;
   top: 0;
   left: 0;
@@ -122,11 +129,8 @@ body::after {
 }
 
 body::before {
+  // background-image set in base.html so it works with reverse proxies etc
   content: "";
-  background-image: url(/static/images/gradient-border.png);
-}
-
-body:before {
   background-size: cover
 }
 
@@ -262,6 +266,7 @@ body:after, body:before {
   }
   legend {
     color: #fff;
+    font-weight: bold;
   }
 }
 
@@ -314,12 +319,10 @@ footer {
     */
 }
 
-
-
 .sticky-tab {
   position: absolute;
-  top: 80px;
-  font-size: 8px;
+  top: 60px;
+  font-size: 65%;
   background: #fff;
   padding: 10px;
   &#left-sticky {
@@ -328,6 +331,11 @@ footer {
   &#right-sticky {
     right: 0px;
   }
+  &#hosted-sticky {
+    right: 0px;
+    top: 100px;
+    font-weight: bold;
+  }
 }
 
 #new-version-text a {
@@ -414,18 +422,35 @@ footer {
   }
 }
 
+
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 800px) {
+
+  div.sticky-tab#hosted-sticky {
+    top: 60px;
+    left: 0px;
+    right: auto;
+  }
+
+  section.content {
+    padding-top: 110px;
+  }
+
+  // Make the tabs easier to hit, they will be all nice and horizontal
+  div.tabs.collapsable ul li {
+    display: block;
+    border-radius: 0px;
+  }
+
+  input[type='text'] {
+    width: 100%;
+  }
+
 /*
 Max width before this PARTICULAR table gets nasty
 This query will take effect for any screen smaller than 760px
 and also iPads specifically.
 */
-
-@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
-
-  input[type='text'] {
-    width: 100%;
-  }
-    
   .watch-table {
     /* Force table to not be like tables anymore */
     thead, tbody, th, td, tr {
@@ -539,9 +564,16 @@ $form-edge-padding: 20px;
     display: block;
   }
 }
-.edit-form {
-  min-width: 70%;
-  .tab-pane-inner {
+
+.login-form {
+  .inner {
+    background: #fff;;
+    padding: $form-edge-padding;
+    border-radius: 5px;
+  }
+}
+
+.tab-pane-inner {
     &:not(:target) {
         display: none;
     }
@@ -550,7 +582,11 @@ $form-edge-padding: 20px;
     }
     // doesnt need padding because theres another row of buttons/activity
     padding: 0px;
-  }
+}
+
+.edit-form {
+  min-width: 70%;
+
   .box-wrap {
     position: relative;
   }
@@ -562,5 +598,14 @@ $form-edge-padding: 20px;
     display: block;
     background: #fff;
   }
+
+  .pure-form-message-inline {
+    padding-left: 0;
+  }
 }
 
+ul {
+    padding-left: 1em;
+    padding-top: 0px;
+    margin-top: 4px;
+}

changedetectionio/store.py

@@ -1,12 +1,18 @@
-from os import unlink, path, mkdir
 import json
-import uuid as uuid_builder
-from threading import Lock
-from copy import deepcopy
-
 import logging
-import time
+import os
 import threading
+import time
+import uuid as uuid_builder
+from copy import deepcopy
+from os import mkdir, path, unlink
+from threading import Lock
+
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+)
 
 
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
@@ -16,6 +22,8 @@ class ChangeDetectionStore:
     lock = Lock()
 
     def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
+        # Should only be active for docker
+        # logging.basicConfig(filename='/dev/stdout', level=logging.INFO)
         self.needs_write = False
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
@@ -38,12 +46,18 @@ class ChangeDetectionStore:
                 },
                 'application': {
                     'password': False,
+                    'base_url' : None,
                     'extract_title_as_title': False,
                     'fetch_backend': 'html_requests',
+                    'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
+                    'global_subtractive_selectors': [],
+                    'ignore_whitespace': False,
                     'notification_urls': [], # Apprise URL list
                     # Custom notification content
-                    'notification_title': None,
-                    'notification_body': None,
+                    'notification_title': default_notification_title,
+                    'notification_body': default_notification_body,
+                    'notification_format': default_notification_format,
+                    'real_browser_save_screenshot': True,
                 }
             }
         }
@@ -64,15 +78,20 @@ class ChangeDetectionStore:
             'previous_md5': "",
             'uuid': str(uuid_builder.uuid4()),
             'headers': {},  # Extra headers to send
+            'body': None,
+            'method': 'GET',
             'history': {},  # Dict of timestamp and output stripped filename
             'ignore_text': [], # List of text to ignore when calculating the comparison checksum
             # Custom notification content
             'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise)
-            'notification_title': None,
-            'notification_body': None,
+            'notification_title': default_notification_title,
+            'notification_body': default_notification_body,
+            'notification_format': default_notification_format,
             'css_filter': "",
+            'subtractive_selectors': [],
             'trigger_text': [],  # List of text or regex to wait for until a change is detected
             'fetch_backend': None,
+            'extract_title_as_title': False
         }
 
         if path.isfile('changedetectionio/source.txt'):
@@ -121,7 +140,7 @@ class ChangeDetectionStore:
                 self.add_watch(url='http://www.quotationspage.com/random.php', tag='test')
                 self.add_watch(url='https://news.ycombinator.com/', tag='Tech news')
                 self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
-                self.add_watch(url='https://changedetection.io', tag='Tech news')
+                self.add_watch(url='https://changedetection.io/CHANGELOG.txt')
 
         self.__data['version_tag'] = version_tag
 
@@ -132,13 +151,19 @@ class ChangeDetectionStore:
             unlink(password_reset_lockfile)
 
         if not 'app_guid' in self.__data:
-            import sys
             import os
+            import sys
             if "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ:
                 self.__data['app_guid'] = "test-" + str(uuid_builder.uuid4())
             else:
                 self.__data['app_guid'] = str(uuid_builder.uuid4())
 
+        # Generate the URL access token for RSS feeds
+        if not 'rss_access_token' in self.__data['settings']['application']:
+            import secrets
+            secret = secrets.token_hex(16)
+            self.__data['settings']['application']['rss_access_token'] = secret
+
         self.needs_write = True
 
         # Finally start the thread that will manage periodic data saves to JSON
@@ -151,6 +176,7 @@ class ChangeDetectionStore:
 
         dates = list(self.__data['watching'][uuid]['history'].keys())
         # Convert to int, sort and back to str again
+        # @todo replace datastore getter that does this automatically
         dates = [int(i) for i in dates]
         dates.sort(reverse=True)
         if len(dates):
@@ -165,10 +191,6 @@ class ChangeDetectionStore:
 
     def update_watch(self, uuid, update_obj):
 
-        # Skip if 'paused' state
-        if self.__data['watching'][uuid]['paused']:
-            return
-
         with self.lock:
 
             # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
@@ -196,13 +218,15 @@ class ChangeDetectionStore:
                 has_unviewed = True
 
             # #106 - Be sure this is None on empty string, False, None, etc
-            if not self.__data['watching'][uuid]['title']:
-                self.__data['watching'][uuid]['title'] = None
-
             # Default var for fetch_backend
             if not self.__data['watching'][uuid]['fetch_backend']:
                 self.__data['watching'][uuid]['fetch_backend'] = self.__data['settings']['application']['fetch_backend']
 
+        # Re #152, Return env base_url if not overriden, @todo also prefer the proxy pass url
+        env_base_url = os.getenv('BASE_URL','')
+        if not self.__data['settings']['application']['base_url']:
+          self.__data['settings']['application']['base_url'] = env_base_url.strip('" ')
+
         self.__data['has_unviewed'] = has_unviewed
 
         return self.__data
@@ -247,24 +271,11 @@ class ChangeDetectionStore:
 
     # Clone a watch by UUID
     def clone(self, uuid):
-        with self.lock:
-            new_uuid = str(uuid_builder.uuid4())
-            _clone = deepcopy(self.data['watching'][uuid])
-            _clone.update({'uuid': new_uuid})
-
-            attributes_to_reset = [
-                'last_checked',
-                'last_changed',
-                'last_viewed',
-                'newest_history_key',
-                'previous_md5',
-                'history'
-            ]
-            for attribute in attributes_to_reset:
-                _clone.update({attribute: self.generic_definition[attribute]})
-
-            self.data['watching'][new_uuid] = _clone
-            self.needs_write = True
+        url = self.data['watching'][uuid]['url']
+        tag = self.data['watching'][uuid]['tag']
+        extras = self.data['watching'][uuid]
+        new_uuid = self.add_watch(url=url, tag=tag, extras=extras)
+        return new_uuid
 
     def url_exists(self, url):
 
@@ -293,10 +304,10 @@ class ChangeDetectionStore:
                 del_timestamps.append(timestamp)
                 changes_removed += 1
 
-                if not limit_timestamp:
-                    self.data['watching'][uuid]['last_checked'] = 0
-                    self.data['watching'][uuid]['last_changed'] = 0
-                    self.data['watching'][uuid]['previous_md5'] = 0
+        if not limit_timestamp:
+            self.data['watching'][uuid]['last_checked'] = 0
+            self.data['watching'][uuid]['last_changed'] = 0
+            self.data['watching'][uuid]['previous_md5'] = ""
 
 
         for timestamp in del_timestamps:
@@ -315,23 +326,33 @@ class ChangeDetectionStore:
                             content = fp.read()
                         self.data['watching'][uuid]['previous_md5'] = hashlib.md5(content).hexdigest()
                     except (FileNotFoundError, IOError):
-                        self.data['watching'][uuid]['previous_md5'] = False
+                        self.data['watching'][uuid]['previous_md5'] = ""
                         pass
 
         self.needs_write = True
         return changes_removed
 
-    def add_watch(self, url, tag):
+    def add_watch(self, url, tag="", extras=None):
+        if extras is None:
+            extras = {}
+
         with self.lock:
             # @todo use a common generic version of this
             new_uuid = str(uuid_builder.uuid4())
             _blank = deepcopy(self.generic_definition)
             _blank.update({
                 'url': url,
-                'tag': tag,
-                'uuid': new_uuid
+                'tag': tag
             })
 
+            # Incase these are copied across, assume it's a reference and deepcopy()
+            apply_extras = deepcopy(extras)
+            for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
+                if k in apply_extras:
+                    del apply_extras[k]
+
+            _blank.update(apply_extras)
+
             self.data['watching'][new_uuid] = _blank
 
         # Get the directory ready
@@ -350,6 +371,10 @@ class ChangeDetectionStore:
         import uuid
 
         output_path = "{}/{}".format(self.datastore_path, watch_uuid)
+        # Incase the operator deleted it, check and create.
+        if not os.path.isdir(output_path):
+            mkdir(output_path)
+
         fname = "{}/{}.stripped.txt".format(output_path, uuid.uuid4())
         with open(fname, 'wb') as f:
             f.write(contents)
@@ -357,21 +382,44 @@ class ChangeDetectionStore:
 
         return fname
 
+    def get_screenshot(self, watch_uuid):
+        output_path = "{}/{}".format(self.datastore_path, watch_uuid)
+        fname = "{}/last-screenshot.png".format(output_path)
+        if path.isfile(fname):
+            return fname
+
+        return False
+
+    # Save as PNG, PNG is larger but better for doing visual diff in the future
+    def save_screenshot(self, watch_uuid, screenshot: bytes):
+        output_path = "{}/{}".format(self.datastore_path, watch_uuid)
+        fname = "{}/last-screenshot.png".format(output_path)
+        with open(fname, 'wb') as f:
+            f.write(screenshot)
+            f.close()
+
     def sync_to_json(self):
-        print("Saving..")
-        data ={}
+        logging.info("Saving JSON..")
 
         try:
             data = deepcopy(self.__data)
-        except RuntimeError:
-            time.sleep(0.5)
-            print ("! Data changed when writing to JSON, trying again..")
+        except RuntimeError as e:
+            # Try again in 15 seconds
+            time.sleep(15)
+            logging.error ("! Data changed when writing to JSON, trying again.. %s", str(e))
             self.sync_to_json()
             return
         else:
-            with open(self.json_store_path, 'w') as json_file:
-                json.dump(data, json_file, indent=4)
-                logging.info("Re-saved index")
+
+            try:
+                # Re #286  - First write to a temp file, then confirm it looks OK and rename it
+                # This is a fairly basic strategy to deal with the case that the file is corrupted,
+                # system was out of memory, out of RAM etc
+                with open(self.json_store_path+".tmp", 'w') as json_file:
+                    json.dump(data, json_file, indent=4)
+                os.replace(self.json_store_path+".tmp", self.json_store_path)
+            except Exception as e:
+                logging.error("Error writing JSON!! (Main JSON file save was skipped) : %s", str(e))
 
             self.needs_write = False
 
@@ -386,7 +434,13 @@ class ChangeDetectionStore:
 
             if self.needs_write:
                 self.sync_to_json()
-            time.sleep(3)
+
+            # Once per minute is enough, more and it can cause high CPU usage
+            # better here is to use something like self.app.config.exit.wait(1), but we cant get to 'app' from here
+            for i in range(30):
+                time.sleep(2)
+                if self.stop_thread:
+                    break
 
     # Go through the datastore path and remove any snapshots that are not mentioned in the index
     # This usually is not used, but can be handy.
@@ -399,6 +453,7 @@ class ChangeDetectionStore:
                 index.append(self.data['watching'][uuid]['history'][str(id)])
 
         import pathlib
+
         # Only in the sub-directories
         for item in pathlib.Path(self.datastore_path).rglob("*/*txt"):
             if not str(item) in index:

changedetectionio/templates/_common_fields.jinja

@@ -1,11 +1,8 @@
 
 {% from '_helpers.jinja' import render_field %}
 
-{% macro render_notifications_field(form) %}
+{% macro render_common_settings_form(form, current_base_url, emailprefix) %}
 
-
-<fieldset>
-                    <div class="field-group">
                         <div class="pure-control-group">
                             {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
     Gitter - gitter://token/room
@@ -13,10 +10,20 @@
     AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
     SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com")
                             }}
-                            <div class="pure-form-message-inline">Use <a target=_new
-                                                                         href="https://github.com/caronc/apprise">AppRise
-                                URLs</a> for notification to just about any service!
+                            <div class="pure-form-message-inline">
+                              <ul>
+                                <li>Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.</li>
+                                <li><code>discord://</code> will silently fail if the total message length is more than 2000 chars.</li>
+                                <li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
+                                <li>Go here for <a href="{{url_for('notification_logs')}}">Notification debug logs</a></li>
+                              </ul>
                             </div>
+                            <br/>
+                            <a id="send-test-notification" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Send test notification</a>
+{% if emailprefix %}
+                            <a id="add-email-helper" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Add email</a>
+
+{% endif %}
                         </div>
                         <div id="notification-customisation">
                             <div class="pure-control-group">
@@ -27,11 +34,14 @@
                                 {{ render_field(form.notification_body , rows=5) }}
                                 <span class="pure-form-message-inline">Body for all notifications</span>
                             </div>
+                            <div class="pure-control-group">
+                                {{ render_field(form.notification_format , rows=5) }}
+                                <span class="pure-form-message-inline">Format for all notifications</span>
+                            </div>
                             <div class="pure-controls">
                             <span class="pure-form-message-inline">
-                                These tokens can be used in the notification body and title to
-                                customise the notification text.
-                            </span>
+                                These tokens can be used in the notification body and title to customise the notification text.
+
                                 <table class="pure-table" id="token-table">
                                     <thead>
                                     <tr>
@@ -64,6 +74,14 @@
                                         <td><code>{preview_url}</code></td>
                                         <td>The URL of the preview page generated by changedetection.io.</td>
                                     </tr>
+                                    <tr>
+                                        <td><code>{diff}</code></td>
+                                        <td>The diff output - differences only</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{diff_full}</code></td>
+                                        <td>The diff output - full difference output</td>
+                                    </tr>
                                     <tr>
                                         <td><code>{diff_url}</code></td>
                                         <td>The URL of the diff page generated by changedetection.io.</td>
@@ -75,16 +93,10 @@
                                     </tr>
                                     </tbody>
                                 </table>
-                                <span class="pure-form-message-inline">
+                                <br/>
                                 URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
-                                Your <code>BASE_URL</code> var is currently "{{base_url}}"
+                                Your <code>BASE_URL</code> var is currently "{{current_base_url}}"
                             </span>
                             </div>
                         </div>
-                        <div class="pure-control-group">
-                            {{ render_field(form.trigger_check) }}
-                        </div>
-                    </div>
-
-                </fieldset>
-{% endmacro %}
+{% endmacro %}

changedetectionio/templates/_helpers.jinja

@@ -25,3 +25,6 @@
 {% endmacro %}
 
 
+{% macro render_button(field) %}
+  {{ field(**kwargs)|safe }}
+{% endmacro %}

changedetectionio/templates/base.html

@@ -12,7 +12,14 @@
         <link rel="stylesheet" href="{{ m }}?ver=1000">
         {% endfor %}
     {% endif %}
+    <style>
+    body::before {
+        background-image: url({{url_for('static_content', group='images', filename='gradient-border.png')}});
+    }
+    </style>
+    <script type="text/javascript" src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
 </head>
+
 <body>
 
 <div class="header">
@@ -35,13 +42,13 @@
         {% if current_user.is_authenticated or not has_password %}
             {% if not current_diff_url %}
             <li class="pure-menu-item">
-                <a href="{{ url_for('get_backup')}}" class="pure-menu-link">BACKUP</a>
+                <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
             </li>
             <li class="pure-menu-item">
                 <a href="{{ url_for('import_page')}}" class="pure-menu-link">IMPORT</a>
             </li>
             <li class="pure-menu-item">
-                <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
+                <a href="{{ url_for('get_backup')}}" class="pure-menu-link">BACKUP</a>
             </li>
             {% else %}
             <li class="pure-menu-item">
@@ -68,7 +75,7 @@
         </ul>
     </div>
 </div>
-
+{% if hosted_sticky %}<div class="sticky-tab" id="hosted-sticky"><a href="https://lemonade.changedetection.io/start?ref={{guid}}">Let us host your instance!</a></div>{% endif %}
 {% if left_sticky %}<div class="sticky-tab" id="left-sticky"><a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a></div> {% endif %}
 {% if right_sticky %}<div class="sticky-tab" id="right-sticky">{{ right_sticky }}</div> {% endif %}
 <section class="content">

changedetectionio/templates/diff.html

@@ -1,7 +1,6 @@
 {% extends 'base.html' %}
 
 {% block content %}
-
 <div id="settings">
     <h1>Differences</h1>
     <form class="pure-form " action="" method="GET">
@@ -35,20 +34,45 @@
 <div id="diff-jump">
     <a onclick="next_diff();">Jump</a>
 </div>
+
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<div class="tabs">
+    <ul>
+        <li class="tab" id="default-tab"><a href="#text">Text</a></li>
+{% if screenshot %}
+        <li class="tab"><a href="#screenshot">Screenshot</a></li>
+{% endif %}
+    </ul>
+</div>
+
 <div id="diff-ui">
-    <table>
-        <tbody>
-        <tr>
-            <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
-            <td id="a" style="display: none;">{{previous}}</td>
-            <td id="b" style="display: none;">{{newest}}</td>
-            <td id="diff-col">
-                <span id="result"></span>
-            </td>
-        </tr>
-        </tbody>
-    </table>
-    Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff">github.com/kpdecker/jsdiff</a>
+     <div class="tab-pane-inner" id="text">
+         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.
+         </div>
+         <table>
+             <tbody>
+             <tr>
+                 <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
+                 <td id="a" style="display: none;">{{previous}}</td>
+                 <td id="b" style="display: none;">{{newest}}</td>
+                 <td id="diff-col">
+                     <span id="result"></span>
+                 </td>
+             </tr>
+             </tbody>
+         </table>
+         Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff">github.com/kpdecker/jsdiff</a>
+     </div>
+
+{% if screenshot %}
+     <div class="tab-pane-inner" id="screenshot">
+         <p>
+         <i>For now, only the most recent screenshot is saved and displayed.</i>
+             </p>
+
+        <img src="{{url_for('static_content', group='screenshot', filename=uuid)}}">
+     </div>
+{% endif %}
 
 </div>
 

changedetectionio/templates/edit.html

@@ -1,28 +1,38 @@
 {% extends 'base.html' %}
 {% block content %}
 {% from '_helpers.jinja' import render_field %}
-{% from '_common_fields.jinja' import render_notifications_field %}
+{% from '_helpers.jinja' import render_button %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script>
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+{% if emailprefix %}
+    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
+{% endif %}
+</script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 
 <div class="edit-form monospaced-textarea">
 
-    <div class="tabs">
+    <div class="tabs collapsable">
         <ul>
             <li class="tab" id="default-tab"><a href="#general">General</a></li>
+            <li class="tab"><a href="#request">Request</a></li>
+            <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
             <li class="tab"><a href="#notifications">Notifications</a></li>
-            <li class="tab"><a href="#filters">Filters</a></li>
-            <li class="tab"><a href="#triggers">Triggers</a></li>
         </ul>
     </div>
 
     <div class="box-wrap inner">
         <form class="pure-form pure-form-stacked"
               action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next') ) }}" method="POST">
+             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
 
             <div class="tab-pane-inner" id="general">
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
+                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span>
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.title, class="m-d") }}
@@ -41,75 +51,137 @@
                                 href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
                         {% endif %}
                     </div>
-                    <fieldset class="pure-group">
-                        {{ render_field(form.headers, rows=5, placeholder="Example
-Cookie: foobar
-User-Agent: wonderbra 1.0") }}
-                        <span class="pure-form-message-inline">
-                            Note: ONLY used by Basic fast Plaintext/HTTP Client
-                          </span>
-                    </fieldset>
                     <div class="pure-control-group">
-                        {{ render_field(form.fetch_backend) }}
-                        <span class="pure-form-message-inline">
-                            <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
-                            <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server. </p>
-                        </span>
+                        {{ render_field(form.extract_title_as_title) }}
                     </div>
                 </fieldset>
             </div>
 
-            <div class="tab-pane-inner" id="notifications">
-                <strong>Note: <i>These settings override the global settings.</i></strong>
-                {{ render_notifications_field(form) }}
+            <div class="tab-pane-inner" id="request">
+                    <div class="pure-control-group">
+                        {{ render_field(form.fetch_backend) }}
+                        <span class="pure-form-message-inline">
+                            <p>Use the <strong>Basic</strong> method (default) where your watched site doesn't need Javascript to render.</p>
+                            <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
+                        </span>
+                    </div>
+
+                <hr/>
+                <fieldset class="pure-group">
+
+                    <span class="pure-form-message-inline">
+                        <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
+                    </span>
+                    <div class="pure-control-group">
+                        {{ render_field(form.method) }}
+                    </div>
+                    <div class="pure-control-group">
+{{ render_field(form.headers, rows=5, placeholder="Example
+Cookie: foobar
+User-Agent: wonderbra 1.0") }}
+                    </div>
+                    <div class="pure-control-group">
+                                        {{ render_field(form.body, rows=5, placeholder="Example
+{
+   \"name\":\"John\",
+   \"age\":30,
+   \"car\":null
+}") }}
+                    </div>
+                    <div>
+                        {{ render_field(form.ignore_status_codes) }}
+                    </div>
+                </fieldset>
+                <br/>
             </div>
 
-            <div class="tab-pane-inner" id="filters">
+            <div class="tab-pane-inner" id="notifications">
+                <strong>Note: <i>These settings override the global settings for this watch.</i></strong>
                 <fieldset>
+                    <div class="field-group">
+                        {{ render_common_settings_form(form, current_base_url, emailprefix) }}
+                    </div>
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="filters-and-triggers">
+                <fieldset>
+                        <div class="pure-control-group">
+                            <strong>Pro-tips:</strong><br/>
+                            <ul>
+                                <li>
+                                    Use the preview page to see your filters and triggers highlighted.
+                                </li>
+                                <li>
+                                    Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a>
+                                </li>
+                            </ul>
+                    </div>
+
                     <div class="pure-control-group">
                         {{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.",
                         class="m-d") }}
                         <span class="pure-form-message-inline">
                     <ul>
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
-                        <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <b>"json:"</b>, <a
+                        <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <code>"json:"</code>, use <code>json:$</code> to force re-formatting if required,  <a
                                 href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
+                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash, example  <code>//*[contains(@class, 'sametext')]</code>, <a
+                                href="http://xpather.com/" target="new">test your XPath here</a></li>
                     </ul>
-                    Please be sure that you thoroughly understand how to write CSS or JSONPath selector rules before filing an issue on GitHub! <a
+                    Please be sure that you thoroughly understand how to write CSS or JSONPath, XPath selector rules before filing an issue on GitHub! <a
                                 href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
                 </span>
                     </div>
-
+                    <fieldset class="pure-group">
+                      {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
+footer
+nav
+.stockticker") }}
+                      <span class="pure-form-message-inline">
+                        <ul>
+                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
+                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
+                        </ul>
+                      </span>
+                    </fieldset>
                 </fieldset>
                 <fieldset class="pure-group">
                     {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
 /some.regex\d{2}/ for case-INsensitive regex
                     ") }}
                     <span class="pure-form-message-inline">
-                    Each line processed separately, any line matching will be ignored.<br/>
-                    Regular Expression support, wrap the line in forward slash <b>/regex/</b>.
+                        <ul>
+                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
+                            <li>Regular Expression support, wrap the line in forward slash <code>/regex/</code></li>
+                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
+                            <li>Use the preview/show current tab to see ignores</li>
+                        </ul>
                 </span>
 
             </fieldset>
-            </div>
-
-            <div class="tab-pane-inner" id="triggers">
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line
 /some.regex\d{2}/ for case-INsensitive regex
-                    ") }}</br>
-                        <span class="pure-form-message-inline">Text to wait for before triggering a change/notification, all text and regex are tested <i>case-insensitive</i>.</span><br/>
-                        <span class="pure-form-message-inline">Trigger text is processed from the result-text that comes out of any <a href="#filters">CSS/JSON Filters</a> for this watch</span>.<br/>
-                        <span class="pure-form-message-inline">Each line is process separately (think of each line as "OR")</span><br/>
-                        <span class="pure-form-message-inline">Note: Wrap in forward slash / to use regex  example: <span style="font-family: monospace; background: #eee">/foo\d/</span> </span>
+                    ") }}
+                        <span class="pure-form-message-inline">
+                    <ul>
+                        <li>Text to wait for before triggering a change/notification, all text and regex are tested <i>case-insensitive</i>.</li>
+                        <li>Trigger text is processed from the result-text that comes out of any CSS/JSON Filters for this watch</li>
+                        <li>Each line is processed separately (think of each line as "OR")</li>
+                        <li>Note: Wrap in forward slash / to use regex  example: <code>/foo\d/</code></li>
+                    </ul>
+                        </span>
                     </div>
                 </fieldset>
             </div>
+
             <div id="actions">
                 <div class="pure-control-group">
 
-                    <button type="submit" class="pure-button pure-button-primary">Save</button>
+                      {{ render_button(form.save_button) }} {{ render_button(form.save_and_preview_button) }}
+
                     <a href="{{url_for('api_delete', uuid=uuid)}}"
                        class="pure-button button-small button-error ">Delete</a>
                     <a href="{{url_for('api_clone', uuid=uuid)}}"

changedetectionio/templates/import.html

@@ -4,8 +4,16 @@
 <div class="edit-form">
      <div class="inner">
         <form class="pure-form pure-form-aligned" action="{{url_for('import_page')}}" method="POST">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
             <fieldset class="pure-group">
-                <legend>One URL per line, URLs that do not pass validation will stay in the textarea.</legend>
+              <legend>
+                Enter one URL per line, and optionally add tags for each URL after a space, delineated by comma (,):
+                <br>
+                <code>https://example.com tag1, tag2, last tag</code>
+                <br>
+                URLs which do not pass validation will stay in the textarea.
+              </legend>
+              
 
                 <textarea name="urls" class="pure-input-1-2" placeholder="https://"
                           style="width: 100%;
@@ -20,4 +28,3 @@
 </div>
 
 {% endblock %}
-

changedetectionio/templates/login.html

@@ -1,15 +1,15 @@
 {% extends 'base.html' %}
 
 {% block content %}
-<div class="edit-form">
-
+<div class="login-form">
  <div class="inner">
     <form class="pure-form pure-form-stacked" action="{{url_for('login')}}" method="POST">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <div class="pure-control-group">
                 <label for="password">Password</label>
                 <input type="password" id="password" required="" name="password" value=""
-                       size="15"/>
+                       size="15" autofocus />
                 <input type="hidden" id="email" name="email" value="defaultuser@changedetection.io" />
             </div>
             <div class="pure-control-group">

changedetectionio/templates/notification-log.html

@@ -0,0 +1,19 @@
+{% extends 'base.html' %}
+
+{% block content %}
+<div class="edit-form">
+     <div class="inner">
+
+         <h4 style="margin-top: 0px;">The following issues were detected when sending notifications</h4>
+                <div id="notification-customisation">
+                <ul style="font-size: 80%; margin:0px; padding: 0 0 0 7px">
+                {% for log in logs|reverse %}
+                    <li>{{log}}</li>
+                {% endfor %}
+                </ul>
+                </div>
+
+     </div>
+</div>
+
+{% endblock %}

changedetectionio/templates/preview.html

@@ -3,24 +3,43 @@
 {% block content %}
 
 <div id="settings">
-    <h1>Current</h1>
+    <h1>Current - {{watch.last_checked|format_timestamp_timeago}}</h1>
 </div>
 
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<div class="tabs">
+    <ul>
+        <li class="tab" id="default-tab"><a href="#text">Text</a></li>
+{% if screenshot %}
+        <li class="tab"><a href="#screenshot">Screenshot</a></li>
+{% endif %}
+    </ul>
+</div>
 
 <div id="diff-ui">
+    <div class="tab-pane-inner" id="text">
+        <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
+        <table>
+            <tbody>
+            <tr>
+                <td id="diff-col">
+                    {% for row in content %}
+                    <div class="{{row.classes}}">{{row.line}}</div>
+                    {% endfor %}
+                </td>
+            </tr>
+            </tbody>
+        </table>
+    </div>
 
-    <table>
-        <tbody>
-        <tr>
-            <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
+{% if screenshot %}
+     <div class="tab-pane-inner" id="screenshot">
+         <p>
+         <i>For now, only the most recent screenshot is saved and displayed.</i>
+             </p>
 
-            <td id="diff-col">
-                <span id="result">{% for row in content %}<pre>{{row}}</pre>{% endfor %}</span>
-            </td>
-        </tr>
-        </tbody>
-    </table>
+        <img src="{{url_for('static_content', group='screenshot', filename=uuid)}}">
+     </div>
+{% endif %}
 </div>
-
-
 {% endblock %}

changedetectionio/templates/scrub.html

@@ -2,7 +2,9 @@
 
 {% block content %}
 <div class="edit-form">
+    <div class="box-wrap inner">
     <form class="pure-form pure-form-stacked" action="{{url_for('scrub_page')}}" method="POST">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <div class="pure-control-group">
                 This will remove all version snapshots/data, but keep your list of URLs. <br/>
@@ -30,6 +32,7 @@
             </div>
         </fieldset>
     </form>
+    </div>
 </div>
 
 {% endblock %}

changedetectionio/templates/settings.html

@@ -1,22 +1,30 @@
 {% extends 'base.html' %}
 
 {% block content %}
-{% from '_helpers.jinja' import render_field %}
-{% from '_common_fields.jinja' import render_notifications_field %}
-
+{% from '_helpers.jinja' import render_field, render_button %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
+<script>
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+{% if emailprefix %}
+    const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
+{% endif %}
+</script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='settings.js')}}" defer></script>
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 
 <div class="edit-form">
-    <div class="tabs">
+    <div class="tabs collapsable">
         <ul>
             <li class="tab" id="default-tab"><a href="#general">General</a></li>
             <li class="tab"><a href="#notifications">Notifications</a></li>
             <li class="tab"><a href="#fetching">Fetching</a></li>
+            <li class="tab"><a href="#filters">Global Filters</a></li>
         </ul>
     </div>
     <div class="box-wrap inner">
         <form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
             <div class="tab-pane-inner" id="general">
                 <fieldset>
                     <div class="pure-control-group">
@@ -24,23 +32,47 @@
                         <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
                     </div>
                     <div class="pure-control-group">
-                        {% if current_user.is_authenticated %}
-                        <a href="{{url_for('settings_page', removepassword='yes')}}"
-                           class="pure-button pure-button-primary">Remove password</a>
+                        {% if not hide_remove_pass %}
+                            {% if current_user.is_authenticated %}
+                                {{ render_button(form.removepassword_button) }}
+                            {% else %}
+                            {{ render_field(form.password) }}
+                            <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
+                            {% endif %}
                         {% else %}
-                        {{ render_field(form.password) }}
-                        <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
+                            <span class="pure-form-message-inline">Password is locked.</span>
                         {% endif %}
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.base_url, placeholder="http://yoursite.com:5000/",
+                        class="m-d") }}
+                        <span class="pure-form-message-inline">
+                            Base URL used for the {base_url} token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{current_base_url}}"),
+                            <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
+                        </span>
+                    </div>
+
                     <div class="pure-control-group">
                         {{ render_field(form.extract_title_as_title) }}
                         <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
                     </div>
+
+                    <div class="pure-control-group">
+                        {{ render_field(form.real_browser_save_screenshot) }}
+                        <span class="pure-form-message-inline">When using a Chrome browser, a screenshot from the last check will be available on the Diff page</span>
+                    </div>
+
                 </fieldset>
             </div>
 
             <div class="tab-pane-inner" id="notifications">
-                {{ render_notifications_field(form) }}
+                <fieldset>
+                    <div class="field-group">
+                        {{ render_common_settings_form(form, current_base_url, emailprefix) }}
+                    </div>
+                </fieldset>
+                <a href="{{url_for('notification_logs')}}">Notification debug logs</a>
+
             </div>
 
             <div class="tab-pane-inner" id="fetching">
@@ -48,17 +80,55 @@
                     {{ render_field(form.fetch_backend) }}
                     <span class="pure-form-message-inline">
                         <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
-                        <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server. </p>
+                        <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
                     </span>
                 </div>
             </div>
+
+
+            <div class="tab-pane-inner" id="filters">
+
+                    <fieldset class="pure-group">
+                    {{ render_field(form.ignore_whitespace) }}
+                    <span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br/>
+                    <i>Note:</i> Changing this will change the status of your existing watches, possibily trigger alerts etc.
+                    </span>
+                    </fieldset>
+
+                    <fieldset class="pure-group">
+                      {{ render_field(form.global_subtractive_selectors, rows=5, placeholder="header
+footer
+nav
+.stockticker") }}
+                      <span class="pure-form-message-inline">
+                        <ul>
+                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
+                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
+                        </ul>
+                      </span>
+                    </fieldset>
+                    <fieldset class="pure-group">
+                    {{ render_field(form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
+/some.regex\d{2}/ for case-INsensitive regex
+                    ") }}
+                    <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br/>
+                    <span class="pure-form-message-inline">
+                        <ul>
+                            <li>Note: This is applied globally in addition to the per-watch rules.</li>
+                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
+                            <li>Regular Expression support, wrap the line in forward slash <code>/regex/</code></li>
+                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
+                            <li>Use the preview/show current tab to see ignores</li>
+                        </ul>
+                     </span>
+                    </fieldset>
+           </div>
+
             <div id="actions">
                 <div class="pure-control-group">
-                    <button type="submit" class="pure-button pure-button-primary">Save</button>
-                                           <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
-                        <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete
-                            History
-                            Snapshot Data</a>
+                    {{ render_button(form.save_button) }}
+                    <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
+                    <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
                 </div>
             </div>
         </form>

changedetectionio/templates/watch-overview.html

@@ -1,10 +1,12 @@
 {% extends 'base.html' %}
 {% block content %}
 {% from '_helpers.jinja' import render_simple_field %}
-
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
 <div class="box">
 
     <form class="pure-form" action="{{ url_for('api_watch_add') }}" method="POST" id="new-watch-form">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <legend>Add a new change detection watch</legend>
                 {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
@@ -42,6 +44,7 @@
             <tr id="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
+                {% if watch.last_notification_error is defined and watch.last_notification_error != False %}error{% endif %}
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
                 {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
                 <td class="inline">{{ loop.index }}</td>
@@ -49,11 +52,14 @@
 
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.url }}"></a>
-                    {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="/static/images/Google-Chrome-icon.png" />{% endif %}
+                    {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
 
                     {% if watch.last_error is defined and watch.last_error != False %}
                     <div class="fetch-error">{{ watch.last_error }}</div>
                     {% endif %}
+                    {% if watch.last_notification_error is defined and watch.last_notification_error != False %}
+                    <div class="fetch-error notification-error">{{ watch.last_notification_error }}</div>
+                    {% endif %}
                     {% if not active_tag %}
                     <span class="watch-tag-list">{{ watch.tag}}</span>
                     {% endif %}
@@ -70,7 +76,7 @@
                        class="pure-button button-small pure-button-primary">Recheck</a>
                     <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
                     {% if watch.history|length >= 2 %}
-                    <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
+                    <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary diff-link">Diff</a>
                     {% else %}
                         {% if watch.history|length == 1 %}
                             <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
@@ -92,9 +98,9 @@
                 all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
             </li>
             <li>
-                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15px"></a>
+                <a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>
             </li>
         </ul>
     </div>
 </div>
-{% endblock %}
+{% endblock %}

changedetectionio/tests/conftest.py

@@ -18,11 +18,11 @@ def cleanup(datastore_path):
              'url-watches.json',
              'notification.txt',
              'count.txt',
-             'endpoint-content.txt']
+             'endpoint-content.txt'
+                 ]
     for file in files:
         try:
             os.unlink("{}/{}".format(datastore_path, file))
-            x = 1
         except FileNotFoundError:
             pass
 
@@ -36,15 +36,15 @@ def app(request):
     except FileExistsError:
         pass
 
-    # Enable a BASE_URL for notifications to work (so we can look for diff/ etc URLs)
-    os.environ["BASE_URL"] = "http://mysite.com/"
     cleanup(datastore_path)
 
-
     app_config = {'datastore_path': datastore_path}
     cleanup(app_config['datastore_path'])
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
     app = changedetection_app(app_config, datastore)
+
+    # Disable CSRF while running tests
+    app.config['WTF_CSRF_ENABLED'] = False
     app.config['STOP_THREADS'] = True
 
     def teardown():

changedetectionio/tests/test_access_control.py

@@ -4,8 +4,8 @@ from flask import url_for
 def test_check_access_control(app, client):
     # Still doesnt work, but this is closer.
 
-    with app.test_client() as c:
-        # Check we dont have any password protection enabled yet.
+    with app.test_client(use_cookies=True) as c:
+        # Check we don't have any password protection enabled yet.
         res = c.get(url_for("settings_page"))
         assert b"Remove password" not in res.data
 
@@ -46,15 +46,20 @@ def test_check_access_control(app, client):
         assert b"BACKUP" in res.data
         assert b"IMPORT" in res.data
         assert b"LOG OUT" in res.data
+        assert b"minutes_between_check" in res.data
+        assert b"fetch_backend" in res.data
 
-        # Now remove the password so other tests function, @todo this should happen before each test automatically
-        res = c.get(url_for("settings_page", removepassword="yes"),
-              follow_redirects=True)
-        assert b"Password protection removed." in res.data
-
-        res = c.get(url_for("index"))
-        assert b"LOG OUT" not in res.data
-
+        res = c.post(
+            url_for("settings_page"),
+            data={
+                "minutes_between_check": 180,
+                "tag": "",
+                "headers": "",
+                "fetch_backend": "html_webdriver",
+                "removepassword_button": "Remove password"
+            },
+            follow_redirects=True,
+        )
 
 # There was a bug where saving the settings form would submit a blank password
 def test_check_access_control_no_blank_password(app, client):
@@ -71,8 +76,7 @@ def test_check_access_control_no_blank_password(app, client):
             data={"password": "",
                   "minutes_between_check": 180,
                   'fetch_backend': "html_requests"},
-
-        follow_redirects=True
+            follow_redirects=True
         )
 
         assert b"Password protection enabled." not in res.data
@@ -91,7 +95,8 @@ def test_check_access_no_remote_access_to_remove_password(app, client):
         # Enable password check.
         res = c.post(
             url_for("settings_page"),
-            data={"password": "password", "minutes_between_check": 180,
+            data={"password": "password",
+                  "minutes_between_check": 180,
                   'fetch_backend': "html_requests"},
             follow_redirects=True
         )
@@ -99,8 +104,17 @@ def test_check_access_no_remote_access_to_remove_password(app, client):
         assert b"Password protection enabled." in res.data
         assert b"Login" in res.data
 
-        res = c.get(url_for("settings_page", removepassword="yes"),
-              follow_redirects=True)
+        res = c.post(
+            url_for("settings_page"),
+            data={
+                "minutes_between_check": 180,
+                "tag": "",
+                "headers": "",
+                "fetch_backend": "html_webdriver",
+                "removepassword_button": "Remove password"
+            },
+            follow_redirects=True,
+        )
         assert b"Password protection removed." not in res.data
 
         res = c.get(url_for("index"),

changedetectionio/tests/test_api.py

@@ -0,0 +1,73 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def set_response_data(test_return_data):
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_snapshot_api_detects_change(client, live_server):
+    test_return_data = "Some initial text"
+
+    test_return_data_modified = "Some NEW nice initial text"
+
+    sleep_time_for_fetch_thread = 3
+
+    set_response_data(test_return_data)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(
+        url_for("api_snapshot", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert test_return_data.encode() == res.data
+
+    #  Make a change
+    set_response_data(test_return_data_modified)
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(
+        url_for("api_snapshot", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert test_return_data_modified.encode() == res.data
+
+def test_snapshot_api_invalid_uuid(client, live_server):
+
+    res = client.get(
+        url_for("api_snapshot", uuid="invalid"),
+        follow_redirects=True
+    )
+
+    assert res.status_code == 400
+

changedetectionio/tests/test_auth.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_basic_auth(client, live_server):
+
+    live_server_setup(live_server)
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@")
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Check form validation
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(1)
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b'myuser mypass basic' in res.data

changedetectionio/tests/test_backend.py

@@ -7,6 +7,13 @@ from . util import set_original_response, set_modified_response, live_server_set
 
 sleep_time_for_fetch_thread = 3
 
+# Basic test to check inscriptus is not adding return line chars, basically works etc
+def test_inscriptus():
+    from inscriptis import get_text
+    html_content="<html><body>test!<br/>ok man</body></html>"
+    stripped_text_from_html = get_text(html_content)
+    assert stripped_text_from_html == 'test!\nok man'
+
 
 def test_check_basic_change_detection_functionality(client, live_server):
     set_original_response()
@@ -18,6 +25,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
         data={"urls": url_for('test_endpoint', _external=True)},
         follow_redirects=True
     )
+
     assert b"1 Imported" in res.data
 
     time.sleep(sleep_time_for_fetch_thread)
@@ -50,7 +58,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
 
     # Force recheck
     res = client.get(url_for("api_watch_checknow"), follow_redirects=True)
-    assert b'1 watches are rechecking.' in res.data
+    assert b'1 watches are queued for rechecking.' in res.data
 
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -59,9 +67,14 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'unviewed' in res.data
 
     # #75, and it should be in the RSS feed
-    res = client.get(url_for("index", rss="true"))
+    res = client.get(url_for("rss"))
     expected_url = url_for('test_endpoint', _external=True)
     assert b'<rss' in res.data
+
+    # re #16 should have the diff in here too
+    assert b'(into   ) which has this one new line' in res.data
+    assert b'CDATA' in res.data
+    
     assert expected_url.encode('utf-8') in res.data
 
     # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times

changedetectionio/tests/test_backup.py

@@ -0,0 +1,25 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from urllib.request import urlopen
+from . util import set_original_response, set_modified_response, live_server_setup
+
+
+def test_backup(client, live_server):
+
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    res = client.get(
+        url_for("get_backup"),
+        follow_redirects=True
+    )
+
+    # Should get the right zip content type
+    assert res.content_type == "application/zip"
+    # Should be PK/ZIP stream
+    assert res.data.count(b'PK') >= 2
+

changedetectionio/tests/test_clone.py

@@ -0,0 +1,30 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+
+
+def test_trigger_functionality(client, live_server):
+
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": "https://changedetection.io"},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+
+    res = client.get(
+        url_for("api_clone", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"Cloned." in res.data

changedetectionio/tests/test_element_removal.py

@@ -0,0 +1,168 @@
+#!/usr/bin/python3
+
+import time
+
+from flask import url_for
+
+from ..html_tools import *
+from .util import live_server_setup
+
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def set_original_response():
+    test_return_data = """<html>
+    <header>
+    <h2>Header</h2>
+    </header>
+    <nav>
+    <ul>
+      <li><a href="#">A</a></li>
+      <li><a href="#">B</a></li>
+      <li><a href="#">C</a></li>
+    </ul>
+    </nav>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+    <div id="changetext">Some text that will change</div>
+     </body>
+    <footer>
+    <p>Footer</p>
+    </footer>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_modified_response():
+    test_return_data = """<html>
+    <header>
+    <h2>Header changed</h2>
+    </header>
+    <nav>
+    <ul>
+      <li><a href="#">A changed</a></li>
+      <li><a href="#">B</a></li>
+      <li><a href="#">C</a></li>
+    </ul>
+    </nav>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+    <div id="changetext">Some text that changes</div>
+     </body>
+    <footer>
+    <p>Footer changed</p>
+    </footer>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_element_removal_output():
+    from changedetectionio import fetch_site_status
+    from inscriptis import get_text
+
+    # Check text with sub-parts renders correctly
+    content = """<html>
+    <header>
+    <h2>Header</h2>
+    </header>
+    <nav>
+    <ul>
+      <li><a href="#">A</a></li>
+    </ul>
+    </nav>
+       <body>
+     Some initial text</br>
+     <p>across multiple lines</p>
+     <div id="changetext">Some text that changes</div>
+     </body>
+    <footer>
+    <p>Footer</p>
+    </footer>
+     </html>
+    """
+    html_blob = element_removal(
+        ["header", "footer", "nav", "#changetext"], html_content=content
+    )
+    text = get_text(html_blob)
+    assert (
+        text
+        == """Some initial text
+
+across multiple lines
+"""
+    )
+
+
+def test_element_removal_full(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for("test_endpoint", _external=True)
+    res = client.post(
+        url_for("import_page"), data={"urls": test_url}, follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Goto the edit page, add the filter data
+    # Not sure why \r needs to be added - absent of the #changetext this is not necessary
+    subtractive_selectors_data = "header\r\nfooter\r\nnav\r\n#changetext"
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "subtractive_selectors": subtractive_selectors_data,
+            "url": test_url,
+            "tag": "",
+            "headers": "",
+            "fetch_backend": "html_requests",
+        },
+        follow_redirects=True,
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(subtractive_selectors_data.encode("utf-8")) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # No change yet - first check
+    res = client.get(url_for("index"))
+    assert b"unviewed" not in res.data
+
+    #  Make a change to header/footer/nav
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # There should not be an unviewed change, as changes should be removed
+    res = client.get(url_for("index"))
+    assert b"unviewed" not in res.data

changedetectionio/tests/test_encoding.py

@@ -0,0 +1,87 @@
+#!/usr/bin/python3
+# coding=utf-8
+
+import time
+from flask import url_for
+from .util import live_server_setup
+import pytest
+
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def set_html_response():
+    test_return_data = """
+<html><body><span class="nav_second_img_text">
+                  &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;铸大国重器，挺制造脊梁，致力能源未来，赋能美好生活。
+                                  </span>
+</body></html>
+    """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+
+# In the case the server does not issue a charset= or doesnt have content_type header set
+def test_check_encoding_detection(client, live_server):
+    set_html_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="text/html", _external=True)
+    client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(2)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should see the proper string
+    assert "铸大国重".encode('utf-8') in res.data
+    # Should not see the failed encoding
+    assert b'\xc2\xa7' not in res.data
+
+
+# In the case the server does not issue a charset= or doesnt have content_type header set
+def test_check_encoding_detection_missing_content_type_header(client, live_server):
+    set_html_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(2)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should see the proper string
+    assert "铸大国重".encode('utf-8') in res.data
+    # Should not see the failed encoding
+    assert b'\xc2\xa7' not in res.data

changedetectionio/tests/test_errorhandling.py

@@ -0,0 +1,65 @@
+#!/usr/bin/python3
+
+import time
+
+from flask import url_for
+from . util import live_server_setup
+
+from ..html_tools import *
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def test_error_handler(client, live_server):
+
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint',
+                       status_code=403,
+                       _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'Status Code 403' in res.data
+    assert bytes("just now".encode('utf-8')) in res.data
+
+# Just to be sure error text is properly handled
+def test_error_text_handler(client, live_server):
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": "https://errorfuldomainthatnevereallyexists12356.com"},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    res = client.get(url_for("index"))
+    assert b'Name or service not known' in res.data
+    assert bytes("just now".encode('utf-8')) in res.data
+

changedetectionio/tests/test_headers.py

@@ -1,80 +0,0 @@
-import json
-import time
-from flask import url_for
-from . util import set_original_response, set_modified_response, live_server_setup
-
-# Hard to just add more live server URLs when one test is already running (I think)
-# So we add our test here (was in a different file)
-def test_headers_in_request(client, live_server):
-    live_server_setup(live_server)
-
-    # Add our URL to the import page
-    test_url = url_for('test_headers', _external=True)
-
-    # Add the test URL twice, we will check
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;'
-
-
-    # Add some headers to a request
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-              "url": test_url,
-              "tag": "",
-              "fetch_backend": "html_requests",
-              "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
-
-    # Give the thread time to pick up the first version
-    time.sleep(5)
-
-    # The service should echo back the request headers
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    # Flask will convert the header key to uppercase
-    assert b"Xxx:ooo" in res.data
-    assert b"Cool:yeah" in res.data
-
-    # The test call service will return the headers as the body
-    from html import escape
-    assert escape(cookie_header).encode('utf-8') in res.data
-
-    time.sleep(5)
-
-    # Re #137 -  Examine the JSON index file, it should have only one set of headers entered
-    watches_with_headers = 0
-    with open('test-datastore/url-watches.json') as f:
-        app_struct = json.load(f)
-        for uuid in app_struct['watching']:
-            if (len(app_struct['watching'][uuid]['headers'])):
-                watches_with_headers += 1
-
-    # Should be only one with headers set
-    assert watches_with_headers==1
-
-
-
-
-
-
-

changedetectionio/tests/test_ignore_regex_text.py

@@ -3,6 +3,7 @@
 import time
 from flask import url_for
 from . util import live_server_setup
+from changedetectionio import html_tools
 
 def test_setup(live_server):
     live_server_setup(live_server)
@@ -23,7 +24,7 @@ def test_strip_regex_text_func():
     ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"]
 
     fetcher = fetch_site_status.perform_site_check(datastore=False)
-    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
+    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
 
     assert b"but 1 lines" in stripped_content
     assert b"igNORe-cAse text" not in stripped_content

changedetectionio/tests/test_ignore_text.py

@@ -3,6 +3,7 @@
 import time
 from flask import url_for
 from . util import live_server_setup
+from changedetectionio import html_tools
 
 def test_setup(live_server):
     live_server_setup(live_server)
@@ -23,7 +24,7 @@ def test_strip_text_func():
     ignore_lines = ["sometimes"]
 
     fetcher = fetch_site_status.perform_site_check(datastore=False)
-    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
+    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
 
     assert b"sometimes" not in stripped_content
     assert b"Some content" in stripped_content
@@ -52,6 +53,8 @@ def set_modified_original_ignore_response():
      <p>Which is across multiple lines</p>
      </br>
      So let's see what happens.  </br>
+     <p>new ignore stuff</p>
+     <p>blah</p>
      </body>
      </html>
 
@@ -67,7 +70,7 @@ def set_modified_ignore_response():
        <body>
      Some initial text</br>
      <p>Which is across multiple lines</p>
-     <P>ZZZZZ</P>
+     <P>ZZZZz</P>
      </br>
      So let's see what happens.  </br>
      </body>
@@ -82,7 +85,8 @@ def set_modified_ignore_response():
 def test_check_ignore_text_functionality(client, live_server):
     sleep_time_for_fetch_thread = 3
 
-    ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
+    # Use a mix of case in ZzZ to prove it works case-insensitive.
+    ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
     set_original_ignore_response()
 
     # Give the endpoint time to spin up
@@ -142,6 +146,103 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
+
+
+
+
+    # Just to be sure.. set a regular modified change..
+    set_modified_original_ignore_response()
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    # Check the preview/highlighter, we should be able to see what we ignored, but it should be highlighted
+    # We only introduce the "modified" content that includes what we ignore so we can prove the newest version also displays
+    # at /preview
+    res = client.get(url_for("preview_page", uuid="first"))
+    # We should be able to see what we ignored
+    assert b'<div class="ignored">new ignore stuff' in res.data
+
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+def test_check_global_ignore_text_functionality(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
+    set_original_ignore_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the settings page, add our ignore text
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "minutes_between_check": 180,
+            "global_ignore_text": ignore_text,
+            'fetch_backend': "html_requests"
+        },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Goto the edit page of the item, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"ignore_text": "something irrelevent but just to check", "url": test_url, 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("settings_page"),
+    )
+    assert bytes(ignore_text.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+    #  Make a change
+    set_modified_ignore_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
     # Just to be sure.. set a regular modified change..
     set_modified_original_ignore_response()
     client.get(url_for("api_watch_checknow"), follow_redirects=True)

changedetectionio/tests/test_ignorestatuscode.py

@@ -0,0 +1,190 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_some_changed_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines, and a new thing too.</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_normal_page_check_works_with_ignore_status_code(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    set_original_response()
+
+    # Goto the settings page, add our ignore text
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "minutes_between_check": 180,
+            "ignore_status_codes": "y",
+            'fetch_backend': "html_requests"
+        },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    set_some_changed_response()
+    time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+    assert b'/test-endpoint' in res.data
+
+
+# Tests the whole stack works with staus codes ignored
+def test_403_page_check_works_with_ignore_status_code(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', status_code=403, _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, check our ignore option
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+    #  Make a change
+    set_some_changed_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should have 'unviewed' still
+    # Because it should be looking at only that 'sametext' id
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+
+# Tests the whole stack works with staus codes ignored
+def test_403_page_check_fails_without_ignore_status_code(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', status_code=403, _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, check our ignore option
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+    #  Make a change
+    set_some_changed_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should have 'unviewed' still
+    # Because it should be looking at only that 'sametext' id
+    res = client.get(url_for("index"))
+    assert b'Status Code 403' in res.data

changedetectionio/tests/test_ignorewhitespace.py

@@ -0,0 +1,96 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+# Should be the same as set_original_ignore_response() but with a little more whitespacing
+def set_original_ignore_response_but_with_whitespace():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>
+
+
+     Which is across multiple lines</p>
+     <br>
+     </br>
+
+         So let's see what happens.  </br>
+
+
+     </body>
+     </html>
+
+    """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+
+# If there was only a change in the whitespacing, then we shouldnt have a change detected
+def test_check_ignore_whitespace(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    set_original_ignore_response()
+
+    # Goto the settings page, add our ignore text
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "minutes_between_check": 180,
+            "ignore_whitespace": "y",
+            'fetch_backend': "html_requests"
+        },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    set_original_ignore_response_but_with_whitespace()
+    time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data

changedetectionio/tests/test_import.py

@@ -0,0 +1,28 @@
+#!/usr/bin/python3
+
+import time
+
+from flask import url_for
+
+from .util import live_server_setup
+
+
+def test_import(client, live_server):
+
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    res = client.post(
+        url_for("import_page"),
+        data={
+            "urls": """https://example.com
+https://example.com tag1
+https://example.com tag1, other tag"""
+        },
+        follow_redirects=True,
+    )
+    assert b"3 Imported" in res.data
+    assert b"tag1" in res.data
+    assert b"other tag" in res.data

changedetectionio/tests/test_jsonpath_selector.py

@@ -1,10 +1,15 @@
 #!/usr/bin/python3
+# coding=utf-8
 
 import time
 from flask import url_for
 from . util import live_server_setup
 import pytest
 
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
 def test_unittest_inline_html_extract():
     # So lets pretend that the JSON we want is inside some HTML
     content="""
@@ -42,6 +47,45 @@ and it can also be repeated
     with pytest.raises(html_tools.JSONNotFound) as e_info:
         html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "$.id")
 
+def set_original_ext_response():
+    data = """
+        [
+        {
+            "isPriceLowered": false,
+            "status": "ForSale",
+            "statusOrig": "for sale"
+        },
+        {
+            "_id": "5e7b3e1fb3262d306323ff1e",
+            "listingsType": "consumer",
+            "status": "ForSale",
+            "statusOrig": "for sale"
+        }
+    ]
+        """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(data)
+
+def set_modified_ext_response():
+    data = """
+    [
+    {
+        "isPriceLowered": false,
+        "status": "Sold",
+        "statusOrig": "sold"
+    },
+    {
+        "_id": "5e7b3e1fb3262d306323ff1e",
+        "listingsType": "consumer",
+        "isPriceLowered": false,
+        "status": "Sold"
+    }
+]
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(data)
 
 def set_original_response():
     test_return_data = """
@@ -60,7 +104,23 @@ def set_original_response():
       ],
       "boss": {
         "name": "Fat guy"
-      }
+      },
+      "available": true
+    }
+    """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+
+def set_response_with_html():
+    test_return_data = """
+    {
+      "test": [
+        {
+          "html": "<b>"
+        }
+      ]
     }
     """
     with open("test-datastore/endpoint-content.txt", "w") as f:
@@ -83,8 +143,9 @@ def set_modified_response():
         }
       ],
       "boss": {
-        "name": "Foobar"
-      }
+        "name": "Örnsköldsvik"
+      },
+      "available": false
     }
         """
 
@@ -93,11 +154,38 @@ def set_modified_response():
 
     return None
 
+def test_check_json_without_filter(client, live_server):
+    # Request a JSON document from a application/json source containing HTML
+    # and be sure it doesn't get chewed up by instriptis
+    set_response_with_html()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
+    client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b'&#34;&lt;b&gt;' in res.data
+    assert res.data.count(b'{\n') >= 2
 
 
 def test_check_json_filter(client, live_server):
-    live_server_setup(live_server)
-
     json_filter = 'json:boss.name'
 
     set_original_response()
@@ -106,7 +194,7 @@ def test_check_json_filter(client, live_server):
     time.sleep(1)
 
     # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
+    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},
@@ -159,5 +247,130 @@ def test_check_json_filter(client, live_server):
 
     # Should not see this, because its not in the JSONPath we entered
     res = client.get(url_for("diff_history_page", uuid="first"))
+
     # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
-    assert b'Foobar' in res.data
+    # And #462 - check we see the proper utf-8 string there
+    assert "Örnsköldsvik".encode('utf-8') in res.data
+
+
+def test_check_json_filter_bool_val(client, live_server):
+    json_filter = "json:$['available']"
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": json_filter,
+              "url": test_url,
+              "tag": "",
+              "headers": "",
+              "fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+    #  Make a change
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    res = client.get(url_for("diff_history_page", uuid="first"))
+    # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
+    assert b'false' in res.data
+
+# Re #265 - Extended JSON selector test
+# Stuff to consider here
+# - Selector should be allowed to return empty when it doesnt match (people might wait for some condition)
+# - The 'diff' tab could show the old and new content
+# - Form should let us enter a selector that doesnt (yet) match anything
+def test_check_json_ext_filter(client, live_server):
+    json_filter = 'json:$[?(@.status==Sold)]'
+
+    set_original_ext_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": json_filter,
+              "url": test_url,
+              "tag": "",
+              "headers": "",
+              "fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(json_filter.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+    #  Make a change
+    set_modified_ext_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(4)
+
+    # It should have 'unviewed'
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    res = client.get(url_for("diff_history_page", uuid="first"))
+
+    # We should never see 'ForSale' because we are selecting on 'Sold' in the rule,
+    # But we should know it triggered ('unviewed' assert above)
+    assert b'ForSale' not in res.data
+    assert b'Sold' in res.data
+

changedetectionio/tests/test_notification.py

@@ -2,26 +2,45 @@ import os
 import time
 import re
 from flask import url_for
-from . util import set_original_response, set_modified_response, live_server_setup
+from . util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup
+import logging
+from changedetectionio.notification import default_notification_body, default_notification_title
+
+def test_setup(live_server):
+    live_server_setup(live_server)
 
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_check_notification(client, live_server):
 
-    live_server_setup(live_server)
     set_original_response()
 
     # Give the endpoint time to spin up
     time.sleep(3)
 
+    # Re 360 - new install should have defaults set
+    res = client.get(url_for("settings_page"))
+    assert default_notification_body.encode() in res.data
+    assert default_notification_title.encode() in res.data
+
+    # When test mode is in BASE_URL env mode, we should see this already configured
+    env_base_url = os.getenv('BASE_URL', '').strip()
+    if len(env_base_url):
+        logging.debug(">>> BASE_URL enabled, looking for %s", env_base_url)
+        res = client.get(url_for("settings_page"))
+        assert bytes(env_base_url.encode('utf-8')) in res.data
+    else:
+        logging.debug(">>> SKIPPING BASE_URL check")
+
+    # re #242 - when you edited an existing new entry, it would not correctly show the notification settings
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
+        url_for("api_watch_add"),
+        data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
-    assert b"1 Imported" in res.data
+    assert b"Watch added" in res.data
 
     # Give the thread time to pick up the first version
     time.sleep(3)
@@ -32,117 +51,95 @@ def test_check_notification(client, live_server):
     notification_url = url.replace('http', 'json')
 
     print (">>>> Notification URL: "+notification_url)
+
+    notification_form_data = {"notification_urls": notification_url,
+                              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
+                              "notification_body": "BASE URL: {base_url}\n"
+                                                   "Watch URL: {watch_url}\n"
+                                                   "Watch UUID: {watch_uuid}\n"
+                                                   "Watch title: {watch_title}\n"
+                                                   "Watch tag: {watch_tag}\n"
+                                                   "Preview: {preview_url}\n"
+                                                   "Diff URL: {diff_url}\n"
+                                                   "Snapshot: {current_snapshot}\n"
+                                                   "Diff: {diff}\n"
+                                                   "Diff Full: {diff_full}\n"
+                                                   ":-)",
+                              "notification_format": "Text"}
+
+    notification_form_data.update({
+        "url": test_url,
+        "tag": "my tag",
+        "title": "my title",
+        "headers": "",
+        "fetch_backend": "html_requests"})
+
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"notification_urls": notification_url,
-              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
-              "notification_body": "BASE URL: {base_url}\n"
-                                   "Watch URL: {watch_url}\n"
-                                   "Watch UUID: {watch_uuid}\n"
-                                   "Watch title: {watch_title}\n"
-                                   "Watch tag: {watch_tag}\n"
-                                   "Preview: {preview_url}\n"
-                                   "Diff URL: {diff_url}\n"
-                                   "Snapshot: {current_snapshot}\n"
-                                   ":-)",
-              "url": test_url,
-              "tag": "my tag",
-              "title": "my title",
-              "headers": "",
-              "fetch_backend": "html_requests",
-              "trigger_check": "y"},
+        data=notification_form_data,
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
-    assert b"Notifications queued" in res.data
+
 
     # Hit the edit page, be sure that we saved it
+    # Re #242 - wasnt saving?
     res = client.get(
         url_for("edit_page", uuid="first"))
     assert bytes(notification_url.encode('utf-8')) in res.data
+    assert bytes("New ChangeDetection.io Notification".encode('utf-8')) in res.data
 
 
-    # Because we hit 'send test notification on save'
+
+    ## Now recheck, and it should have sent the notification
     time.sleep(3)
+    set_modified_response()
 
     notification_submission = None
 
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(3)
     # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
-        # Did we see the URL that had a change, in the notification?
-
-    assert test_url in notification_submission
-
     os.unlink("test-datastore/notification.txt")
 
-    set_modified_response()
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(3)
-
-    # Did the front end see it?
-    res = client.get(
-        url_for("index"))
-
-    assert bytes("just now".encode('utf-8')) in res.data
-
-    notification_submission=None
-    # Verify what was sent as a notification
-    with open("test-datastore/notification.txt", "r") as f:
-        notification_submission = f.read()
-        # Did we see the URL that had a change, in the notification?
-
+    # Did we see the URL that had a change, in the notification?
+    # Diff was correctly executed
     assert test_url in notification_submission
+    assert ':-)' in notification_submission
+    assert "Diff Full: Some initial text" in notification_submission
+    assert "Diff: (changed) Which is across multiple lines" in notification_submission
+    assert "(into   ) which has this one new line" in notification_submission
+    # Re #342 - check for accidental python byte encoding of non-utf8/string
+    assert "b'" not in notification_submission
+    assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE)
+    assert "Watch title: my title" in notification_submission
+    assert "Watch tag: my tag" in notification_submission
+    assert "diff/" in notification_submission
+    assert "preview/" in notification_submission
+    assert ":-)" in notification_submission
+    assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
 
-        # Re #65 - did we see our foobar.com BASE_URL ?
-        #assert bytes("https://foobar.com".encode('utf-8')) in notification_submission
+    if env_base_url:
+        # Re #65 - did we see our BASE_URl ?
+        logging.debug (">>> BASE_URL checking in notification: %s", env_base_url)
+        assert env_base_url in notification_submission
+    else:
+        logging.debug(">>> Skipping BASE_URL check")
 
 
-    ##  Now configure something clever, we go into custom config (non-default) mode, this is returned by the endpoint
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(";jasdhflkjadshf kjhsdfkjl ahslkjf haslkjd hfaklsj hf\njl;asdhfkasj stuff we will detect\n")
 
-    res = client.post(
-        url_for("settings_page"),
-        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
-              "notification_urls": "json://foobar.com", #Re #143 should not see that it sent without [test checkbox]
-              "minutes_between_check": 180,
-              "fetch_backend": "html_requests",
-              },
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
-    # Re #143 - should not see this if we didnt hit the test box
-    assert b"Notifications queued" not in res.data
-
-    # Trigger a check
+    # This should insert the {current_snapshot}
+    set_more_modified_response()
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
     time.sleep(3)
-
-    # Did the front end see it?
-    res = client.get(
-        url_for("index"))
-
-    assert bytes("just now".encode('utf-8')) in res.data
-
+    # Verify what was sent as a notification, this file should exist
     with open("test-datastore/notification.txt", "r") as f:
         notification_submission = f.read()
+    assert "Ohh yeah awesome" in notification_submission
 
-        assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE)
-        assert "Watch title: my title" in notification_submission
-        assert "Watch tag: my tag" in notification_submission
-        assert "diff/" in notification_submission
-        assert "preview/" in notification_submission
-        assert ":-)" in notification_submission
-        assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
-        # This should insert the {current_snapshot}
-        assert "stuff we will detect" in notification_submission
 
     # Prove that "content constantly being marked as Changed with no Updating causes notification" is not a thing
     # https://github.com/dgtlmoon/changedetection.io/discussions/192
@@ -150,24 +147,68 @@ def test_check_notification(client, live_server):
 
     # Trigger a check
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
-    time.sleep(3)
+    time.sleep(1)
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
-    time.sleep(3)
+    time.sleep(1)
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
-    time.sleep(3)
+    time.sleep(1)
     assert os.path.exists("test-datastore/notification.txt") == False
 
+    # cleanup for the next
+    client.get(
+        url_for("api_delete", uuid="first"),
+        follow_redirects=True
+    )
+
+
+def test_notification_validation(client, live_server):
+    #live_server_setup(live_server)
+    time.sleep(3)
+    # re #242 - when you edited an existing new entry, it would not correctly show the notification settings
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("api_watch_add"),
+        data={"url": test_url, "tag": 'nice one'},
+        follow_redirects=True
+    )
+    with open("xxx.bin", "wb") as f:
+        f.write(res.data)
+    assert b"Watch added" in res.data
+
+    # Re #360 some validation
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"notification_urls": 'json://localhost/foobar',
+              "notification_title": "",
+              "notification_body": "",
+              "notification_format": "Text",
+              "url": test_url,
+              "tag": "my tag",
+              "title": "my title",
+              "headers": "",
+              "fetch_backend": "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Notification Body and Title is required when a Notification URL is used" in res.data
 
     # Now adding a wrong token should give us an error
     res = client.post(
         url_for("settings_page"),
         data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
               "notification_body": "Rubbish: {rubbish}\n",
-              "notification_urls": "json://foobar.com",
-              "minutes_between_check": 180,
+              "notification_format": "Text",
+              "notification_urls": "json://localhost/foobar",
+              "time_between_check": {'seconds': 180},
               "fetch_backend": "html_requests"
               },
         follow_redirects=True
     )
 
-    assert bytes("is not a valid token".encode('utf-8')) in res.data
+    assert bytes("is not a valid token".encode('utf-8')) in res.data
+
+    # cleanup for the next
+    client.get(
+        url_for("api_delete", uuid="first"),
+        follow_redirects=True
+    )

changedetectionio/tests/test_notification_errors.py

@@ -0,0 +1,66 @@
+import os
+import time
+import re
+from flask import url_for
+from . util import set_original_response, set_modified_response, live_server_setup
+import logging
+
+def test_check_notification_error_handling(client, live_server):
+
+    live_server_setup(live_server)
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(3)
+
+    # use a different URL so that it doesnt interfere with the actual check until we are ready
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("api_watch_add"),
+        data={"url": "https://changedetection.io/CHANGELOG.txt", "tag": ''},
+        follow_redirects=True
+    )
+    assert b"Watch added" in res.data
+
+    time.sleep(10)
+
+    # Check we capture the failure, we can just use trigger_check = y here
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"notification_urls": "jsons://broken-url.changedetection.io/test",
+              "notification_title": "xxx",
+              "notification_body": "xxxxx",
+              "notification_format": "Text",
+              "url": test_url,
+              "tag": "",
+              "title": "",
+              "headers": "",
+              "minutes_between_check": "180",
+              "fetch_backend": "html_requests",
+              "trigger_check": "y"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    found=False
+    for i in range(1, 10):
+        time.sleep(1)
+        logging.debug("Fetching watch overview....")
+        res = client.get(
+            url_for("index"))
+
+        if bytes("Notification error detected".encode('utf-8')) in res.data:
+            found=True
+            break
+
+
+    assert found
+
+
+    # The error should show in the notification logs
+    res = client.get(
+        url_for("notification_logs"))
+    assert bytes("Name or service not known".encode('utf-8')) in res.data
+
+
+    # And it should be listed on the watch overview

changedetectionio/tests/test_request.py

@@ -0,0 +1,214 @@
+import json
+import time
+from flask import url_for
+from . util import set_original_response, set_modified_response, live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+# Hard to just add more live server URLs when one test is already running (I think)
+# So we add our test here (was in a different file)
+def test_headers_in_request(client, live_server):
+    # Add our URL to the import page
+    test_url = url_for('test_headers', _external=True)
+
+    # Add the test URL twice, we will check
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;'
+
+
+    # Add some headers to a request
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": test_url,
+              "tag": "",
+              "fetch_backend": "html_requests",
+              "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+
+    # Give the thread time to pick up the first version
+    time.sleep(5)
+
+    # The service should echo back the request headers
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Flask will convert the header key to uppercase
+    assert b"Xxx:ooo" in res.data
+    assert b"Cool:yeah" in res.data
+
+    # The test call service will return the headers as the body
+    from html import escape
+    assert escape(cookie_header).encode('utf-8') in res.data
+
+    time.sleep(5)
+
+    # Re #137 -  Examine the JSON index file, it should have only one set of headers entered
+    watches_with_headers = 0
+    with open('test-datastore/url-watches.json') as f:
+        app_struct = json.load(f)
+        for uuid in app_struct['watching']:
+            if (len(app_struct['watching'][uuid]['headers'])):
+                watches_with_headers += 1
+
+    # Should be only one with headers set
+    assert watches_with_headers==1
+
+def test_body_in_request(client, live_server):
+    # Add our URL to the import page
+    test_url = url_for('test_body', _external=True)
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    body_value = 'Test Body Value'
+
+    # Add a properly formatted body with a proper method
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": test_url,
+              "tag": "",
+              "method": "POST",
+              "fetch_backend": "html_requests",
+              "body": body_value},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    time.sleep(3)
+
+    # The service should echo back the body
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # If this gets stuck something is wrong, something should always be there
+    assert b"No history found" not in res.data
+    # We should see what we sent in the reply
+    assert str.encode(body_value) in res.data
+
+    ####### data sanity checks
+    # Add the test URL twice, we will check
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    watches_with_body = 0
+    with open('test-datastore/url-watches.json') as f:
+        app_struct = json.load(f)
+        for uuid in app_struct['watching']:
+            if app_struct['watching'][uuid]['body']==body_value:
+                watches_with_body += 1
+
+    # Should be only one with body set
+    assert watches_with_body==1
+
+    # Attempt to add a body with a GET method
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": test_url,
+              "tag": "",
+              "method": "GET",
+              "fetch_backend": "html_requests",
+              "body": "invalid"},
+        follow_redirects=True
+    )
+    assert b"Body must be empty when Request Method is set to GET" in res.data
+
+
+def test_method_in_request(client, live_server):
+    # Add our URL to the import page
+    test_url = url_for('test_method', _external=True)
+
+    # Add the test URL twice, we will check
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Attempt to add a method which is not valid
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "tag": "",
+            "fetch_backend": "html_requests",
+            "method": "invalid"},
+        follow_redirects=True
+    )
+    assert b"Not a valid choice" in res.data
+
+    # Add a properly formatted body
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "tag": "",
+            "fetch_backend": "html_requests",
+            "method": "PATCH"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Give the thread time to pick up the first version
+    time.sleep(5)
+
+    # The service should echo back the request verb
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # The test call service will return the verb as the body
+    assert b"PATCH" in res.data
+
+    time.sleep(5)
+
+    watches_with_method = 0
+    with open('test-datastore/url-watches.json') as f:
+        app_struct = json.load(f)
+        for uuid in app_struct['watching']:
+            if app_struct['watching'][uuid]['method'] == 'PATCH':
+                watches_with_method += 1
+
+    # Should be only one with method set to PATCH
+    assert watches_with_method == 1
+

changedetectionio/tests/test_security.py

@@ -0,0 +1,36 @@
+from flask import url_for
+from . util import set_original_response, set_modified_response, live_server_setup
+import time
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def test_file_access(client, live_server):
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": 'https://localhost'},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    # Attempt to add a body with a GET method
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": 'file:///etc/passwd',
+              "tag": "",
+              "method": "GET",
+              "fetch_backend": "html_requests",
+              "body": ""},
+        follow_redirects=True
+    )
+    time.sleep(3)
+
+    res = client.get(
+        url_for("index", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b'denied for security reasons' in res.data

changedetectionio/tests/test_trigger.py

@@ -129,3 +129,8 @@ def test_trigger_functionality(client, live_server):
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
+
+    # Check the preview/highlighter, we should be able to see what we triggered on, but it should be highlighted
+    res = client.get(url_for("preview_page", uuid="first"))
+    # We should be able to see what we ignored
+    assert b'<div class="triggered">foobar' in res.data

changedetectionio/tests/test_xpath_selector.py

@@ -0,0 +1,119 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+from ..html_tools import *
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div class="sametext">Some text thats the same</div>
+     <div class="changetext">Some text that will change</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  THIS CHANGES AND SHOULDNT TRIGGER A CHANGE</br>
+     <div class="sametext">Some text thats the same</div>
+     <div class="changetext">Some new text</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+def test_check_markup_xpath_filter_restriction(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    xpath_filter = "//*[contains(@class, 'sametext')]"
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # view it/reset state back to viewed
+    client.get(url_for("diff_history_page", uuid="first"), follow_redirects=True)
+
+    #  Make a change
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+
+def test_xpath_validation(client, live_server):
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"is not a valid XPath expression" in res.data

changedetectionio/tests/unit/__init__.py

@@ -0,0 +1 @@
+"""Unit tests for the app."""

changedetectionio/tests/unit/test-content/README.md

@@ -0,0 +1,5 @@
+# What is this?
+This is test content for the python diff engine, we use the JS interface for the front end, because you can explore 
+differences in words etc, but we use (at the moment) the python difflib engine.
+
+This content `before.txt` and `after.txt` is for unit testing

changedetectionio/tests/unit/test-content/after-2.txt

@@ -0,0 +1,3 @@
+After twenty years, as cursed as I may be
+ok
+and insure that I'm one of those computer nerds.

changedetectionio/tests/unit/test-content/after.txt

@@ -0,0 +1,7 @@
+After twenty years, as cursed as I may be
+for having learned computerese,
+I continue to examine bits, bytes and words
+xok
+next-x-ok
+and insure that I'm one of those computer nerds.
+and something new

changedetectionio/tests/unit/test-content/before.txt

@@ -0,0 +1,5 @@
+After twenty years, as cursed as I may be
+for having learned computerese,
+I continue to examine bits, bytes and words
+ok
+and insure that I'm one of those computer nerds.

changedetectionio/tests/unit/test_notification_diff.py

@@ -0,0 +1,32 @@
+#!/usr/bin/python3
+
+# run from dir above changedetectionio/ dir
+# python3 -m unittest changedetectionio.tests.unit.test_notification_diff
+
+import unittest
+import os
+
+from changedetectionio import diff
+
+# mostly
+class TestDiffBuilder(unittest.TestCase):
+
+    def test_expected_diff_output(self):
+        base_dir = os.path.dirname(__file__)
+        output = diff.render_diff(previous_file=base_dir + "/test-content/before.txt", newest_file=base_dir + "/test-content/after.txt")
+        output = output.split("\n")
+        self.assertIn('(changed) ok', output)
+        self.assertIn('(into   ) xok', output)
+        self.assertIn('(into   ) next-x-ok', output)
+        self.assertIn('(added  ) and something new', output)
+
+
+        output = diff.render_diff(previous_file=base_dir + "/test-content/before.txt", newest_file=base_dir + "/test-content/after-2.txt")
+        output = output.split("\n")
+        self.assertIn('(removed) for having learned computerese,', output)
+        self.assertIn('(removed) I continue to examine bits, bytes and words', output)
+
+        # @todo test blocks of changed, blocks of added, blocks of removed
+
+if __name__ == '__main__':
+    unittest.main()

changedetectionio/tests/util.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python3
 
+from flask import make_response, request
 
 def set_original_response():
     test_return_data = """<html>
@@ -34,21 +35,45 @@ def set_modified_response():
 
     return None
 
+def set_more_modified_response():
+    test_return_data = """<html>
+    <head><title>modified head title</title></head>
+    <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     Ohh yeah awesome<br/>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
 
 def live_server_setup(live_server):
 
-
     @live_server.app.route('/test-endpoint')
     def test_endpoint():
-        # Tried using a global var here but didn't seem to work, so reading from a file instead.
-        with open("test-datastore/endpoint-content.txt", "r") as f:
-            return f.read()
+        ctype = request.args.get('content_type')
+        status_code = request.args.get('status_code')
+
+        try:
+            # Tried using a global var here but didn't seem to work, so reading from a file instead.
+            with open("test-datastore/endpoint-content.txt", "r") as f:
+                resp = make_response(f.read(), status_code)
+                resp.headers['Content-Type'] = ctype if ctype else 'text/html'
+                return resp
+        except FileNotFoundError:
+            return make_response('', status_code)
 
     # Just return the headers in the request
     @live_server.app.route('/test-headers')
     def test_headers():
 
-        from flask import request
         output= []
 
         for header in request.headers:
@@ -56,19 +81,34 @@ def live_server_setup(live_server):
 
         return "\n".join(output)
 
+    # Just return the body in the request
+    @live_server.app.route('/test-body', methods=['POST', 'GET'])
+    def test_body():
+        return request.data
+
+    # Just return the verb in the request
+    @live_server.app.route('/test-method', methods=['POST', 'GET', 'PATCH'])
+    def test_method():
+        return request.method
 
     # Where we POST to as a notification
     @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET'])
     def test_notification_endpoint():
-        from flask import request
-
         with open("test-datastore/notification.txt", "wb") as f:
             # Debug method, dump all POST to file also, used to prove #65
             data = request.stream.read()
             if data != None:
                 f.write(data)
 
-        print("\n>> Test notification endpoint was hit.\n")
+        print("\n>> Test notification endpoint was hit.\n", data)
         return "Text was set"
 
+
+    # Just return the verb in the request
+    @live_server.app.route('/test-basicauth', methods=['GET'])
+    def test_basicauth_method():
+        auth = request.authorization
+        ret = " ".join([auth.username, auth.password, auth.type])
+        return ret
+
     live_server.start()

changedetectionio/update_worker.py

@@ -2,7 +2,12 @@ import threading
 import queue
 import time
 
-# Requests for checking on the site use a pool of thread Workers managed by a Queue.
+# A single update worker
+#
+# Requests for checking on a single site(watch) from a queue of watches
+# (another process inserts watches into the queue that are time-ready for checking)
+
+
 class update_worker(threading.Thread):
     current_uuid = None
 
@@ -33,53 +38,61 @@ class update_worker(threading.Thread):
 
                     changed_detected = False
                     contents = ""
+                    screenshot = False
                     update_obj= {}
+                    now = time.time()
 
                     try:
-                        now = time.time()
-                        changed_detected, update_obj, contents = update_handler.run(uuid)
-
-                        # Always record that we atleast tried
-                        self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3)})
+                        changed_detected, update_obj, contents, screenshot = update_handler.run(uuid)
 
+                        # Re #342
+                        # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
+                        # We then convert/.decode('utf-8') for the notification etc
+                        if not isinstance(contents, (bytes, bytearray)):
+                            raise Exception("Error - returned data from the fetch handler SHOULD be bytes")
                     except PermissionError as e:
                         self.app.logger.error("File permission error updating", uuid, str(e))
                     except content_fetcher.EmptyReply as e:
-                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error':str(e)})
-
+                        # Some kind of custom to-str handler in the exception handler that does this?
+                        err_text = "EmptyReply: Status Code {}".format(e.status_code)
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                                                                           'last_check_status': e.status_code})
                     except Exception as e:
-                        self.app.logger.error("Exception reached processing watch UUID:%s - %s", uuid, str(e))
+                        self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
                         self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
 
                     else:
-                        if update_obj:
-                            try:
-                                self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
-                                if changed_detected:
+                        try:
+                            watch = self.datastore.data['watching'][uuid]
+                            fname = "" # Saved history text filename
 
-                                    # A change was detected
-                                    newest_version_file_contents = ""
-                                    fname = self.datastore.save_history_text(watch_uuid=uuid, contents=contents)
+                            # For the FIRST time we check a site, or a change detected, save the snapshot.
+                            if changed_detected or not watch['last_checked']:
+                                # A change was detected
+                                fname = self.datastore.save_history_text(watch_uuid=uuid, contents=contents)
+                                # Should always be keyed by string(timestamp)
+                                self.datastore.update_watch(uuid, {"history": {str(round(time.time())): fname}})
 
-                                    # Update history with the stripped text for future reference, this will also mean we save the first
-                                    # Should always be keyed by string(timestamp)
-                                    self.datastore.update_watch(uuid, {"history": {str(update_obj["last_checked"]): fname}})
+                            # Generally update anything interesting returned
+                            self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
 
-                                    watch = self.datastore.data['watching'][uuid]
+                            # A change was detected
+                            if changed_detected:
+                                n_object = {}
+                                print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))
 
-                                    print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))
+                                # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
+                                if len(watch['history']) > 1:
 
-                                    # Get the newest snapshot data to be possibily used in a notification
-                                    newest_key = self.datastore.get_newest_history_key(uuid)
-                                    if newest_key:
-                                        with open(watch['history'][newest_key], 'r') as f:
-                                            newest_version_file_contents = f.read().strip()
+                                    dates = list(watch['history'].keys())
+                                    # Convert to int, sort and back to str again
+                                    # @todo replace datastore getter that does this automatically
+                                    dates = [int(i) for i in dates]
+                                    dates.sort(reverse=True)
+                                    dates = [str(i) for i in dates]
+
+                                    prev_fname = watch['history'][dates[1]]
 
-                                    n_object = {
-                                        'watch_url': watch['url'],
-                                        'uuid': uuid,
-                                        'current_snapshot': newest_version_file_contents
-                                    }
 
                                     # Did it have any notification alerts to hit?
                                     if len(watch['notification_urls']):
@@ -87,7 +100,7 @@ class update_worker(threading.Thread):
                                         n_object['notification_urls'] = watch['notification_urls']
                                         n_object['notification_title'] = watch['notification_title']
                                         n_object['notification_body'] = watch['notification_body']
-                                        self.notification_q.put(n_object)
+                                        n_object['notification_format'] = watch['notification_format']
 
                                     # No? maybe theres a global setting, queue them all
                                     elif len(self.datastore.data['settings']['application']['notification_urls']):
@@ -95,14 +108,47 @@ class update_worker(threading.Thread):
                                         n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
                                         n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title']
                                         n_object['notification_body'] = self.datastore.data['settings']['application']['notification_body']
-                                        self.notification_q.put(n_object)
+                                        n_object['notification_format'] = self.datastore.data['settings']['application']['notification_format']
                                     else:
                                         print(">>> NO notifications queued, watch and global notification URLs were empty.")
 
-                            except Exception as e:
-                                print("!!!! Exception in update_worker !!!\n", e)
+                                    # Only prepare to notify if the rules above matched
+                                    if 'notification_urls' in n_object:
+                                        # HTML needs linebreak, but MarkDown and Text can use a linefeed
+                                        if n_object['notification_format'] == 'HTML':
+                                            line_feed_sep = "</br>"
+                                        else:
+                                            line_feed_sep = "\n"
+
+                                        from changedetectionio import diff
+                                        n_object.update({
+                                            'watch_url': watch['url'],
+                                            'uuid': uuid,
+                                            'current_snapshot': contents.decode('utf-8'),
+                                            'diff': diff.render_diff(prev_fname, fname, line_feed_sep=line_feed_sep),
+                                            'diff_full': diff.render_diff(prev_fname, fname, True, line_feed_sep=line_feed_sep)
+                                        })
+
+                                        self.notification_q.put(n_object)
+
+                        except Exception as e:
+                            # Catch everything possible here, so that if a worker crashes, we don't lose it until restart!
+                            print("!!!! Exception in update_worker !!!\n", e)
+                            self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
+                            self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
+
+                    finally:
+                        # Always record that we atleast tried
+                        self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
+                                                                           'last_checked': round(time.time())})
+                        # Always save the screenshot if it's available
+                        if screenshot:
+                            self.datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
 
                 self.current_uuid = None  # Done
                 self.q.task_done()
 
+                # Give the CPU time to interrupt
+                time.sleep(0.1)
+
             self.app.config.exit.wait(1)

docker-compose.yml

@@ -1,9 +1,9 @@
 version: '2'
 services:
-    changedetection.io:
-      image: dgtlmoon/changedetection.io
-      container_name: changedetection.io
-      hostname: changedetection.io
+    changedetection:
+      image: ghcr.io/dgtlmoon/changedetection.io
+      container_name: changedetection
+      hostname: changedetection
       volumes:
         - changedetection-data:/datastore
 
@@ -13,13 +13,23 @@ services:
 
   #      - PUID=1000
   #      - PGID=1000
-  #       # Alternative WebDriver/selenium URL, do not use "'s or 's!
+  #
+  #       Alternative WebDriver/selenium URL, do not use "'s or 's!
   #      - WEBDRIVER_URL=http://browser-chrome:4444/wd/hub
-  #        Proxy support example.
+  #
+  #       WebDriver proxy settings webdriver_proxyType, webdriver_ftpProxy, webdriver_httpProxy, webdriver_noProxy,
+  #                                webdriver_proxyAutoconfigUrl, webdriver_sslProxy, webdriver_autodetect,
+  #                                webdriver_socksProxy, webdriver_socksUsername, webdriver_socksVersion, webdriver_socksPassword
+  #
+  #             https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.proxy
+  #
+  #        Plain requsts - proxy support example.
   #      - HTTP_PROXY=socks5h://10.10.1.10:1080
   #      - HTTPS_PROXY=socks5h://10.10.1.10:1080
+  #
   #        An exclude list (useful for notification URLs above) can be specified by with
   #      - NO_PROXY="localhost,192.168.0.0/24"
+  #
   #        Base URL of your changedetection.io install (Added to the notification alert)
   #      - BASE_URL=https://mysite.com
 
@@ -33,13 +43,17 @@ services:
       restart: unless-stopped
 
      # Used for fetching pages via WebDriver+Chrome where you need Javascript support.
-     # Does not work on rPi, https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver
+     # Now working on arm64 (needs testing on rPi - tested on Oracle ARM instance)
+     # replace image with seleniarm/standalone-chromium:4.0.0-20211213
 
 #    browser-chrome:
 #        hostname: browser-chrome
 #        image: selenium/standalone-chrome-debug:3.141.59
 #        environment:
 #            - VNC_NO_PASSWORD=1
+#            - SCREEN_WIDTH=1920
+#            - SCREEN_HEIGHT=1080
+#            - SCREEN_DEPTH=24
 #        volumes:
 #            # Workaround to avoid the browser crashing inside a docker container
 #            # See https://github.com/SeleniumHQ/docker-selenium#quick-start

heroku.yml

@@ -1,5 +1,2 @@
-build:
-  docker:
-    changedetection: Dockerfile
 run:
-  changedetection: python ./changedetection.py -d /datastore -p $PORT
+  changedetection: python3 ./changedetection.py -C -d ./datastore -p $PORT

requirements.txt

@@ -1,22 +1,42 @@
-chardet==2.3.0
-flask~= 1.0
-
+flask~= 2.0
+flask_wtf
 eventlet>=0.31.0
-requests[socks] ~= 2.15
 validators
 timeago ~=1.0
-inscriptis ~= 1.2
+inscriptis ~= 2.2
 feedgen ~= 0.9
 flask-login ~= 0.5
 pytz
-urllib3
+
+# Set these versions together to avoid a RequestsDependencyWarning
+requests[socks] ~= 2.26
+urllib3 > 1.26
+chardet > 2.3.0
+
 wtforms ~= 2.3.3
 jsonpath-ng ~= 1.5.3
 
 # Notification library
-apprise ~= 0.9
+apprise ~= 0.9.7
 
-# Used for CSS filtering, replace with soupsieve and lxml for xpath
+# apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
+paho-mqtt
+
+# Pinned version of cryptography otherwise
+# ERROR: Could not build wheels for cryptography which use PEP 517 and cannot be installed directly
+cryptography ~= 3.4
+
+# Used for CSS filtering
 bs4
 
-selenium ~= 3.141
+# XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
+lxml
+
+# 3.141 was missing socksVersion, 3.150 was not in pypi, so we try 4.1.0
+selenium ~= 4.1.0
+
+# https://stackoverflow.com/questions/71652965/importerror-cannot-import-name-safe-str-cmp-from-werkzeug-security/71653849#71653849
+# ImportError: cannot import name 'safe_str_cmp' from 'werkzeug.security'
+# need to revisit flask login versions
+werkzeug ~= 2.0.0
+

runtime.txt

@@ -0,0 +1 @@
+python-3.8.12

setup.py

@@ -32,11 +32,11 @@ setup(
     long_description_content_type='text/markdown',
     keywords='website change monitor for changes notification change detection '
              'alerts tracking website tracker change alert website and monitoring',
-    zip_safe=False,
-    entry_points={"console_scripts": ["changedetection.io=changedetection:main"]},
+    entry_points={"console_scripts": ["changedetection.io=changedetectionio.changedetection:main"]},
+    zip_safe=True,
+    scripts=["changedetection.py"],
     author='dgtlmoon',
     url='https://changedetection.io',
-    scripts=['changedetection.py'],
     packages=['changedetectionio'],
     include_package_data=True,
     install_requires=install_requires,