0.39.7

HTTP Basic Auth form validation

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,41 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Version**
+In the top right area: 0....
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS] 
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,23 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+**Version and OS**
+For example, 0.123 on linux/docker
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe the use-case and give concrete real-world examples**
+Attach any HTML/JSON, give links to sites, screenshots etc, we are not mind readers
+
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/codeql-analysis.yml

@@ -12,11 +12,6 @@
 name: "CodeQL"
 
 on:
-  push:
-    branches: [ master ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ master ]
   schedule:
     - cron: '27 9 * * 4'
 

.github/workflows/containers.yml

@@ -0,0 +1,127 @@
+name: Build and push containers
+
+on:
+  # Automatically triggered by a testing workflow passing, but this is only checked when it lands in the `master`/default branch
+  workflow_run:
+    workflows: ["ChangeDetection.io Test"]
+    branches: [master]
+    tags: ['0.*']
+    types: [completed]
+
+  # Or a new tagged release
+  release:
+    types: [published, edited]
+
+jobs:
+  metadata:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Show metadata
+      run: |
+        echo SHA ${{ github.sha }}
+        echo github.ref:  ${{ github.ref }}
+        echo github_ref: $GITHUB_REF
+        echo Event name: ${{ github.event_name }}
+        echo Ref ${{ github.ref }}
+        echo c: ${{ github.event.workflow_run.conclusion }}
+        echo r: ${{ github.event.workflow_run }}
+        echo tname: "${{ github.event.release.tag_name }}"
+        echo headbranch: -${{ github.event.workflow_run.head_branch }}-
+        set
+
+  build-push-containers:
+    runs-on: ubuntu-latest
+    # If the testing workflow has a success, then we build to :latest
+    # Or if we are in a tagged release scenario.
+    if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+      - name: Create release metadata
+        run: |
+          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > changedetectionio/source.txt
+          echo ${{ github.ref }} > changedetectionio/tag.txt
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: all
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to Docker Hub Container Registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+          version: latest
+          driver-opts: image=moby/buildkit:master
+
+      # master always builds :latest
+      - name: Build and push :latest
+        id: docker_build
+        if: ${{ github.ref }} == "refs/heads/master"
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
+            ghcr.io/${{ github.repository }}:latest
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+
+      # A new tagged release is required, which builds :tag
+      - name: Build and push :tag
+        id: docker_build_tag_release
+        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
+            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+
+      - name: Image digest
+        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+

.github/workflows/image.yml

@@ -1,77 +0,0 @@
-name: Test, build and push to Docker Hub
-
-on:
-  push:
-    branches: [ master ]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install flake8 pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-      - name: Test with pytest
-        run: |
-          cd backend; pytest
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          image: tonistiigi/binfmt:latest
-          platforms: all
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-          version: latest
-          driver-opts: image=moby/buildkit:master
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./
-          file: ./Dockerfile
-          push: true
-          tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
-          #                ${{ secrets.DOCKER_HUB_USERNAME }}:/changedetection.io:${{ env.RELEASE_VERSION }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
-
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
-
-      - name: Cache Docker layers
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-
-

.github/workflows/pypi.yml

@@ -0,0 +1,44 @@
+name: PyPi Test and Push tagged release
+
+# Triggers the workflow on push or pull request events
+on:
+  workflow_run:
+    workflows: ["ChangeDetection.io Test"]
+    tags: '*.*'
+    types: [completed]
+
+
+jobs:
+  test-build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+#      - name: Install dependencies
+#        run: |
+#          python -m pip install --upgrade pip
+#          pip install flake8 pytest
+#          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+#          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+      - name: Test that pip builds without error
+        run: |
+          pip3 --version
+          python3 -m pip install wheel
+          python3 setup.py bdist_wheel
+          python3 -m pip install dist/changedetection.io-*-none-any.whl --force
+          changedetection.io -d /tmp -p 10000 &
+          sleep 3
+          curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+          killall -9 changedetection.io
+
+      # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
+      # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
+
+      # https://github.com/docker/buildx/issues/495#issuecomment-918925854
+#if: ${{ github.event_name == 'release'}}

.github/workflows/test-only.yml

@@ -1,13 +1,10 @@
-name: Test only
+name: ChangeDetection.io Test
 
-on:
-  push:
-    branches:
-      - /refs/heads/*
-      - !master
+# Triggers the workflow on push or pull request events
+on: [push, pull_request]
 
 jobs:
-  build:
+  test-build:
     runs-on: ubuntu-latest
     steps:
 
@@ -17,19 +14,32 @@ jobs:
         with:
           python-version: 3.9
 
+      - name: Show env vars
+        run: set
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or undefined names
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Unit tests
+        run: |
+          python3 -m unittest changedetectionio.tests.unit.test_notification_diff
+
       - name: Test with pytest
         run: |
-          cd backend; pytest
+          # Each test is totally isolated and performs its own cleanup/reset
+          cd changedetectionio; ./run_all_tests.sh
 
+      # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
+      # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
+
+      # https://github.com/docker/buildx/issues/495#issuecomment-918925854

.gitignore

@@ -5,3 +5,6 @@ datastore/url-watches.json
 datastore/*
 __pycache__
 .pytest_cache
+build
+dist
+.vscode/settings.json

CONTRIBUTING.md

@@ -0,0 +1,5 @@
+Contributing is always welcome!
+
+I am no professional flask developer, if you know a better way that something can be done, please let me know!
+
+Otherwise, it's always best to PR into the `dev` branch.

Dockerfile

@@ -1,30 +1,61 @@
+# pip dependencies install stage
+FROM python:3.8-slim as builder
+
+# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
+ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl-dev \
+    libffi-dev \
+    gcc \
+    libc-dev \
+    libxslt-dev \
+    zlib1g-dev \
+    g++
+
+RUN mkdir /install
+WORKDIR /install
+
+COPY requirements.txt /requirements.txt
+
+RUN pip install --target=/dependencies -r /requirements.txt
+
+# Final image stage
 FROM python:3.8-slim
-COPY requirements.txt /tmp/requirements.txt
-RUN apt-get update && apt-get install -y gcc libc-dev libxslt-dev zlib1g-dev g++ --no-install-recommends && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
 
-RUN pip3 install --no-cache-dir -r /tmp/requirements.txt
+# Actual packages needed at runtime, usually due to the notification (apprise) backend
+# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
+ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
+# Re #93, #73, excluding rustc (adds another 430Mb~)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl-dev \
+    libffi-dev \
+    gcc \
+    libc-dev \
+    libxslt-dev \
+    zlib1g-dev \
+    g++
+
+# https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
+ENV PYTHONUNBUFFERED=1
 
-RUN [ ! -d "/app" ] && mkdir /app
 RUN [ ! -d "/datastore" ] && mkdir /datastore
 
-# The actual flask app
-COPY backend /app/backend
+# Re #80, sets SECLEVEL=1 in openssl.conf to allow monitoring sites with weak/old cipher suites
+RUN sed -i 's/^CipherString = .*/CipherString = DEFAULT@SECLEVEL=1/' /etc/ssl/openssl.cnf
 
+# Copy modules over to the final image and add their dir to PYTHONPATH
+COPY --from=builder /dependencies /usr/local
+ENV PYTHONPATH=/usr/local
+
+EXPOSE 5000
+
+# The actual flask app
+COPY changedetectionio /app/changedetectionio
 # The eventlet server wrapper
 COPY changedetection.py /app/changedetection.py
 
 WORKDIR /app
 
-# https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
-ENV PYTHONUNBUFFERED=1
-
-# Attempt to store the triggered commit
-ARG SOURCE_COMMIT
-ARG SOURCE_BRANCH
-RUN echo "commit: $SOURCE_COMMIT branch: $SOURCE_BRANCH" >/source.txt
-
 CMD [ "python", "./changedetection.py" , "-d", "/datastore"]
-
-
-

MANIFEST.in

@@ -0,0 +1,6 @@
+recursive-include changedetectionio/templates *
+recursive-include changedetectionio/static *
+include changedetection.py
+global-exclude *.pyc
+global-exclude *node_modules*
+global-exclude venv

Procfile

@@ -0,0 +1 @@
+web: python3 ./changedetection.py -C -d ./datastore -p $PORT

README-pip.md

@@ -0,0 +1,71 @@
+#  changedetection.io
+![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
+<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
+  <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
+</a>
+<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
+  <img src="https://img.shields.io/github/v/release/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
+</a>
+
+## Self-hosted open source change monitoring of web pages.
+
+_Know when web pages change! Stay ontop of new information!_ 
+
+Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
+
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
+
+#### Example use cases
+
+Know when ...
+
+- Government department updates (changes are often only on their websites)
+- Local government news (changes are often only on their websites)
+- New software releases, security advisories when you're not on their mailing list.
+- Festivals with changes
+- Realestate listing changes
+- COVID related news from government websites
+- Detect and monitor changes in JSON API responses 
+- API monitoring and alerting
+
+**Get monitoring now!**
+
+```bash
+$ pip3 install changedetection.io   
+```
+
+Specify a target for the *datastore path* with `-d` (required) and a *listening port* with `-p` (defaults to `5000`)
+
+```bash
+$ changedetection.io -d /path/to/empty/data/dir -p 5000
+```
+
+
+Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
+
+### Features
+- Website monitoring
+- Change detection of content and analyses
+- Filters on change (Select by CSS or JSON)
+- Triggers (Wait for text, wait for regex)
+- Notification support
+- JSON API Monitoring
+- Parse JSON embedded in HTML
+- (Reverse) Proxy support
+- Javascript support via WebDriver
+- RaspberriPi (arm v6/v7/64 support)
+
+See https://github.com/dgtlmoon/changedetection.io for more information.
+
+
+
+### Support us
+
+Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
+
+Please support us, even small amounts help a LOT.
+
+BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />

README.md

@@ -1,56 +1,174 @@
 #  changedetection.io
-![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/image.yml/badge.svg?branch=master)
-<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
-  <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
-</a>
-<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
-  <img src="https://img.shields.io/github/v/release/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
-</a>
+[![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)
 
-## Self-hosted change monitoring of web pages.
+![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
+
+## Self-Hosted, Open Source, Change Monitoring of Web Pages
 
 _Know when web pages change! Stay ontop of new information!_ 
 
 Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
 
+Open source web page monitoring, notification and change detection.
+
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
 
+[![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://dashboard.heroku.com/new?template=https%3A%2F%2Fgithub.com%2Fdgtlmoon%2Fchangedetection.io%2Ftree%2Fmaster)
+
+Read the [Heroku notes and limitations wiki page first](https://github.com/dgtlmoon/changedetection.io/wiki/Heroku-notes)
+
 #### Example use cases
 
-Know when ...
-
-- Government department updates (changes are often only on their websites)
-- Local government news (changes are often only on their websites)
+- Products and services have a change in pricing
+- Governmental department updates (changes are often only on their websites)
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
 - Realestate listing changes
+- COVID related news from government websites
+- University/organisation news from their website
+- Detect and monitor changes in JSON API responses 
+- API monitoring and alerting
+- Changes in legal and other documents
+- Trigger API calls via notifications when text appears on a website
+- Glue together APIs using the JSON filter and JSON notifications
+- Create RSS feeds based on changes in web content
+- You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
+ 
+_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver!</a>_
+
+**Get monitoring now! super simple.**
+
+<a href="https://dashboard.heroku.com/new?template=https%3A%2F%2Fgithub.com%2Fdgtlmoon%2Fchangedetection.io%2Ftree%2Fmaster">Deploy to Heroku for free</a>, Run this python directly, or with <a href="https://docs.docker.com/get-docker/">docker</a> and/or <a href="https://www.digitalocean.com/community/tutorial_collections/how-to-install-docker-compose">docker-compose</a>
+
+## Screenshots
+
+Examining differences in content.
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
+
+Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
 
-**Get monitoring now! super simple, one command!**
+## Installation
+
+### Docker
+
+With Docker composer, just clone this repository and..
+```bash
+$ docker-compose up -d
+```
+Docker standalone
+```bash
+$ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
+```
+
+### Python Pip
+
+Check out our pypi page https://pypi.org/project/changedetection.io/
 
 ```bash
-docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
-```  
+$ pip3 install changedetection.io
+$ changedetection.io -d /path/to/empty/data/dir -p 5000
+```
 
-Now visit http://127.0.0.1:5000 , You should now be able to access the UI.
+Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
 
-#### Updating to latest version
+_Now with per-site configurable support for using a fast built in HTTP fetcher or use a Chrome based fetcher for monitoring of JavaScript websites!_
 
-Highly recommended :)
+## Updating changedetection.io
 
-```bash
+### Docker
+```
 docker pull dgtlmoon/changedetection.io
 docker kill $(docker ps -a|grep changedetection.io|awk '{print $1}')
 docker rm $(docker ps -a|grep changedetection.io|awk '{print $1}')
 docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```
-  
-### Screenshots
 
-Examining differences in content.
+### docker-compose
+
+```bash
+docker-compose pull && docker-compose up -d
+```
+### Filters
+XPath, JSONPath and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools.
+
+### Notifications
+
+ChangeDetection.io supports a massive amount of notifications (including email, office365, custom APIs, etc) when a web-page has a change detected thanks to the <a href="https://github.com/caronc/apprise">apprise</a> library.
+Simply set one or more notification URL's in the _[edit]_ tab of that watch.
+
+Just some examples
+
+    discord://webhook_id/webhook_token
+    flock://app_token/g:channel_id
+    gitter://token/room
+    gchat://workspace/key/token
+    msteams://TokenA/TokenB/TokenC/
+    o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+    rocket://user:password@hostname/#Channel
+    mailto://user:pass@example.com?to=receivingAddress@example.com
+    json://someserver.com/custom-api
+    syslog://
+ 
+<a href="https://github.com/caronc/apprise#popular-notification-services">And everything else in this list!</a>
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
+
+Now you can also customise your notification content!
+
+### JSON API Monitoring
+
+Detect changes and monitor data in JSON API's by using the built-in JSONPath selectors as a filter / selector.
+
+![image](https://user-images.githubusercontent.com/275001/125165842-0ce01980-e1dc-11eb-9e73-d8137dd162dc.png)
+
+This will re-parse the JSON and apply formatting to the text, making it super easy to monitor and detect changes in JSON API results
+
+![image](https://user-images.githubusercontent.com/275001/125165995-d9ea5580-e1dc-11eb-8030-f0deced2661a.png)
+
+#### Parse JSON embedded in HTML!
+
+When you enable a `json:` filter, you can even automatically extract and parse embedded JSON inside a HTML page! Amazingly handy for sites that build content based on JSON, such as many e-commerce websites. 
+
+```
+<html>
+...
+<script type="application/ld+json">
+  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","price": 23.50 }
+</script>
+```  
+
+`json:$.price` would give `23.50`, or you can extract the whole structure
+
+### Proxy configuration
+
+See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration
+
+### Raspberry Pi support?
+
+Raspberry Pi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! 
+
+### Windows native support?
+
+Sorry not yet :( https://github.com/dgtlmoon/changedetection.io/labels/windows
+
+### Support us
+
+Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
+
+Please support us, even small amounts help a LOT.
+
+BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />
 
 
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
+[release-shield]: https://img.shields.io/github/v/release/dgtlmoon/changedetection.io?style=for-the-badge
+[docker-pulls]: https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io?style=for-the-badge
+[test-shield]: https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master
 
-Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
+[license-shield]: https://img.shields.io/github/license/dgtlmoon/changedetection.io.svg?style=for-the-badge
+[release-link]: https://github.com/dgtlmoon.com/changedetection.io/releases
+[docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io

app.json

@@ -0,0 +1,21 @@
+{
+  "name": "ChangeDetection.io",
+  "description": "The best and simplest self-hosted open source website change detection monitoring and notification service.",
+  "keywords": [
+    "changedetection",
+    "website monitoring"
+  ],
+  "repository": "https://github.com/dgtlmoon/changedetection.io",
+  "success_url": "/",
+  "scripts": {
+  },
+  "env": {
+  },
+  "formation": {
+    "web": {
+      "quantity": 1,
+      "size": "free"
+    }
+  },
+  "image": "heroku/python"
+}

backend/README-pytest.md

@@ -1 +0,0 @@
-Note: run `pytest` from this directory.

backend/__init__.py

@@ -1,790 +0,0 @@
-#!/usr/bin/python3
-
-
-# @todo logging
-# @todo extra options for url like , verify=False etc.
-# @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option?
-# @todo option for interval day/6 hour/etc
-# @todo on change detected, config for calling some API
-# @todo make tables responsive!
-# @todo fetch title into json
-# https://distill.io/features
-# proxy per check
-#  - flask_cors, itsdangerous,MarkupSafe
-
-import time
-import os
-import timeago
-import flask_login
-from flask_login import login_required
-
-import threading
-from threading import Event
-
-import queue
-
-from flask import Flask, render_template, request, send_from_directory, abort, redirect, url_for
-
-from feedgen.feed import FeedGenerator
-from flask import make_response
-import datetime
-import pytz
-
-datastore = None
-
-# Local
-running_update_threads = []
-ticker_thread = None
-
-messages = []
-extra_stylesheets = []
-
-update_q = queue.Queue()
-
-app = Flask(__name__, static_url_path="/var/www/change-detection/backen/static")
-
-# Stop browser caching of assets
-app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
-
-app.config.exit = Event()
-
-app.config['NEW_VERSION_AVAILABLE'] = False
-
-app.config['LOGIN_DISABLED'] = False
-
-# Disables caching of the templates
-app.config['TEMPLATES_AUTO_RELOAD'] = True
-
-
-# We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
-# running or something similar.
-@app.template_filter('format_last_checked_time')
-def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
-    # Worker thread tells us which UUID it is currently processing.
-    for t in running_update_threads:
-        if t.current_uuid == watch_obj['uuid']:
-            return "Checking now.."
-
-    if watch_obj['last_checked'] == 0:
-        return 'Not yet'
-
-    return timeago.format(int(watch_obj['last_checked']), time.time())
-
-
-# @app.context_processor
-# def timeago():
-#    def _timeago(lower_time, now):
-#        return timeago.format(lower_time, now)
-#    return dict(timeago=_timeago)
-
-@app.template_filter('format_timestamp_timeago')
-def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
-    return timeago.format(timestamp, time.time())
-    # return timeago.format(timestamp, time.time())
-    # return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
-
-
-class User(flask_login.UserMixin):
-    id=None
-
-    def set_password(self, password):
-        return True
-    def get_user(self, email="defaultuser@changedetection.io"):
-        return self
-    def is_authenticated(self):
-        return True
-    def is_active(self):
-        return True
-    def is_anonymous(self):
-        return False
-    def get_id(self):
-        return str(self.id)
-
-    def check_password(self, password):
-
-        import hashlib
-        import base64
-
-        # Getting the values back out
-        raw_salt_pass = base64.b64decode(datastore.data['settings']['application']['password'])
-        salt_from_storage = raw_salt_pass[:32]  # 32 is the length of the salt
-
-        # Use the exact same setup you used to generate the key, but this time put in the password to check
-        new_key = hashlib.pbkdf2_hmac(
-            'sha256',
-            password.encode('utf-8'),  # Convert the password to bytes
-            salt_from_storage,
-            100000
-        )
-        new_key =  salt_from_storage + new_key
-
-        return new_key == raw_salt_pass
-
-    pass
-
-def changedetection_app(conig=None, datastore_o=None):
-    global datastore
-    datastore = datastore_o
-
-    app.config.update(dict(DEBUG=True))
-    #app.config.update(config or {})
-
-    login_manager = flask_login.LoginManager(app)
-    login_manager.login_view = 'login'
-
-
-    # Setup cors headers to allow all domains
-    # https://flask-cors.readthedocs.io/en/latest/
-    #    CORS(app)
-
-    @login_manager.user_loader
-    def user_loader(email):
-        user = User()
-        user.get_user(email)
-        return user
-
-    @login_manager.unauthorized_handler
-    def unauthorized_handler():
-        # @todo validate its a URL of this host and use that
-        return redirect(url_for('login', next=url_for('index')))
-
-    @app.route('/logout')
-    def logout():
-        flask_login.logout_user()
-        return redirect(url_for('index'))
-
-    # https://github.com/pallets/flask/blob/93dd1709d05a1cf0e886df6223377bdab3b077fb/examples/tutorial/flaskr/__init__.py#L39
-    # You can divide up the stuff like this
-    @app.route('/login', methods=['GET', 'POST'])
-    def login():
-
-        global messages
-
-        if request.method == 'GET':
-            output = render_template("login.html", messages=messages)
-            # Show messages but once.
-            messages = []
-            return output
-
-        user = User()
-        user.id = "defaultuser@changedetection.io"
-
-        password = request.form.get('password')
-
-        if (user.check_password(password)):
-            flask_login.login_user(user, remember=True)
-            next = request.args.get('next')
-            #            if not is_safe_url(next):
-            #                return flask.abort(400)
-            return redirect(next or url_for('index'))
-
-        else:
-            messages.append({'class': 'error', 'message': 'Incorrect password'})
-
-        return redirect(url_for('login'))
-
-    @app.before_request
-    def do_something_whenever_a_request_comes_in():
-        # Disable password  loginif there is not one set
-        app.config['LOGIN_DISABLED'] = datastore.data['settings']['application']['password'] == False
-
-    @app.route("/", methods=['GET'])
-    @login_required
-    def index():
-        global messages
-        limit_tag = request.args.get('tag')
-
-        pause_uuid = request.args.get('pause')
-
-        if pause_uuid:
-            try:
-                datastore.data['watching'][pause_uuid]['paused'] ^= True
-                datastore.needs_write = True
-
-                return redirect(url_for('index', tag = limit_tag))
-            except KeyError:
-                pass
-
-
-        # Sort by last_changed and add the uuid which is usually the key..
-        sorted_watches = []
-        for uuid, watch in datastore.data['watching'].items():
-
-            if limit_tag != None:
-                # Support for comma separated list of tags.
-                for tag_in_watch in watch['tag'].split(','):
-                    tag_in_watch = tag_in_watch.strip()
-                    if tag_in_watch == limit_tag:
-                        watch['uuid'] = uuid
-                        sorted_watches.append(watch)
-
-            else:
-                watch['uuid'] = uuid
-                sorted_watches.append(watch)
-
-        sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)
-
-        existing_tags = datastore.get_all_tags()
-        rss = request.args.get('rss')
-
-        if rss:
-            fg = FeedGenerator()
-            fg.title('changedetection.io')
-            fg.description('Feed description')
-            fg.link(href='https://changedetection.io')
-
-            for watch in sorted_watches:
-                if not watch['viewed']:
-                    fe = fg.add_entry()
-                    fe.title(watch['url'])
-                    fe.link(href=watch['url'])
-                    fe.description(watch['url'])
-                    fe.guid(watch['uuid'], permalink=False)
-                    dt = datetime.datetime.fromtimestamp(int(watch['newest_history_key']))
-                    dt = dt.replace(tzinfo=pytz.UTC)
-                    fe.pubDate(dt)
-
-            response = make_response(fg.rss_str())
-            response.headers.set('Content-Type', 'application/rss+xml')
-            return response
-
-        else:
-            output = render_template("watch-overview.html",
-                                     watches=sorted_watches,
-                                     messages=messages,
-                                     tags=existing_tags,
-                                     active_tag=limit_tag,
-                                     has_unviewed=datastore.data['has_unviewed'])
-
-            # Show messages but once.
-            messages = []
-
-        return output
-
-    @app.route("/scrub", methods=['GET', 'POST'])
-    @login_required
-    def scrub_page():
-        from pathlib import Path
-
-        global messages
-
-        if request.method == 'POST':
-            confirmtext = request.form.get('confirmtext')
-            limit_timestamp = int(request.form.get('limit_date'))
-
-            if confirmtext == 'scrub':
-
-                for uuid, watch in datastore.data['watching'].items():
-                    if len(str(limit_timestamp)) == 10:
-                        datastore.scrub_watch(uuid, limit_timestamp = limit_timestamp)
-                    else:
-                        datastore.scrub_watch(uuid)
-
-                messages.append({'class': 'ok', 'message': 'Cleaned all version history.'})
-            else:
-                messages.append({'class': 'error', 'message': 'Wrong confirm text.'})
-
-            return redirect(url_for('index'))
-
-        return render_template("scrub.html")
-
-    # If they edited an existing watch, we need to know to reset the current/previous md5 to include
-    # the excluded text.
-    def get_current_checksum_include_ignore_text(uuid):
-
-        import hashlib
-        from backend import fetch_site_status
-
-        # Get the most recent one
-        newest_history_key = datastore.get_val(uuid, 'newest_history_key')
-
-        # 0 means that theres only one, so that there should be no 'unviewed' history availabe
-        if newest_history_key == 0:
-            newest_history_key = list(datastore.data['watching'][uuid]['history'].keys())[0]
-
-        if newest_history_key:
-            with open(datastore.data['watching'][uuid]['history'][newest_history_key],
-                      encoding='utf-8') as file:
-                raw_content = file.read()
-
-                handler = fetch_site_status.perform_site_check(datastore=datastore)
-                stripped_content = handler.strip_ignore_text(raw_content,
-                                                             datastore.data['watching'][uuid]['ignore_text'])
-
-                checksum = hashlib.md5(stripped_content).hexdigest()
-                return checksum
-
-        return datastore.data['watching'][uuid]['previous_md5']
-
-    @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
-    @login_required
-    def edit_page(uuid):
-        global messages
-        import validators
-
-        # More for testing, possible to return the first/only
-        if uuid == 'first':
-            uuid = list(datastore.data['watching'].keys()).pop()
-
-        if request.method == 'POST':
-
-            url = request.form.get('url').strip()
-            tag = request.form.get('tag').strip()
-
-            # Extra headers
-            form_headers = request.form.get('headers').strip().split("\n")
-            extra_headers = {}
-            if form_headers:
-                for header in form_headers:
-                    if len(header):
-                        parts = header.split(':', 1)
-                        if len(parts) == 2:
-                            extra_headers.update({parts[0].strip(): parts[1].strip()})
-
-            update_obj = {'url': url,
-                          'tag': tag,
-                          'headers': extra_headers
-                          }
-
-            # Ignore text
-            form_ignore_text = request.form.get('ignore-text').strip()
-            ignore_text = []
-            if len(form_ignore_text):
-                for text in form_ignore_text.split("\n"):
-                    text = text.strip()
-                    if len(text):
-                        ignore_text.append(text)
-
-                datastore.data['watching'][uuid]['ignore_text'] = ignore_text
-
-                # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
-                if len(datastore.data['watching'][uuid]['history']):
-                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
-
-            validators.url(url)  # @todo switch to prop/attr/observer
-            datastore.data['watching'][uuid].update(update_obj)
-            datastore.needs_write = True
-
-            messages.append({'class': 'ok', 'message': 'Updated watch.'})
-
-            return redirect(url_for('index'))
-
-        else:
-            output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], messages=messages)
-
-        return output
-
-    @app.route("/settings", methods=['GET', "POST"])
-    @login_required
-    def settings_page():
-        global messages
-
-        if request.method == 'GET':
-            if request.values.get('removepassword'):
-                from pathlib import Path
-
-                datastore.data['settings']['application']['password'] = False
-                messages.append({'class': 'notice', 'message': "Password protection removed."})
-                flask_login.logout_user()
-
-                return redirect(url_for('settings_page'))
-
-        if request.method == 'POST':
-
-            password = request.values.get('password')
-            if password:
-                import hashlib
-                import base64
-                import secrets
-
-                # Make a new salt on every new password and store it with the password
-                salt = secrets.token_bytes(32)
-
-                key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
-                store = base64.b64encode(salt + key).decode('ascii')
-                datastore.data['settings']['application']['password'] = store
-                messages.append({'class': 'notice', 'message': "Password protection enabled."})
-                flask_login.logout_user()
-                return redirect(url_for('index'))
-
-            try:
-                minutes = int(request.values.get('minutes').strip())
-            except ValueError:
-                messages.append({'class': 'error', 'message': "Invalid value given, use an integer."})
-
-            else:
-                if minutes >= 5:
-                    datastore.data['settings']['requests']['minutes_between_check'] = minutes
-                    datastore.needs_write = True
-
-                    messages.append({'class': 'ok', 'message': "Updated"})
-                else:
-                    messages.append(
-                        {'class': 'error', 'message': "Must be atleast 5 minutes."})
-
-
-
-        output = render_template("settings.html", messages=messages,
-                                 minutes=datastore.data['settings']['requests']['minutes_between_check'])
-        messages = []
-
-        return output
-
-    @app.route("/import", methods=['GET', "POST"])
-    @login_required
-    def import_page():
-        import validators
-        global messages
-        remaining_urls = []
-
-        good = 0
-
-        if request.method == 'POST':
-            urls = request.values.get('urls').split("\n")
-            for url in urls:
-                url = url.strip()
-                if len(url) and validators.url(url):
-                    new_uuid = datastore.add_watch(url=url.strip(), tag="")
-                    # Straight into the queue.
-                    update_q.put(new_uuid)
-                    good += 1
-                else:
-                    if len(url):
-                        remaining_urls.append(url)
-
-            messages.append({'class': 'ok', 'message': "{} Imported, {} Skipped.".format(good, len(remaining_urls))})
-
-            if len(remaining_urls) == 0:
-                # Looking good, redirect to index.
-                return redirect(url_for('index'))
-
-        # Could be some remaining, or we could be on GET
-        output = render_template("import.html",
-                                 messages=messages,
-                                 remaining="\n".join(remaining_urls)
-                                 )
-        messages = []
-
-        return output
-
-    # Clear all statuses, so we do not see the 'unviewed' class
-    @app.route("/api/mark-all-viewed", methods=['GET'])
-    @login_required
-    def mark_all_viewed():
-
-        # Save the current newest history as the most recently viewed
-        for watch_uuid, watch in datastore.data['watching'].items():
-            datastore.set_last_viewed(watch_uuid, watch['newest_history_key'])
-
-        messages.append({'class': 'ok', 'message': "Cleared all statuses."})
-        return redirect(url_for('index'))
-
-    @app.route("/diff/<string:uuid>", methods=['GET'])
-    @login_required
-    def diff_history_page(uuid):
-        global messages
-
-        # More for testing, possible to return the first/only
-        if uuid == 'first':
-            uuid = list(datastore.data['watching'].keys()).pop()
-
-        extra_stylesheets = ['/static/css/diff.css']
-        try:
-            watch = datastore.data['watching'][uuid]
-        except KeyError:
-            messages.append({'class': 'error', 'message': "No history found for the specified link, bad link?"})
-            return redirect(url_for('index'))
-
-        dates = list(watch['history'].keys())
-        # Convert to int, sort and back to str again
-        dates = [int(i) for i in dates]
-        dates.sort(reverse=True)
-        dates = [str(i) for i in dates]
-
-        if len(dates) < 2:
-            messages.append(
-                {'class': 'error', 'message': "Not enough saved change detection snapshots to produce a report."})
-            return redirect(url_for('index'))
-
-        # Save the current newest history as the most recently viewed
-        datastore.set_last_viewed(uuid, dates[0])
-
-        newest_file = watch['history'][dates[0]]
-        with open(newest_file, 'r') as f:
-            newest_version_file_contents = f.read()
-
-        previous_version = request.args.get('previous_version')
-
-        try:
-            previous_file = watch['history'][previous_version]
-        except KeyError:
-            # Not present, use a default value, the second one in the sorted list.
-            previous_file = watch['history'][dates[1]]
-
-        with open(previous_file, 'r') as f:
-            previous_version_file_contents = f.read()
-
-        output = render_template("diff.html", watch_a=watch,
-                                 messages=messages,
-                                 newest=newest_version_file_contents,
-                                 previous=previous_version_file_contents,
-                                 extra_stylesheets=extra_stylesheets,
-                                 versions=dates[1:],
-                                 newest_version_timestamp=dates[0],
-                                 current_previous_version=str(previous_version),
-                                 current_diff_url=watch['url'])
-
-        return output
-
-    @app.route("/preview/<string:uuid>", methods=['GET'])
-    @login_required
-    def preview_page(uuid):
-        global messages
-
-        # More for testing, possible to return the first/only
-        if uuid == 'first':
-            uuid = list(datastore.data['watching'].keys()).pop()
-
-        extra_stylesheets = ['/static/css/diff.css']
-
-        try:
-            watch = datastore.data['watching'][uuid]
-        except KeyError:
-            messages.append({'class': 'error', 'message': "No history found for the specified link, bad link?"})
-            return redirect(url_for('index'))
-
-        print(watch)
-        with open(list(watch['history'].values())[-1], 'r') as f:
-            content = f.readlines()
-
-        output = render_template("preview.html", content=content, extra_stylesheets=extra_stylesheets)
-        return output
-
-
-    @app.route("/favicon.ico", methods=['GET'])
-    def favicon():
-        return send_from_directory("/app/static/images", filename="favicon.ico")
-
-    # We're good but backups are even better!
-    @app.route("/backup", methods=['GET'])
-    @login_required
-    def get_backup():
-
-        import zipfile
-        from pathlib import Path
-
-        # create a ZipFile object
-        backupname = "changedetection-backup-{}.zip".format(int(time.time()))
-
-        # We only care about UUIDS from the current index file
-        uuids = list(datastore.data['watching'].keys())
-        backup_filepath = os.path.join(app.config['datastore_path'], backupname)
-
-        with zipfile.ZipFile(backup_filepath, "w",
-                             compression=zipfile.ZIP_DEFLATED,
-                             compresslevel=8) as zipObj:
-
-            # Be sure we're written fresh
-            datastore.sync_to_json()
-
-            # Add the index
-            zipObj.write(os.path.join(app.config['datastore_path'], "url-watches.json"), arcname="url-watches.json")
-
-            # Add the flask app secret
-            zipObj.write(os.path.join(app.config['datastore_path'], "secret.txt"), arcname="secret.txt")
-
-            # Add any snapshot data we find, use the full path to access the file, but make the file 'relative' in the Zip.
-            for txt_file_path in Path(app.config['datastore_path']).rglob('*.txt'):
-                parent_p = txt_file_path.parent
-                if parent_p.name in uuids:
-                    zipObj.write(txt_file_path,
-                                 arcname=str(txt_file_path).replace(app.config['datastore_path'], ''),
-                                 compress_type=zipfile.ZIP_DEFLATED,
-                                 compresslevel=8)
-
-        return send_from_directory(app.config['datastore_path'], backupname)
-
-    @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
-    def static_content(group, filename):
-        # These files should be in our subdirectory
-        full_path = os.path.realpath(__file__)
-        p = os.path.dirname(full_path)
-
-        try:
-            return send_from_directory("{}/static/{}".format(p, group), filename=filename)
-        except FileNotFoundError:
-            abort(404)
-
-    @app.route("/api/add", methods=['POST'])
-    @login_required
-    def api_watch_add():
-        global messages
-
-        url = request.form.get('url').strip()
-        if datastore.url_exists(url):
-            messages.append({'class': 'error', 'message': 'The URL {} already exists'.format(url)})
-            return redirect(url_for('index'))
-
-        # @todo add_watch should throw a custom Exception for validation etc
-        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
-        # Straight into the queue.
-        update_q.put(new_uuid)
-
-        messages.append({'class': 'ok', 'message': 'Watch added.'})
-        return redirect(url_for('index'))
-
-    @app.route("/api/delete", methods=['GET'])
-    @login_required
-    def api_delete():
-        global messages
-        uuid = request.args.get('uuid')
-        datastore.delete(uuid)
-        messages.append({'class': 'ok', 'message': 'Deleted.'})
-
-        return redirect(url_for('index'))
-
-    @app.route("/api/checknow", methods=['GET'])
-    @login_required
-    def api_watch_checknow():
-
-        global messages
-
-        tag = request.args.get('tag')
-        uuid = request.args.get('uuid')
-        i = 0
-
-        running_uuids = []
-        for t in running_update_threads:
-            running_uuids.append(t.current_uuid)
-
-        # @todo check thread is running and skip
-
-        if uuid:
-            if uuid not in running_uuids:
-                update_q.put(uuid)
-            i = 1
-
-        elif tag != None:
-            # Items that have this current tag
-            for watch_uuid, watch in datastore.data['watching'].items():
-                if (tag != None and tag in watch['tag']):
-                    if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                        update_q.put(watch_uuid)
-                        i += 1
-
-        else:
-            # No tag, no uuid, add everything.
-            for watch_uuid, watch in datastore.data['watching'].items():
-
-                if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
-                    update_q.put(watch_uuid)
-                    i += 1
-
-        messages.append({'class': 'ok', 'message': "{} watches are rechecking.".format(i)})
-        return redirect(url_for('index', tag=tag))
-
-    # @todo handle ctrl break
-    ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
-
-    # Check for new release version
-    threading.Thread(target=check_for_new_version).start()
-    return app
-
-
-# Check for new version and anonymous stats
-def check_for_new_version():
-    import requests
-
-    import urllib3
-    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-    while not app.config.exit.is_set():
-        try:
-            r = requests.post("https://changedetection.io/check-ver.php",
-                              data={'version': datastore.data['version_tag'],
-                                    'app_guid': datastore.data['app_guid']},
-
-                              verify=False)
-        except:
-            pass
-
-        try:
-            if "new_version" in r.text:
-                app.config['NEW_VERSION_AVAILABLE'] = True
-        except:
-            pass
-
-        # Check daily
-        app.config.exit.wait(86400)
-
-
-# Requests for checking on the site use a pool of thread Workers managed by a Queue.
-class Worker(threading.Thread):
-    current_uuid = None
-
-    def __init__(self, q, *args, **kwargs):
-        self.q = q
-        super().__init__(*args, **kwargs)
-
-    def run(self):
-        from backend import fetch_site_status
-
-        update_handler = fetch_site_status.perform_site_check(datastore=datastore)
-
-        while not app.config.exit.is_set():
-
-            try:
-                uuid = self.q.get(block=False)
-            except queue.Empty:
-                pass
-
-            else:
-                self.current_uuid = uuid
-
-                if uuid in list(datastore.data['watching'].keys()):
-                    try:
-                        changed_detected, result, contents = update_handler.run(uuid)
-
-                    except PermissionError as s:
-                        app.logger.error("File permission error updating", uuid, str(s))
-                    else:
-                        if result:
-                            datastore.update_watch(uuid=uuid, update_obj=result)
-                            if changed_detected:
-                                # A change was detected
-                                datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
-
-                self.current_uuid = None  # Done
-                self.q.task_done()
-
-            app.config.exit.wait(1)
-
-
-# Thread runner to check every minute, look for new watches to feed into the Queue.
-def ticker_thread_check_time_launch_checks():
-    # Spin up Workers.
-    for _ in range(datastore.data['settings']['requests']['workers']):
-        new_worker = Worker(update_q)
-        running_update_threads.append(new_worker)
-        new_worker.start()
-
-    while not app.config.exit.is_set():
-
-        running_uuids = []
-        for t in running_update_threads:
-            if t.current_uuid:
-                running_uuids.append(t.current_uuid)
-
-        # Look at the dataset, find a stale watch to process
-
-        # Every minute check for new UUIDs to follow up on, should be inside the loop incase it changes.
-        minutes = datastore.data['settings']['requests']['minutes_between_check']
-
-        threshold = time.time() - (minutes * 60)
-        for uuid, watch in datastore.data['watching'].items():
-            if not watch['paused'] and watch['last_checked'] <= threshold:
-                if not uuid in running_uuids and uuid not in update_q.queue:
-                    update_q.put(uuid)
-
-        # Should be low so we can break this out in testing
-        app.config.exit.wait(1)

backend/fetch_site_status.py

@@ -1,118 +0,0 @@
-import time
-import requests
-import hashlib
-from inscriptis import get_text
-import urllib3
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-# Some common stuff here that can be moved to a base class
-class perform_site_check():
-
-    def __init__(self, *args, datastore, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.datastore = datastore
-
-    def strip_ignore_text(self, content, list_ignore_text):
-        ignore = []
-        for k in list_ignore_text:
-            ignore.append(k.encode('utf8'))
-
-        output = []
-        for line in content.splitlines():
-            line = line.encode('utf8')
-
-            # Always ignore blank lines in this mode. (when this function gets called)
-            if len(line.strip()):
-                if not any(skip_text in line for skip_text in ignore):
-                    output.append(line)
-
-        return "\n".encode('utf8').join(output)
-
-
-
-    def run(self, uuid):
-        timestamp = int(time.time())  # used for storage etc too
-        stripped_text_from_html = False
-        changed_detected = False
-
-        update_obj = {'previous_md5': self.datastore.data['watching'][uuid]['previous_md5'],
-                      'history': {},
-                      "last_checked": timestamp
-                      }
-
-        extra_headers = self.datastore.get_val(uuid, 'headers')
-
-        # Tweak the base config with the per-watch ones
-        request_headers = self.datastore.data['settings']['headers']
-        request_headers.update(extra_headers)
-
-        # https://github.com/psf/requests/issues/4525
-        # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
-        # do this by accident.
-        if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
-            request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
-
-        try:
-            timeout = self.datastore.data['settings']['requests']['timeout']
-        except KeyError:
-            # @todo yeah this should go back to the default value in store.py, but this whole object should abstract off it
-            timeout = 15
-
-        try:
-            url = self.datastore.get_val(uuid, 'url')
-
-            r = requests.get(url,
-                             headers=request_headers,
-                             timeout=timeout,
-                             verify=False)
-
-            stripped_text_from_html = get_text(r.text)
-
-        # Usually from networkIO/requests level
-        except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
-            update_obj["last_error"] = str(e)
-
-            print(str(e))
-
-        except requests.exceptions.MissingSchema:
-            print("Skipping {} due to missing schema/bad url".format(uuid))
-
-        # Usually from html2text level
-        except UnicodeDecodeError as e:
-
-            update_obj["last_error"] = str(e)
-            print(str(e))
-            # figure out how to deal with this cleaner..
-            # 'utf-8' codec can't decode byte 0xe9 in position 480: invalid continuation byte
-
-        else:
-            # We rely on the actual text in the html output.. many sites have random script vars etc,
-            # in the future we'll implement other mechanisms.
-
-            update_obj["last_check_status"] = r.status_code
-            update_obj["last_error"] = False
-
-            if not len(r.text):
-                update_obj["last_error"] = "Empty reply"
-
-            # If there's text to skip
-            # @todo we could abstract out the get_text() to handle this cleaner
-            if len(self.datastore.data['watching'][uuid]['ignore_text']):
-                content = self.strip_ignore_text(stripped_text_from_html,
-                                                 self.datastore.data['watching'][uuid]['ignore_text'])
-            else:
-                content = stripped_text_from_html.encode('utf8')
-
-            fetched_md5 = hashlib.md5(content).hexdigest()
-
-            # could be None or False depending on JSON type
-            if self.datastore.data['watching'][uuid]['previous_md5'] != fetched_md5:
-                changed_detected = True
-
-                # Don't confuse people by updating as last-changed, when it actually just changed from None..
-                if self.datastore.get_val(uuid, 'previous_md5'):
-                    update_obj["last_changed"] = timestamp
-
-                update_obj["previous_md5"] = fetched_md5
-
-        return changed_detected, update_obj, stripped_text_from_html

backend/static/css/styles.css

@@ -1,277 +0,0 @@
-/*
- * -- BASE STYLES --
- * Most of these are inherited from Base, but I want to change a few.
- */
-body {
-  color: #333;
-  background: #262626;
-}
-
-.pure-table-even {
-  background: #fff;
-}
-/* Some styles from https://css-tricks.com/ */
-
-a {
-  text-decoration: none;
-  color: #1b98f8;
-}
-a.github-link {
-  color: #fff;
-}
-
-
-.pure-menu-horizontal {
-  background: #fff;
-  padding: 5px;
-  display: flex;
-  justify-content: space-between;
-  border-bottom: 2px solid #ed5900;
-
-  align-items: center;
-}
-
-section.content {
-
-  padding-top: 5em;
-  padding-bottom: 5em;
-  flex-direction: column;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-}
-
-.pure-table.watch-table td {
-  font-size: 80%;
-}
-
-/* table related */
-.watch-table {
-  width: 100%;
-
-}
-
-.watch-table tr.unviewed {
-    font-weight: bold;
-}
-
-.watch-tag-list {
-  color: #e70069;
-  white-space: nowrap;
-}
-
-.box {
-  max-width: 80%;
-  flex-direction: column;
-  display: flex;
-  justify-content: center;
-}
-
-.watch-table .error {
-  color: #a00;
-}
-
-.watch-table td {
-  white-space: nowrap;
-}
-
-.watch-table td.title-col {
-  word-break: break-all;
-  white-space: normal;
-}
-.watch-table th {
-  white-space: nowrap;
-}
-
-.watch-table .title-col a[target="_blank"]::after, .current-diff-url::after {
-  content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
-  margin: 0 3px 0 5px;
-}
-
-#post-list-buttons {
-  text-align: right;
-  padding: 0px;
-  margin: 0px;
-}
-#post-list-buttons li {
-  display: inline-block;
-}
-
-#post-list-buttons a {
-   border-top-left-radius: initial;
-   border-top-right-radius: initial;
-   border-bottom-left-radius: 5px;
-   border-bottom-right-radius: 5px;
-}
-
-
-body:after {
-  content: "";
-  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)
-}
-
-body:after, body:before {
-  display: block;
-  height: 600px;
-  position: absolute;
-  top: 0;
-  left: 0;
-  width: 100%;
-  z-index: -1;
-}
-body::after {
-  opacity: 0.91;
-}
-body::before {
-  content: "";
-  background-image: url(/static/images/gradient-border.png);
-}
-body:before {
-  background-size: cover
-}
-
-body:after, body:before {
-  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
-  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
-}
-
-
-.button-small {
-  font-size: 85%;
-}
-
-
-.fetch-error {
-  padding-top: 1em;
-  font-size: 60%;
-  max-width: 400px;
-  display: block;
-}
-
-.edit-form {
-   background: #fff;
-   padding: 2em;
-   margin: 1em;
-   border-radius: 5px;
-}
-.button-secondary {
-    color: white;
-    border-radius: 4px;
-    text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
-}
-
-.button-success {
-    background: rgb(28, 184, 65);
-    /* this is a green */
-}
-
-.button-tag {
-    background: rgb(99, 99, 99);
-    color: #fff;
-    font-size: 65%;
-    border-bottom-left-radius: initial;
-    border-bottom-right-radius: initial;
-}
-.button-tag.active {
-    background: #9c9c9c;
-    font-weight: bold;
-}
-.button-error {
-    background: rgb(202, 60, 60);
-    /* this is a maroon */
-}
-
-.button-warning {
-    background: rgb(223, 117, 20);
-    /* this is an orange */
-}
-
-.button-secondary {
-    background: rgb(66, 184, 221);
-    /* this is a light blue */
-}
-
-
-.button-cancel {
-    background: rgb(200, 200, 200);
-    /* this is a green */
-}
-
-.messages {
-    padding: 1em;
-    background: rgba(255,255,255,.2);
-    border-radius: 10px;
-    color: #fff;
-    font-weight: bold;
-}
-
-.pure-form label  {
-  font-weight: bold;
-}
-
-#new-watch-form {
-    background: rgba(0,0,0,.05);
-    padding: 1em;
-    border-radius: 10px;
-    margin-bottom: 1em;
-}
-#new-watch-form legend {
-    color: #fff;
-}
-
-#diff-col {
-    padding-left:40px;
-}
-#diff-jump {
-    position: fixed;
-    left: 0px;
-    top: 80px;
-    background: #fff;
-    padding: 10px;
-    border-top-right-radius: 5px;
-    border-bottom-right-radius: 5px;
-    box-shadow: 5px 0 5px -2px #888;
-}
-
-#diff-jump a {
-    color: #1b98f8;
-    cursor: grabbing;
-    -moz-user-select: none;
-    -webkit-user-select: none;
-    -ms-user-select:none;
-    user-select:none;
-    -o-user-select:none;
-}
-
-footer {
-    padding: 10px;
-    background: #fff;
-    color: #444;
-    text-align: center;
-}
-
-#feed-icon {
-    vertical-align: middle;
-}
-
-#version {
-    position: absolute;
-    top: 80px;
-    right: 0px;
-    font-size: 8px;
-    background: #fff;
-    padding: 10px;
-}
-
-#new-version-text a{
-    color: #e07171;
-}
-
-.paused-state.state-False img {
-    opacity: 0.2;
-}
-
-
-.paused-state.state-False:hover img{
-    opacity: 0.8;
-}

backend/templates/edit.html

@@ -1,73 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-<div class="edit-form">
-
-
-    <form class="pure-form pure-form-stacked" action="/edit/{{uuid}}" method="POST">
-        <fieldset>
-            <div class="pure-control-group">
-                <label for="url">URL</label>
-                <input type="url" id="url" required="" placeholder="https://..." name="url" value="{{ watch.url}}"
-                       size="50"/>
-                <span class="pure-form-message-inline">This is a required field.</span>
-            </div>
-            <div class="pure-control-group">
-                <label for="tag">Tag</label>
-                <input type="text" placeholder="tag" size="10" id="tag" name="tag" value="{{ watch.tag}}"/>
-                <span class="pure-form-message-inline">Grouping tags, can be a comma separated list.</span>
-            </div>
-
-            <!-- @todo: move to tabs --->
-            <fieldset class="pure-group">
-                <label for="ignore-text">Ignore text</label>
-
-                <textarea id="ignore-text" name="ignore-text" class="pure-input-1-2" placeholder=""
-                          style="width: 100%;
-                            font-family:monospace;
-                            white-space: pre;
-                            overflow-wrap: normal;
-                            overflow-x: scroll;" rows="5">{% for value in watch.ignore_text %}{{ value }}
-{% endfor %}</textarea>
-                <span class="pure-form-message-inline">Each line will be processed separately as an ignore rule.</span>
-
-            </fieldset>
-
-            <!-- @todo: move to tabs --->
-            <fieldset class="pure-group">
-                <label for="headers">Extra request headers</label>
-
-                <textarea id="headers" name="headers" class="pure-input-1-2" placeholder="Example
-Cookie: foobar
-User-Agent: wonderbra 1.0"
-                          style="width: 100%;
-                            font-family:monospace;
-                            white-space: pre;
-                            overflow-wrap: normal;
-                            overflow-x: scroll;" rows="5">{% for key, value in watch.headers.items() %}{{ key }}: {{ value }}
-{% endfor %}</textarea>
-                <br/>
-
-            </fieldset>
-
-
-            <div class="pure-control-group">
-                <button type="submit" class="pure-button pure-button-primary">Save</button>
-            </div>
-            <br/>
-
-            <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Cancel</a>
-                <a href="/api/delete?uuid={{uuid}}"
-                   class="pure-button button-small button-error ">Delete</a>
-
-            </div>
-
-
-        </fieldset>
-    </form>
-
-
-</div>
-
-{% endblock %}

backend/templates/import.html

@@ -1,26 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-<div class="edit-form">
-
-
-    <form class="pure-form pure-form-aligned" action="/import" method="POST">
-
-        <fieldset class="pure-group">
-            <legend>One URL per line, URLs that do not pass validation will stay in the textarea.</legend>
-
-            <textarea name="urls" class="pure-input-1-2" placeholder="https://"
-                      style="width: 100%;
-                            font-family:monospace;
-                            white-space: pre;
-                            overflow-wrap: normal;
-                            overflow-x: scroll;" rows="25">{{ remaining }}</textarea>
-        </fieldset>
-        <button type="submit" class="pure-button pure-input-1-2 pure-button-primary">Import</button>
-
-    </form>
-
-</div>
-
-{% endblock %}
-

backend/templates/settings.html

@@ -1,42 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-<div class="edit-form">
-
-
-    <form class="pure-form pure-form-stacked" action="/settings" method="POST">
-        <fieldset>
-            <div class="pure-control-group">
-                <label for="minutes">Maximum time in minutes until recheck.</label>
-                <input type="text" id="minutes" required="" name="minutes" value="{{minutes}}"
-                       size="5"/>
-                <span class="pure-form-message-inline">This is a required field.</span>
-            </div>
-            <br/>
-            <div class="pure-control-group">
-                <label for="minutes">Password protection</label>
-                <input type="password" id="password" name="password" size="15"/>
-                {% if current_user.is_authenticated %}
-                    <a href="/settings?removepassword=true" class="pure-button pure-button-primary">Remove password</a>
-                {% endif %}
-            </div>
-
-            <br/>
-            <div class="pure-control-group">
-                <button type="submit" class="pure-button pure-button-primary">Save</button>
-            </div>
-            <br/>
-
-            <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Back</a>
-                <a href="/scrub" class="pure-button button-small button-cancel">Delete history version data</a>
-            </div>
-
-
-        </fieldset>
-    </form>
-
-
-</div>
-
-{% endblock %}

changedetection.py

@@ -8,43 +8,26 @@ import sys
 
 import eventlet
 import eventlet.wsgi
-import backend
+import changedetectionio
 
-from backend import store
+from changedetectionio import store
 
-
-def init_app_secret(datastore_path):
-    secret = ""
-
-    path = "{}/secret.txt".format(datastore_path)
-
-    try:
-        with open(path, "r") as f:
-            secret = f.read()
-
-    except FileNotFoundError:
-
-        import secrets
-        with open(path, "w") as f:
-            secret = secrets.token_hex(32)
-            f.write(secret)
-
-    return secret
-
-
-def main(argv):
+def main():
     ssl_mode = False
-    port = 5000
+    port = os.environ.get('PORT') or 5000
+    do_cleanup = False
 
     # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
     datastore_path = os.path.join(os.getcwd(), "datastore")
 
     try:
-        opts, args = getopt.getopt(argv, "sd:p:", "purge")
+        opts, args = getopt.getopt(sys.argv[1:], "Ccsd:p:", "port")
     except getopt.GetoptError:
         print('backend.py -s SSL enable -p [port] -d [datastore path]')
         sys.exit(2)
 
+    create_datastore_dir = False
+
     for opt, arg in opts:
         #        if opt == '--purge':
         # Remove history, the actual files you need to delete manually.
@@ -60,21 +43,53 @@ def main(argv):
         if opt == '-d':
             datastore_path = arg
 
+        # Cleanup (remove text files that arent in the index)
+        if opt == '-c':
+            do_cleanup = True
+
+        # Create the datadir if it doesnt exist
+        if opt == '-C':
+            create_datastore_dir = True
+
     # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
     app_config = {'datastore_path': datastore_path}
 
-    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'])
-    app = backend.changedetection_app(app_config, datastore)
+    if not os.path.isdir(app_config['datastore_path']):
+        if create_datastore_dir:
+            os.mkdir(app_config['datastore_path'])
+        else:
+            print ("ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists.\n"
+                   "Alternatively, use the -d parameter.".format(app_config['datastore_path']),file=sys.stderr)
+            sys.exit(2)
+
+    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=changedetectionio.__version__)
+    app = changedetectionio.changedetection_app(app_config, datastore)
+
+    # Go into cleanup mode
+    if do_cleanup:
+        datastore.remove_unused_snapshots()
+
+    app.config['datastore_path'] = datastore_path
 
-    app.secret_key = init_app_secret(app_config['datastore_path'])
 
     @app.context_processor
     def inject_version():
-        return dict(version=datastore.data['version_tag'])
+        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
+                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
+                    has_password=datastore.data['settings']['application']['password'] != False
+                    )
 
-    @app.context_processor
-    def inject_new_version_available():
-        return dict(new_version_available=app.config['NEW_VERSION_AVAILABLE'])
+    # Proxy sub-directory support
+    # Set environment var USE_X_SETTINGS=1 on this script
+    # And then in your proxy_pass settings
+    #
+    #         proxy_set_header Host "localhost";
+    #         proxy_set_header X-Forwarded-Prefix /app;
+
+    if os.getenv('USE_X_SETTINGS'):
+        print ("USE_X_SETTINGS is ENABLED\n")
+        from werkzeug.middleware.proxy_fix import ProxyFix
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
 
     if ssl_mode:
         # @todo finalise SSL config, but this should get you in the right direction if you need it.
@@ -84,8 +99,8 @@ def main(argv):
                                                server_side=True), app)
 
     else:
-        eventlet.wsgi.server(eventlet.listen(('', port)), app)
+        eventlet.wsgi.server(eventlet.listen(('', int(port))), app)
 
 
 if __name__ == '__main__':
-    main(sys.argv[1:])
+    main()

changedetectionio/content_fetcher.py

@@ -0,0 +1,170 @@
+import os
+import time
+from abc import ABC, abstractmethod
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
+from selenium.common.exceptions import WebDriverException
+import urllib3.exceptions
+
+
+class EmptyReply(Exception):
+    def __init__(self, status_code, url):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        return
+
+    pass
+
+class Fetcher():
+    error = None
+    status_code = None
+    content = None # Should always be bytes.
+    headers = None
+
+    fetcher_description ="No description"
+
+    @abstractmethod
+    def get_error(self):
+        return self.error
+
+    @abstractmethod
+    def run(self, url, timeout, request_headers, request_body, request_method):
+        # Should set self.error, self.status_code and self.content
+        pass
+
+    @abstractmethod
+    def get_last_status_code(self):
+        return self.status_code
+
+    @abstractmethod
+    # Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
+    def is_ready(self):
+        return True
+
+#   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
+#   the current one would return javascript output (as we use JS to generate the diff)
+#
+#   Returns tuple(mime_type, stream)
+#    @abstractmethod
+#    def return_diff(self, stream_a, stream_b):
+#        return
+
+def available_fetchers():
+        import inspect
+        from changedetectionio import content_fetcher
+        p=[]
+        for name, obj in inspect.getmembers(content_fetcher):
+            if inspect.isclass(obj):
+                # @todo html_ is maybe better as fetcher_ or something
+                # In this case, make sure to edit the default one in store.py and fetch_site_status.py
+                if "html_" in name:
+                    t=tuple([name,obj.fetcher_description])
+                    p.append(t)
+
+        return p
+
+class html_webdriver(Fetcher):
+    if os.getenv("WEBDRIVER_URL"):
+        fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))
+    else:
+        fetcher_description = "WebDriver Chrome/Javascript"
+
+    command_executor = ''
+
+    # Configs for Proxy setup
+    # In the ENV vars, is prefixed with "webdriver_", so it is for example "webdriver_sslProxy"
+    selenium_proxy_settings_mappings = ['proxyType', 'ftpProxy', 'httpProxy', 'noProxy',
+                                        'proxyAutoconfigUrl', 'sslProxy', 'autodetect',
+                                        'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
+
+
+
+    proxy=None
+
+    def __init__(self):
+        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+        self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
+
+        # If any proxy settings are enabled, then we should setup the proxy object
+        proxy_args = {}
+        for k in self.selenium_proxy_settings_mappings:
+            v = os.getenv('webdriver_' + k, False)
+            if v:
+                proxy_args[k] = v.strip('"')
+
+        if proxy_args:
+            self.proxy = SeleniumProxy(raw=proxy_args)
+
+    def run(self, url, timeout, request_headers, request_body, request_method):
+
+        # request_body, request_method unused for now, until some magic in the future happens.
+
+        # check env for WEBDRIVER_URL
+        driver = webdriver.Remote(
+            command_executor=self.command_executor,
+            desired_capabilities=DesiredCapabilities.CHROME,
+            proxy=self.proxy)
+
+        try:
+            driver.get(url)
+        except WebDriverException as e:
+            # Be sure we close the session window
+            driver.quit()
+            raise
+
+        # @todo - how to check this? is it possible?
+        self.status_code = 200
+        # @todo somehow we should try to get this working for WebDriver
+        # raise EmptyReply(url=url, status_code=r.status_code)
+
+        # @todo - dom wait loaded?
+        time.sleep(5)
+        self.content = driver.page_source
+        self.headers = {}
+
+        driver.quit()
+
+
+    def is_ready(self):
+        from selenium import webdriver
+        from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+        from selenium.common.exceptions import WebDriverException
+
+        driver = webdriver.Remote(
+            command_executor=self.command_executor,
+            desired_capabilities=DesiredCapabilities.CHROME)
+
+        # driver.quit() seems to cause better exceptions
+        driver.quit()
+
+        return True
+
+# "html_requests" is listed as the default fetcher in store.py!
+class html_requests(Fetcher):
+    fetcher_description = "Basic fast Plaintext/HTTP Client"
+
+    def run(self, url, timeout, request_headers, request_body, request_method):
+        import requests
+
+        r = requests.request(method=request_method,
+                         data=request_body,
+                         url=url,
+                         headers=request_headers,
+                         timeout=timeout,
+                         verify=False)
+
+        # https://stackoverflow.com/questions/44203397/python-requests-get-returns-improperly-decoded-text-instead-of-utf-8
+        # Return bytes here
+        html = r.text
+
+        # @todo test this
+        # @todo maybe you really want to test zero-byte return pages?
+        if not r or not html or not len(html):
+            raise EmptyReply(url=url, status_code=r.status_code)
+
+        self.status_code = r.status_code
+        self.content = html
+        self.headers = r.headers
+

changedetectionio/diff.py

@@ -0,0 +1,43 @@
+# used for the notifications, the front-end is using a JS library
+
+import difflib
+
+# like .compare but a little different output
+def customSequenceMatcher(before, after, include_equal=False):
+    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
+
+    for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
+        if include_equal and tag == 'equal':
+            g = before[alo:ahi]
+            yield g
+        elif tag == 'delete':
+            g = "(removed) {}".format(before[alo])
+            yield g
+        elif tag == 'replace':
+            g = ["(changed) {}".format(before[alo]), "(-> into) {}".format(after[blo])]
+            yield g
+        elif tag == 'insert':
+            g = "(added) {}".format(after[blo])
+            yield g
+
+# only_differences - only return info about the differences, no context
+# line_feed_sep could be "<br/>" or "<li>" or "\n" etc
+def render_diff(previous_file, newest_file, include_equal=False, line_feed_sep="\n"):
+    with open(newest_file, 'r') as f:
+        newest_version_file_contents = f.read()
+        newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
+
+    if previous_file:
+        with open(previous_file, 'r') as f:
+            previous_version_file_contents = f.read()
+            previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
+    else:
+        previous_version_file_contents = ""
+
+    rendered_diff = customSequenceMatcher(previous_version_file_contents,
+                                          newest_version_file_contents,
+                                          include_equal)
+
+    # Recursively join lists
+    f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
+    return f(rendered_diff)

changedetectionio/fetch_site_status.py

@@ -0,0 +1,194 @@
+import time
+from changedetectionio import content_fetcher
+import hashlib
+from inscriptis import get_text
+import urllib3
+from . import html_tools
+import re
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+# Some common stuff here that can be moved to a base class
+class perform_site_check():
+
+    def __init__(self, *args, datastore, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.datastore = datastore
+
+    def strip_ignore_text(self, content, list_ignore_text):
+        import re
+        ignore = []
+        ignore_regex = []
+        for k in list_ignore_text:
+
+            # Is it a regex?
+            if k[0] == '/':
+                ignore_regex.append(k.strip(" /"))
+            else:
+                ignore.append(k)
+
+        output = []
+        for line in content.splitlines():
+
+            # Always ignore blank lines in this mode. (when this function gets called)
+            if len(line.strip()):
+                regex_matches = False
+
+                # if any of these match, skip
+                for regex in ignore_regex:
+                    try:
+                        if re.search(regex, line, re.IGNORECASE):
+                            regex_matches = True
+                    except Exception as e:
+                        continue
+
+                if not regex_matches and not any(skip_text in line for skip_text in ignore):
+                    output.append(line.encode('utf8'))
+
+        return "\n".encode('utf8').join(output)
+
+
+
+    def run(self, uuid):
+        timestamp = int(time.time())  # used for storage etc too
+
+        changed_detected = False
+        stripped_text_from_html = ""
+
+        watch = self.datastore.data['watching'][uuid]
+
+        update_obj = {}
+
+        extra_headers = self.datastore.get_val(uuid, 'headers')
+
+        # Tweak the base config with the per-watch ones
+        request_headers = self.datastore.data['settings']['headers'].copy()
+        request_headers.update(extra_headers)
+
+        # https://github.com/psf/requests/issues/4525
+        # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
+        # do this by accident.
+        if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
+            request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
+
+        # @todo check the failures are really handled how we expect
+
+        else:
+            timeout = self.datastore.data['settings']['requests']['timeout']
+            url = self.datastore.get_val(uuid, 'url')
+            request_body = self.datastore.get_val(uuid, 'body')
+            request_method = self.datastore.get_val(uuid, 'method')
+
+            # Pluggable content fetcher
+            prefer_backend = watch['fetch_backend']
+            if hasattr(content_fetcher, prefer_backend):
+                klass = getattr(content_fetcher, prefer_backend)
+            else:
+                # If the klass doesnt exist, just use a default
+                klass = getattr(content_fetcher, "html_requests")
+
+
+            fetcher = klass()
+            fetcher.run(url, timeout, request_headers, request_body, request_method)
+            # Fetching complete, now filters
+            # @todo move to class / maybe inside of fetcher abstract base?
+
+            # @note: I feel like the following should be in a more obvious chain system
+            #  - Check filter text
+            #  - Is the checksum different?
+            #  - Do we convert to JSON?
+            # https://stackoverflow.com/questions/41817578/basic-method-chaining ?
+            # return content().textfilter().jsonextract().checksumcompare() ?
+
+            is_json = fetcher.headers.get('Content-Type', '') == 'application/json'
+            is_html = not is_json
+            css_filter_rule = watch['css_filter']
+
+            has_filter_rule = css_filter_rule and len(css_filter_rule.strip())
+            if is_json and not has_filter_rule:
+                css_filter_rule = "json:$"
+                has_filter_rule = True
+
+            if has_filter_rule:
+                if 'json:' in css_filter_rule:
+                    stripped_text_from_html = html_tools.extract_json_as_string(content=fetcher.content, jsonpath_filter=css_filter_rule)
+                    is_html = False
+
+            if is_html:
+                # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+                html_content = fetcher.content
+                if has_filter_rule:
+                    # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
+                    if css_filter_rule[0] == '/':
+                        html_content = html_tools.xpath_filter(xpath_filter=css_filter_rule, html_content=fetcher.content)
+                    else:
+                        # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+                        html_content = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
+
+                # get_text() via inscriptis
+                stripped_text_from_html = get_text(html_content)
+
+            # Re #340 - return the content before the 'ignore text' was applied
+            text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
+
+            # We rely on the actual text in the html output.. many sites have random script vars etc,
+            # in the future we'll implement other mechanisms.
+
+            update_obj["last_check_status"] = fetcher.get_last_status_code()
+            update_obj["last_error"] = False
+
+            # If there's text to skip
+            # @todo we could abstract out the get_text() to handle this cleaner
+            text_to_ignore = watch.get('ignore_text', []) + self.datastore.data['settings']['application'].get('global_ignore_text', [])
+            if len(text_to_ignore):
+                stripped_text_from_html = self.strip_ignore_text(stripped_text_from_html, text_to_ignore)
+            else:
+                stripped_text_from_html = stripped_text_from_html.encode('utf8')
+
+            # Re #133 - if we should strip whitespaces from triggering the change detected comparison
+            if self.datastore.data['settings']['application'].get('ignore_whitespace', False):
+                fetched_md5 = hashlib.md5(stripped_text_from_html.translate(None, b'\r\n\t ')).hexdigest()
+            else:
+                fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest()
+
+            # On the first run of a site, watch['previous_md5'] will be an empty string, set it the current one.
+            if not len(watch['previous_md5']):
+                watch['previous_md5'] = fetched_md5
+                update_obj["previous_md5"] = fetched_md5
+
+            blocked_by_not_found_trigger_text = False
+
+            if len(watch['trigger_text']):
+                blocked_by_not_found_trigger_text = True
+                for line in watch['trigger_text']:
+                    # Because JSON wont serialize a re.compile object
+                    if line[0] == '/' and line[-1] == '/':
+                        regex = re.compile(line.strip('/'), re.IGNORECASE)
+                        # Found it? so we don't wait for it anymore
+                        r = re.search(regex, str(stripped_text_from_html))
+                        if r:
+                            blocked_by_not_found_trigger_text = False
+                            break
+
+                    elif line.lower() in str(stripped_text_from_html).lower():
+                        # We found it don't wait for it.
+                        blocked_by_not_found_trigger_text = False
+                        break
+
+
+
+            if not blocked_by_not_found_trigger_text and watch['previous_md5'] != fetched_md5:
+                changed_detected = True
+                update_obj["previous_md5"] = fetched_md5
+                update_obj["last_changed"] = timestamp
+
+
+            # Extract title as title
+            if is_html:
+                if self.datastore.data['settings']['application']['extract_title_as_title'] or watch['extract_title_as_title']:
+                    if not watch['title'] or not len(watch['title']):
+                        update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
+
+
+        return changed_detected, update_obj, text_content_before_ignored_filter

changedetectionio/forms.py

@@ -0,0 +1,314 @@
+from wtforms import Form, SelectField, RadioField, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \
+    Field
+from wtforms import widgets
+from wtforms.validators import ValidationError
+from wtforms.fields import html5
+from changedetectionio import content_fetcher
+import re
+
+from changedetectionio.notification import default_notification_format, valid_notification_formats, default_notification_body, default_notification_title
+
+valid_method = {
+    'GET',
+    'POST',
+    'PUT',
+    'PATCH',
+    'DELETE',
+}
+
+default_method = 'GET'
+
+class StringListField(StringField):
+    widget = widgets.TextArea()
+
+    def _value(self):
+        if self.data:
+            return "\r\n".join(self.data)
+        else:
+            return u''
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            # Remove empty strings
+            cleaned = list(filter(None, valuelist[0].split("\n")))
+            self.data = [x.strip() for x in cleaned]
+            p = 1
+        else:
+            self.data = []
+
+
+
+class SaltyPasswordField(StringField):
+    widget = widgets.PasswordInput()
+    encrypted_password = ""
+
+    def build_password(self, password):
+        import hashlib
+        import base64
+        import secrets
+
+        # Make a new salt on every new password and store it with the password
+        salt = secrets.token_bytes(32)
+
+        key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
+        store = base64.b64encode(salt + key).decode('ascii')
+
+        return store
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            # Be really sure it's non-zero in length
+            if len(valuelist[0].strip()) > 0:
+                self.encrypted_password = self.build_password(valuelist[0])
+                self.data = ""
+        else:
+            self.data = False
+
+
+# Separated by  key:value
+class StringDictKeyValue(StringField):
+    widget = widgets.TextArea()
+
+    def _value(self):
+        if self.data:
+            output = u''
+            for k in self.data.keys():
+                output += "{}: {}\r\n".format(k, self.data[k])
+
+            return output
+        else:
+            return u''
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            self.data = {}
+            # Remove empty strings
+            cleaned = list(filter(None, valuelist[0].split("\n")))
+            for s in cleaned:
+                parts = s.strip().split(':', 1)
+                if len(parts) == 2:
+                    self.data.update({parts[0].strip(): parts[1].strip()})
+
+        else:
+            self.data = {}
+
+class ValidateContentFetcherIsReady(object):
+    """
+    Validates that anything that looks like a regex passes as a regex
+    """
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        from changedetectionio import content_fetcher
+        import urllib3.exceptions
+
+        # Better would be a radiohandler that keeps a reference to each class
+        if field.data is not None:
+            klass = getattr(content_fetcher, field.data)
+            some_object = klass()
+            try:
+                ready = some_object.is_ready()
+
+            except urllib3.exceptions.MaxRetryError as e:
+                driver_url = some_object.command_executor
+                message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data))
+                message += '<br/>' + field.gettext(
+                    'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.')
+                message += '<br/>' + field.gettext('Did you follow the instructions in the wiki?')
+                message += '<br/><br/>' + field.gettext('WebDriver Host: %s' % (driver_url))
+                message += '<br/><a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">Go here for more information</a>'
+                message += '<br/>'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e)))
+
+                raise ValidationError(message)
+
+            except Exception as e:
+                message = field.gettext('Content fetcher \'%s\' did not respond properly, unable to use it.\n %s')
+                raise ValidationError(message % (field.data, e))
+
+
+class ValidateNotificationBodyAndTitleWhenURLisSet(object):
+    """
+       Validates that they entered something in both notification title+body when the URL is set
+       Due to https://github.com/dgtlmoon/changedetection.io/issues/360
+       """
+
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        if len(field.data):
+            if not len(form.notification_title.data) or not len(form.notification_body.data):
+                message = field.gettext('Notification Body and Title is required when a Notification URL is used')
+                raise ValidationError(message)
+
+class ValidateAppRiseServers(object):
+    """
+       Validates that each URL given is compatible with AppRise
+       """
+
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        import apprise
+        apobj = apprise.Apprise()
+
+        for server_url in field.data:
+            if not apobj.add(server_url):
+                message = field.gettext('\'%s\' is not a valid AppRise URL.' % (server_url))
+                raise ValidationError(message)
+
+class ValidateTokensList(object):
+    """
+    Validates that a {token} is from a valid set
+    """
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        from changedetectionio import notification
+        regex = re.compile('{.*?}')
+        for p in re.findall(regex, field.data):
+            if not p.strip('{}') in notification.valid_tokens:
+                message = field.gettext('Token \'%s\' is not a valid token.')
+                raise ValidationError(message % (p))
+            
+class validateURL(object):
+    
+    """
+       Flask wtform validators wont work with basic auth
+    """
+
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        import validators
+        try:
+            validators.url(field.data.strip())
+        except validators.ValidationFailure:
+            message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
+            raise ValidationError(message)
+        
+class ValidateListRegex(object):
+    """
+    Validates that anything that looks like a regex passes as a regex
+    """
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+
+        for line in field.data:
+            if line[0] == '/' and line[-1] == '/':
+                # Because internally we dont wrap in /
+                line = line.strip('/')
+                try:
+                    re.compile(line)
+                except re.error:
+                    message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
+                    raise ValidationError(message % (line))
+
+class ValidateCSSJSONXPATHInput(object):
+    """
+    Filter validation
+    @todo CSS validator ;)
+    """
+
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+
+        # Nothing to see here
+        if not len(field.data.strip()):
+            return
+
+        # Does it look like XPath?
+        if field.data.strip()[0] == '/':
+            from lxml import html, etree
+            tree = html.fromstring("<html></html>")
+
+            try:
+                tree.xpath(field.data.strip())
+            except etree.XPathEvalError as e:
+                message = field.gettext('\'%s\' is not a valid XPath expression. (%s)')
+                raise ValidationError(message % (field.data, str(e)))
+            except:
+                raise ValidationError("A system-error occurred when validating your XPath expression")
+
+        if 'json:' in field.data:
+            from jsonpath_ng.exceptions import JsonPathParserError, JsonPathLexerError
+            from jsonpath_ng.ext import parse
+
+            input = field.data.replace('json:', '')
+
+            try:
+                parse(input)
+            except (JsonPathParserError, JsonPathLexerError) as e:
+                message = field.gettext('\'%s\' is not a valid JSONPath expression. (%s)')
+                raise ValidationError(message % (input, str(e)))
+            except:
+                raise ValidationError("A system-error occurred when validating your JSONPath expression")
+
+            # Re #265 - maybe in the future fetch the page and offer a
+            # warning/notice that its possible the rule doesnt yet match anything?
+
+class quickWatchForm(Form):
+    # https://wtforms.readthedocs.io/en/2.3.x/fields/#module-wtforms.fields.html5
+    # `require_tld` = False is needed even for the test harness "http://localhost:5005.." to run
+    url = html5.URLField('URL', validators=[validateURL()])
+    tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)])
+
+class commonSettingsForm(Form):
+
+    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateNotificationBodyAndTitleWhenURLisSet(), ValidateAppRiseServers()])
+    notification_title = StringField('Notification Title', default=default_notification_title, validators=[validators.Optional(), ValidateTokensList()])
+    notification_body = TextAreaField('Notification Body', default=default_notification_body, validators=[validators.Optional(), ValidateTokensList()])
+    notification_format = SelectField('Notification Format', choices=valid_notification_formats.keys(), default=default_notification_format)
+    trigger_check = BooleanField('Send test notification on save')
+    fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
+
+class watchForm(commonSettingsForm):
+
+    url = html5.URLField('URL', validators=[validateURL()])
+    tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)])
+
+    minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
+                                               [validators.Optional(), validators.NumberRange(min=1)])
+    css_filter = StringField('CSS/JSON/XPATH Filter', [ValidateCSSJSONXPATHInput()])
+    title = StringField('Title')
+
+    ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
+    headers = StringDictKeyValue('Request Headers')
+    body = TextAreaField('Request Body', [validators.Optional()])
+    method = SelectField('Request Method', choices=valid_method, default=default_method)
+    trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
+
+    def validate(self, **kwargs):
+        if not super().validate():
+            return False
+
+        result = True
+
+        # Fail form validation when a body is set for a GET
+        if self.method.data == 'GET' and self.body.data:
+            self.body.errors.append('Body must be empty when Request Method is set to GET')
+            result = False
+
+        return result
+
+class globalSettingsForm(commonSettingsForm):
+
+    password = SaltyPasswordField()
+    minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
+                                               [validators.NumberRange(min=1)])
+    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title')
+    base_url = StringField('Base URL', validators=[validators.Optional()])
+    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
+    ignore_whitespace = BooleanField('Ignore whitespace')

changedetectionio/html_tools.py

@@ -0,0 +1,107 @@
+import json
+from bs4 import BeautifulSoup
+from jsonpath_ng.ext import parse
+
+
+class JSONNotFound(ValueError):
+    def __init__(self, msg):
+        ValueError.__init__(self, msg)
+
+# Given a CSS Rule, and a blob of HTML, return the blob of HTML that matches
+def css_filter(css_filter, html_content):
+    soup = BeautifulSoup(html_content, "html.parser")
+    html_block = ""
+    for item in soup.select(css_filter, separator=""):
+        html_block += str(item)
+
+    return html_block + "\n"
+
+
+# Return str Utf-8 of matched rules
+def xpath_filter(xpath_filter, html_content):
+    from lxml import html
+    from lxml import etree
+
+    tree = html.fromstring(html_content)
+    html_block = ""
+
+    for item in tree.xpath(xpath_filter.strip()):
+        html_block+= etree.tostring(item, pretty_print=True).decode('utf-8')+"<br/>"
+
+    return html_block
+
+
+# Extract/find element
+def extract_element(find='title', html_content=''):
+
+    #Re #106, be sure to handle when its not found
+    element_text = None
+
+    soup = BeautifulSoup(html_content, 'html.parser')
+    result = soup.find(find)
+    if result and result.string:
+        element_text = result.string.strip()
+
+    return element_text
+
+#
+def _parse_json(json_data, jsonpath_filter):
+    s=[]
+    jsonpath_expression = parse(jsonpath_filter.replace('json:', ''))
+    match = jsonpath_expression.find(json_data)
+
+    # More than one result, we will return it as a JSON list.
+    if len(match) > 1:
+        for i in match:
+            s.append(i.value)
+
+    # Single value, use just the value, as it could be later used in a token in notifications.
+    if len(match) == 1:
+        s = match[0].value
+
+    # Re #257 - Better handling where it does not exist, in the case the original 's' value was False..
+    if not match:
+        # Re 265 - Just return an empty string when filter not found
+        return ''
+
+    stripped_text_from_html = json.dumps(s, indent=4)
+
+    return stripped_text_from_html
+
+def extract_json_as_string(content, jsonpath_filter):
+
+    stripped_text_from_html = False
+
+    # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded <script type=ldjson>
+    try:
+        stripped_text_from_html = _parse_json(json.loads(content), jsonpath_filter)
+    except json.JSONDecodeError:
+
+        # Foreach <script json></script> blob.. just return the first that matches jsonpath_filter
+        s = []
+        soup = BeautifulSoup(content, 'html.parser')
+        bs_result = soup.findAll('script')
+
+        if not bs_result:
+            raise JSONNotFound("No parsable JSON found in this document")
+
+        for result in bs_result:
+            # Skip empty tags, and things that dont even look like JSON
+            if not result.string or not '{' in result.string:
+                continue
+                
+            try:
+                json_data = json.loads(result.string)
+            except json.JSONDecodeError:
+                # Just skip it
+                continue
+            else:
+                stripped_text_from_html = _parse_json(json_data, jsonpath_filter)
+                if stripped_text_from_html:
+                    break
+
+    if not stripped_text_from_html:
+        # Re 265 - Just return an empty string when filter not found
+        return ''
+
+    return stripped_text_from_html

changedetectionio/notification.py

@@ -0,0 +1,107 @@
+import apprise
+from apprise import NotifyFormat
+
+valid_tokens = {
+    'base_url': '',
+    'watch_url': '',
+    'watch_uuid': '',
+    'watch_title': '',
+    'watch_tag': '',
+    'diff': '',
+    'diff_full': '',
+    'diff_url': '',
+    'preview_url': '',
+    'current_snapshot': ''
+}
+
+valid_notification_formats = {
+    'Text': NotifyFormat.TEXT,
+    'Markdown': NotifyFormat.MARKDOWN,
+    'HTML': NotifyFormat.HTML,
+}
+
+default_notification_format = 'Text'
+default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n'
+default_notification_title = 'ChangeDetection.io Notification - {watch_url}'
+
+def process_notification(n_object, datastore):
+    import logging
+    log = logging.getLogger('apprise')
+    log.setLevel('TRACE')
+    apobj = apprise.Apprise(debug=True)
+
+    for url in n_object['notification_urls']:
+        url = url.strip()
+        print (">> Process Notification: AppRise notifying {}".format(url))
+        apobj.add(url)
+
+    # Get the notification body from datastore
+    n_body = n_object.get('notification_body', default_notification_body)
+    n_title = n_object.get('notification_title', default_notification_title)
+    n_format = valid_notification_formats.get(
+        n_object['notification_format'],
+        valid_notification_formats[default_notification_format],
+    )
+
+
+    # Insert variables into the notification content
+    notification_parameters = create_notification_parameters(n_object, datastore)
+
+    for n_k in notification_parameters:
+        token = '{' + n_k + '}'
+        val = notification_parameters[n_k]
+        n_title = n_title.replace(token, val)
+        n_body = n_body.replace(token, val)
+
+    apobj.notify(
+        body=n_body,
+        title=n_title,
+        body_format=n_format,
+    )
+
+# Notification title + body content parameters get created here.
+def create_notification_parameters(n_object, datastore):
+    from copy import deepcopy
+
+    # in the case we send a test notification from the main settings, there is no UUID.
+    uuid = n_object['uuid'] if 'uuid' in n_object else ''
+
+    if uuid != '':
+        watch_title = datastore.data['watching'][uuid]['title']
+        watch_tag = datastore.data['watching'][uuid]['tag']
+    else:
+        watch_title = 'Change Detection'
+        watch_tag = ''
+
+    # Create URLs to customise the notification with
+    base_url = datastore.data['settings']['application']['base_url']
+
+    watch_url = n_object['watch_url']
+
+    # Re #148 - Some people have just {base_url} in the body or title, but this may break some notification services
+    #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.
+    if base_url == '':
+        base_url = "<base-url-env-var-not-set>"
+
+    diff_url = "{}/diff/{}".format(base_url, uuid)
+    preview_url = "{}/preview/{}".format(base_url, uuid)
+
+    # Not sure deepcopy is needed here, but why not
+    tokens = deepcopy(valid_tokens)
+
+    # Valid_tokens also used as a field validator
+    tokens.update(
+        {
+            'base_url': base_url if base_url is not None else '',
+            'watch_url': watch_url,
+            'watch_uuid': uuid,
+            'watch_title': watch_title if watch_title is not None else '',
+            'watch_tag': watch_tag if watch_tag is not None else '',
+            'diff_url': diff_url,
+            'diff': n_object.get('diff', ''),  # Null default in the case we use a test
+            'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
+            'preview_url': preview_url,
+            'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
+        })
+
+    return tokens

changedetectionio/pytest.ini

@@ -1,7 +1,7 @@
 [pytest]
 addopts = --no-start-live-server --live-server-port=5005
 #testpaths = tests pytest_invenio
-#live_server_scope = session
+#live_server_scope = function
 
 filterwarnings =
     ignore::DeprecationWarning:urllib3.*:

changedetectionio/run_all_tests.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+
+# live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions
+# and I like to restart the server for each test (and have the test cleanup after each test)
+# merge request welcome :)
+
+
+# exit when any command fails
+set -e
+
+find tests/test_*py -type f|while read test_name
+do
+  echo "TEST RUNNING $test_name"
+  pytest $test_name
+done
+
+echo "RUNNING WITH BASE_URL SET"
+
+# Now re-run some tests with BASE_URL enabled
+# Re #65 - Ability to include a link back to the installation, in the notification.
+export BASE_URL="https://really-unique-domain.io"
+pytest tests/test_notification.py
+

changedetectionio/static/js/settings.js

@@ -0,0 +1,13 @@
+window.addEventListener("load", (event) => {
+  // just an example for now
+  function toggleVisible(elem) {
+    // theres better ways todo this
+    var x = document.getElementById(elem);
+    if (x.style.display === "block") {
+      x.style.display = "none";
+    } else {
+      x.style.display = "block";
+    }
+  }
+});
+

changedetectionio/static/js/tabs.js

@@ -0,0 +1,51 @@
+// Rewrite this is a plugin.. is all this JS really 'worth it?'
+
+
+window.addEventListener('hashchange', function() {
+  var tabs = document.getElementsByClassName('active');
+  while (tabs[0]) {
+    tabs[0].classList.remove('active')
+  }
+  set_active_tab();
+}, false);
+
+var has_errors=document.querySelectorAll(".messages .error");
+if (!has_errors.length) {
+    if (document.location.hash == "" ) {
+        document.location.hash = "#general";
+        document.getElementById("default-tab").className = "active";
+    } else {
+        set_active_tab();
+    }
+} else {
+  focus_error_tab();
+}
+
+
+function set_active_tab() {
+  var tab=document.querySelectorAll("a[href='"+location.hash+"']");
+  if (tab.length) {
+    tab[0].parentElement.className="active";
+  }
+    // hash could move the page down
+    window.scrollTo(0, 0);
+}
+
+function focus_error_tab() {
+  // time to use jquery or vuejs really,
+  // activate the tab with the error
+    var tabs = document.querySelectorAll('.tabs li a'),i;
+    for (i = 0; i < tabs.length; ++i) {
+      var tab_name=tabs[i].hash.replace('#','');
+      var pane_errors=document.querySelectorAll('#'+tab_name+' .error')
+      if (pane_errors.length) {
+        document.location.hash = '#'+tab_name;
+        return true;
+      }
+    }
+    return false;
+}
+
+
+
+

changedetectionio/static/styles/.gitignore

@@ -0,0 +1 @@
+node_modules

changedetectionio/static/styles/diff.css

@@ -0,0 +1,56 @@
+#diff-ui {
+  background: #fff;
+  padding: 2em;
+  margin: 1em;
+  border-radius: 5px;
+  font-size: 11px; }
+  #diff-ui table {
+    table-layout: fixed;
+    width: 100%; }
+  #diff-ui td {
+    padding: 3px 4px;
+    border: 1px solid transparent;
+    vertical-align: top;
+    font: 1em monospace;
+    text-align: left; }
+  #diff-ui pre {
+    white-space: pre-wrap; }
+
+h1 {
+  display: inline;
+  font-size: 100%; }
+
+del {
+  text-decoration: none;
+  color: #b30000;
+  background: #fadad7; }
+
+ins {
+  background: #eaf2c2;
+  color: #406619;
+  text-decoration: none; }
+
+#result {
+  white-space: pre-wrap; }
+
+#settings {
+  background: rgba(0, 0, 0, 0.05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+  color: #fff;
+  font-size: 80%; }
+  #settings label {
+    margin-left: 1em;
+    display: inline-block;
+    font-weight: normal; }
+
+.source {
+  position: absolute;
+  right: 1%;
+  top: .2em; }
+
+@-moz-document url-prefix() {
+  body {
+    height: 99%;
+    /* Hide scroll bar in Firefox */ } }

changedetectionio/static/styles/diff.scss

@@ -1,15 +1,25 @@
-table {
-	table-layout: fixed;
-	width: 100%;
-}
-td {
-	width: 33%;
-	padding: 3px 4px;
-	border: 1px solid transparent;
-	vertical-align: top;
-	font: 1em monospace;
-	text-align: left;
-	white-space: pre-wrap;
+#diff-ui {
+
+    background: #fff;
+    padding: 2em;
+    margin: 1em;
+    border-radius: 5px;
+    font-size: 11px;
+
+    table {
+        table-layout: fixed;
+        width: 100%;
+    }
+    td {
+        padding: 3px 4px;
+        border: 1px solid transparent;
+        vertical-align: top;
+        font: 1em monospace;
+        text-align: left;
+    }
+    pre {
+            white-space: pre-wrap;
+    }
 }
 h1 {
 	display: inline;
@@ -33,16 +43,16 @@ ins {
 
 #settings {
     background: rgba(0,0,0,.05);
-        padding: 1em;
+    padding: 1em;
     border-radius: 10px;
     margin-bottom: 1em;
     color: #fff;
     font-size: 80%;
-}
-#settings label {
-	margin-left: 1em;
-	display: inline-block;
-	font-weight: normal;
+    label {
+	    margin-left: 1em;
+	    display: inline-block;
+	    font-weight: normal;
+    }
 }
 
 .source {
@@ -56,11 +66,3 @@ ins {
 		height: 99%; /* Hide scroll bar in Firefox */
 	}
 }
-
-#diff-ui {
-    background: #fff;
-    padding: 2em;
-    margin: 1em;
-    border-radius: 5px;
-    font-size: 9px;
-}

changedetectionio/static/styles/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "changedetection.io-theme",
+  "version": "0.0.3",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "build": "node-sass styles.scss diff.scss -o .",
+    "watch": "node-sass --watch styles.scss diff.scss -o ."
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "node-sass": "^6.0.1",
+    "tar": "^6.1.9",
+    "trim-newlines": "^3.0.1"
+  }
+}

changedetectionio/static/styles/styles.scss

@@ -0,0 +1,569 @@
+/*
+ * -- BASE STYLES --
+ * Most of these are inherited from Base, but I want to change a few.
+ * nvm use v14.18.1
+ * npm install
+ * npm run build
+ * or npm run watch
+ */
+body {
+  color: #333;
+  background: #262626;
+}
+.pure-table-even {
+  background: #fff;
+}
+
+/* Some styles from https://css-tricks.com/ */
+a {
+  text-decoration: none;
+  color: #1b98f8;
+}
+
+a.github-link {
+  color: #fff;
+}
+
+.pure-menu-horizontal {
+  background: #fff;
+  padding: 5px;
+  display: flex;
+  justify-content: space-between;
+  border-bottom: 2px solid #ed5900;
+  align-items: center;
+}
+
+section.content {
+  padding-top: 5em;
+  padding-bottom: 5em;
+  flex-direction: column;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+/* table related */
+.watch-table {
+  width: 100%;
+
+  tr.unviewed {
+    font-weight: bold;
+  }
+
+  .error {
+    color: #a00;
+  }
+
+  td {
+    font-size: 80%;
+    white-space: nowrap;
+  }
+
+  td.title-col {
+    word-break: break-all;
+    white-space: normal;
+  }
+
+  th {
+    white-space: nowrap;
+  }
+
+  .title-col a[target="_blank"]::after, .current-diff-url::after {
+    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+    margin: 0 3px 0 5px;
+  }
+}
+
+.watch-tag-list {
+  color: #e70069;
+  white-space: nowrap;
+}
+
+.box {
+  max-width: 80%;
+  flex-direction: column;
+  display: flex;
+  justify-content: center;
+}
+
+
+#post-list-buttons {
+  text-align: right;
+  padding: 0px;
+  margin: 0px;
+
+  li {
+    display: inline-block;
+  }
+
+  a {
+    border-top-left-radius: initial;
+    border-top-right-radius: initial;
+    border-bottom-left-radius: 5px;
+    border-bottom-right-radius: 5px;
+  }
+}
+
+
+body:after {
+  content: "";
+  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)
+}
+
+body:after, body:before {
+  display: block;
+  height: 600px;
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  z-index: -1;
+}
+
+body::after {
+  opacity: 0.91;
+}
+
+body::before {
+  content: "";
+  background-image: url(/static/images/gradient-border.png);
+}
+
+body:before {
+  background-size: cover
+}
+
+body:after, body:before {
+  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
+  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
+}
+
+.arrow {
+  border: solid black;
+  border-width: 0 3px 3px 0;
+  display: inline-block;
+  padding: 3px;
+    &.right {
+      transform: rotate(-45deg);
+      -webkit-transform: rotate(-45deg);
+    }
+    &.left {
+      transform: rotate(135deg);
+      -webkit-transform: rotate(135deg);
+    }
+    &.up {
+      transform: rotate(-135deg);
+      -webkit-transform: rotate(-135deg);
+    }
+    &.down {
+      transform: rotate(45deg);
+      -webkit-transform: rotate(45deg);
+    }
+}
+
+.button-small {
+  font-size: 85%;
+}
+
+.fetch-error {
+  padding-top: 1em;
+  font-size: 60%;
+  max-width: 400px;
+  display: block;
+}
+
+
+.button-secondary {
+  color: white;
+  border-radius: 4px;
+  text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
+}
+
+.button-success {
+  background: rgb(28, 184, 65);
+  /* this is a green */
+}
+
+.button-tag {
+  background: rgb(99, 99, 99);
+  color: #fff;
+  font-size: 65%;
+  border-bottom-left-radius: initial;
+  border-bottom-right-radius: initial;
+
+  &.active {
+    background: #9c9c9c;
+    font-weight: bold;
+  }
+
+}
+
+.button-error {
+  background: rgb(202, 60, 60);
+  /* this is a maroon */
+}
+
+.button-warning {
+  background: rgb(223, 117, 20);
+  /* this is an orange */
+}
+
+.button-secondary {
+  background: rgb(66, 184, 221);
+  /* this is a light blue */
+}
+
+
+.button-cancel {
+  background: rgb(200, 200, 200);
+  /* this is a green */
+}
+
+.messages {
+    li {
+        list-style: none;
+        padding: 1em;
+        border-radius: 10px;
+        color: #fff;
+        font-weight: bold;
+        &.message {
+            background: rgba(255, 255, 255, .2);
+        }
+        &.error {
+            background: rgba(255, 1, 1, .5);
+        }
+        &.notice {
+            background: rgba(255, 255, 255, .5);
+        }
+    }
+}
+
+#notification-customisation {
+    border: 1px solid #ccc;
+    padding: 1rem;
+    border-radius: 5px;
+}
+
+
+#token-table {
+    &.pure-table td, &.pure-table th {
+        font-size: 80%;
+    }
+}
+
+#new-watch-form {
+  background: rgba(0, 0, 0, .05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+  input {
+    width: auto !important;
+    display: inline-block;
+  }
+  .label {
+    display: none;
+  }
+  legend {
+    color: #fff;
+  }
+}
+
+
+
+
+#diff-col {
+  padding-left: 40px;
+}
+
+#diff-jump {
+  position: fixed;
+  left: 0px;
+  top: 120px;
+  background: #fff;
+  padding: 10px;
+  border-top-right-radius: 5px;
+  border-bottom-right-radius: 5px;
+  box-shadow: 5px 0 5px -2px #888;
+     a {
+      color: #1b98f8;
+      cursor: grabbing;
+      -moz-user-select: none;
+      -webkit-user-select: none;
+      -ms-user-select: none;
+      user-select: none;
+      -o-user-select: none;
+    }
+}
+
+footer {
+  padding: 10px;
+  background: #fff;
+  color: #444;
+  text-align: center;
+}
+
+#feed-icon {
+  vertical-align: middle;
+}
+
+#top-right-menu {
+// Just let flex overflow the x axis for now
+/*
+    position: absolute;
+    right: 0px;
+    background: linear-gradient(to right, #fff0, #fff 10%);
+    padding-left: 20px;
+    padding-right: 10px;
+    */
+}
+
+
+
+.sticky-tab {
+  position: absolute;
+  top: 80px;
+  font-size: 8px;
+  background: #fff;
+  padding: 10px;
+  &#left-sticky {
+    left: 0px;
+  }
+  &#right-sticky {
+    right: 0px;
+  }
+}
+
+#new-version-text a {
+  color: #e07171;
+}
+
+.paused-state {
+  &.state-False img {
+    opacity: 0.2;
+  }
+
+  &.state-False:hover img {
+    opacity: 0.8;
+  }
+}
+
+.monospaced-textarea {
+    textarea {
+        width: 100%;
+        font-family: monospace;
+        white-space: pre;
+        overflow-wrap: normal;
+        overflow-x: scroll;
+    }
+}
+
+
+.pure-form {
+    .pure-control-group, .pure-group, .pure-controls {
+        padding-bottom: 1em;
+        div {
+            margin: 0px;
+        }
+    }
+  /* The input fields with errors */
+  .error {
+    input {
+        background-color: #ffebeb;
+    }
+  }
+
+  /* The list of errors */
+  ul.errors {
+    padding: .5em .6em;
+    border: 1px solid #dd0000;
+    border-radius: 4px;
+    vertical-align: middle;
+    -webkit-box-sizing: border-box;
+    box-sizing: border-box;
+    li {
+        margin-left: 1em;
+        color: #dd0000;
+    }
+  }
+
+  label {
+    font-weight: bold;
+  }
+
+  textarea {
+    width: 100%;
+  }
+  ul#fetch_backend {
+    margin: 0px;
+    list-style: none;
+    > li {
+        > * {
+            display: inline-block;
+        }
+    }
+  }
+}
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+  .box {
+    max-width: 95%
+  }
+  .edit-form {
+    padding: 0.5em;
+    margin: 0;
+  }
+  #nav-menu {
+    overflow-x: scroll;
+  }
+}
+
+/*
+Max width before this PARTICULAR table gets nasty
+This query will take effect for any screen smaller than 760px
+and also iPads specifically.
+*/
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+
+  input[type='text'] {
+    width: 100%;
+  }
+    
+  .watch-table {
+    /* Force table to not be like tables anymore */
+    thead, tbody, th, td, tr {
+      display: block;
+    }
+
+    .last-checked::before {
+      color: #555;
+      content: "Last Checked ";
+    }
+
+    .last-changed::before {
+      color: #555;
+      content: "Last Changed ";
+    }
+
+    /* Force table to not be like tables anymore */
+    td.inline {
+      display: inline-block;
+    }
+
+    /* Hide table headers (but not display: none;, for accessibility) */
+    thead tr {
+      position: absolute;
+      top: -9999px;
+      left: -9999px;
+    }
+
+    .pure-table td, .pure-table th {
+      border: none;
+    }
+
+    td {
+      /* Behave  like a "row" */
+      border: none;
+      border-bottom: 1px solid #eee;
+
+      &:before {
+        /* Top/left values mimic padding */
+        top: 6px;
+        left: 6px;
+        width: 45%;
+        padding-right: 10px;
+        white-space: nowrap;
+      }
+    }
+
+    &.pure-table-striped {
+      tr {
+        background-color: #fff;
+      }
+
+      tr:nth-child(2n-1) {
+        background-color: #eee;
+      }
+
+      tr:nth-child(2n-1) td {
+        background-color: inherit;
+      }
+    }
+
+  }
+}
+
+
+/** Desktop vs mobile input field strategy
+- We dont use 'size' with <input> because `size` is too unreliable to override, and will often push-out
+- Rely always on width in CSS
+*/
+@media only screen and (min-width: 761px) {
+/* m-d is medium-desktop */
+    .m-d {
+        min-width: 80%;
+    }
+}
+
+
+
+.tabs {
+  ul {
+    margin: 0px;
+    padding: 0px;
+    display:block;
+    li {
+      margin-right: 3px;
+      display: inline-block;
+      color: #fff;
+      border-top-left-radius: 5px;
+      border-top-right-radius: 5px;
+      background-color: rgba(255, 255, 255, 0.2);
+
+      &.active,:target {
+        background-color: #fff;
+        a {
+          color: #222;
+          font-weight: bold;
+        }
+      }
+      a {
+        display: block;
+        padding: 0.8em;
+        color: #fff;
+      }
+    }
+  }
+}
+
+$form-edge-padding: 20px;
+.pure-form-stacked {
+  >div:first-child {
+    display: block;
+  }
+}
+.edit-form {
+  min-width: 70%;
+  .tab-pane-inner {
+    &:not(:target) {
+        display: none;
+    }
+    &:target {
+      display: block;
+    }
+    // doesnt need padding because theres another row of buttons/activity
+    padding: 0px;
+  }
+  .box-wrap {
+    position: relative;
+  }
+  .inner {
+    background: #fff;;
+    padding: $form-edge-padding;
+  }
+  #actions {
+    display: block;
+    background: #fff;
+  }
+}
+

changedetectionio/store.py

@@ -1,15 +1,15 @@
+from os import unlink, path, mkdir
 import json
 import uuid as uuid_builder
-import os.path
-from os import path
 from threading import Lock
-
 from copy import deepcopy
 
 import logging
 import time
 import threading
+import os
 
+from changedetectionio.notification import default_notification_format, default_notification_body, default_notification_title
 
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
@@ -17,7 +17,9 @@ import threading
 class ChangeDetectionStore:
     lock = Lock()
 
-    def __init__(self, datastore_path="/datastore", include_default_watches=True):
+    def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
+        # Should only be active for docker
+        # logging.basicConfig(filename='/dev/stdout', level=logging.INFO)
         self.needs_write = False
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
@@ -39,7 +41,17 @@ class ChangeDetectionStore:
                     'workers': 10  # Number of threads, lower is better for slow connections
                 },
                 'application': {
-                    'password': False
+                    'password': False,
+                    'base_url' : None,
+                    'extract_title_as_title': False,
+                    'fetch_backend': 'html_requests',
+                    'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
+                    'ignore_whitespace': False,
+                    'notification_urls': [], # Apprise URL list
+                    # Custom notification content
+                    'notification_title': default_notification_title,
+                    'notification_body': default_notification_body,
+                    'notification_format': default_notification_format,
                 }
             }
         }
@@ -54,15 +66,29 @@ class ChangeDetectionStore:
             'last_viewed': 0,  # history key value of the last viewed via the [diff] link
             'newest_history_key': "",
             'title': None,
+            # Re #110, so then if this is set to None, we know to use the default value instead
+            # Requires setting to None on submit if it's the same as the default
+            'minutes_between_check': None,
             'previous_md5': "",
             'uuid': str(uuid_builder.uuid4()),
             'headers': {},  # Extra headers to send
+            'body': None,
+            'method': 'GET',
             'history': {},  # Dict of timestamp and output stripped filename
-            'ignore_text': [] # List of text to ignore when calculating the comparison checksum
+            'ignore_text': [], # List of text to ignore when calculating the comparison checksum
+            # Custom notification content
+            'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise)
+            'notification_title': default_notification_title,
+            'notification_body': default_notification_body,
+            'notification_format': default_notification_format,
+            'css_filter': "",
+            'trigger_text': [],  # List of text or regex to wait for until a change is detected
+            'fetch_backend': None,
+            'extract_title_as_title': False
         }
 
-        if path.isfile('/source.txt'):
-            with open('/source.txt') as f:
+        if path.isfile('changedetectionio/source.txt'):
+            with open('changedetectionio/source.txt') as f:
                 # Should be set in Dockerfile to look for /source.txt , this will give us the git commit #
                 # So when someone gives us a backup file to examine, we know exactly what code they were running.
                 self.__data['build_sha'] = f.read()
@@ -109,11 +135,27 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
                 self.add_watch(url='https://changedetection.io', tag='Tech news')
 
+        self.__data['version_tag'] = version_tag
 
-        self.__data['version_tag'] = "0.28"
+        # Helper to remove password protection
+        password_reset_lockfile = "{}/removepassword.lock".format(self.datastore_path)
+        if path.isfile(password_reset_lockfile):
+            self.__data['settings']['application']['password'] = False
+            unlink(password_reset_lockfile)
 
         if not 'app_guid' in self.__data:
-            self.__data['app_guid'] = str(uuid_builder.uuid4())
+            import sys
+            import os
+            if "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ:
+                self.__data['app_guid'] = "test-" + str(uuid_builder.uuid4())
+            else:
+                self.__data['app_guid'] = str(uuid_builder.uuid4())
+
+        # Generate the URL access token for RSS feeds
+        if not 'rss_access_token' in self.__data['settings']['application']:
+            import secrets
+            secret = secrets.token_hex(16)
+            self.__data['settings']['application']['rss_access_token'] = secret
 
         self.needs_write = True
 
@@ -127,6 +169,7 @@ class ChangeDetectionStore:
 
         dates = list(self.__data['watching'][uuid]['history'].keys())
         # Convert to int, sort and back to str again
+        # @todo replace datastore getter that does this automatically
         dates = [int(i) for i in dates]
         dates.sort(reverse=True)
         if len(dates):
@@ -161,7 +204,6 @@ class ChangeDetectionStore:
 
     @property
     def data(self):
-
         has_unviewed = False
         for uuid, v in self.__data['watching'].items():
             self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
@@ -172,6 +214,16 @@ class ChangeDetectionStore:
                 self.__data['watching'][uuid]['viewed'] = False
                 has_unviewed = True
 
+            # #106 - Be sure this is None on empty string, False, None, etc
+            # Default var for fetch_backend
+            if not self.__data['watching'][uuid]['fetch_backend']:
+                self.__data['watching'][uuid]['fetch_backend'] = self.__data['settings']['application']['fetch_backend']
+
+        # Re #152, Return env base_url if not overriden, @todo also prefer the proxy pass url
+        env_base_url = os.getenv('BASE_URL','')
+        if not self.__data['settings']['application']['base_url']:
+          self.__data['settings']['application']['base_url'] = env_base_url.strip('" ')
+
         self.__data['has_unviewed'] = has_unviewed
 
         return self.__data
@@ -191,7 +243,7 @@ class ChangeDetectionStore:
 
     def unlink_history_file(self, path):
         try:
-            os.unlink(path)
+            unlink(path)
         except (FileNotFoundError, IOError):
             pass
 
@@ -214,6 +266,14 @@ class ChangeDetectionStore:
 
             self.needs_write = True
 
+    # Clone a watch by UUID
+    def clone(self, uuid):
+        url = self.data['watching'][uuid]['url']
+        tag = self.data['watching'][uuid]['tag']
+        extras = self.data['watching'][uuid]
+        new_uuid = self.add_watch(url=url, tag=tag, extras=extras)
+        return new_uuid
+
     def url_exists(self, url):
 
         # Probably their should be dict...
@@ -233,18 +293,20 @@ class ChangeDetectionStore:
         import hashlib
         del_timestamps = []
 
+        changes_removed = 0
+
         for timestamp, path in self.data['watching'][uuid]['history'].items():
             if not limit_timestamp or (limit_timestamp is not False and int(timestamp) > limit_timestamp):
                 self.unlink_history_file(path)
                 del_timestamps.append(timestamp)
+                changes_removed += 1
 
+        if not limit_timestamp:
+            self.data['watching'][uuid]['last_checked'] = 0
+            self.data['watching'][uuid]['last_changed'] = 0
+            self.data['watching'][uuid]['previous_md5'] = ""
 
 
-                if not limit_timestamp:
-                    self.data['watching'][uuid]['last_checked'] = 0
-                    self.data['watching'][uuid]['last_changed'] = 0
-                    self.data['watching'][uuid]['previous_md5'] = 0
-
         for timestamp in del_timestamps:
             del self.data['watching'][uuid]['history'][str(timestamp)]
 
@@ -261,30 +323,39 @@ class ChangeDetectionStore:
                             content = fp.read()
                         self.data['watching'][uuid]['previous_md5'] = hashlib.md5(content).hexdigest()
                     except (FileNotFoundError, IOError):
-                        self.data['watching'][uuid]['previous_md5'] = False
+                        self.data['watching'][uuid]['previous_md5'] = ""
                         pass
 
-
         self.needs_write = True
+        return changes_removed
 
+    def add_watch(self, url, tag, extras=None):
+        if extras is None:
+            extras = {}
 
-    def add_watch(self, url, tag):
         with self.lock:
             # @todo use a common generic version of this
             new_uuid = str(uuid_builder.uuid4())
             _blank = deepcopy(self.generic_definition)
             _blank.update({
                 'url': url,
-                'tag': tag,
-                'uuid': new_uuid
+                'tag': tag
             })
 
+            # Incase these are copied across, assume it's a reference and deepcopy()
+            apply_extras = deepcopy(extras)
+            for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
+                if k in apply_extras:
+                    del apply_extras[k]
+
+            _blank.update(apply_extras)
+
             self.data['watching'][new_uuid] = _blank
 
         # Get the directory ready
         output_path = "{}/{}".format(self.datastore_path, new_uuid)
         try:
-            os.mkdir(output_path)
+            mkdir(output_path)
         except FileExistsError:
             print(output_path, "already exists.")
 
@@ -293,35 +364,46 @@ class ChangeDetectionStore:
 
     # Save some text file to the appropriate path and bump the history
     # result_obj from fetch_site_status.run()
-    def save_history_text(self, uuid, result_obj, contents):
+    def save_history_text(self, watch_uuid, contents):
+        import uuid
 
-        output_path = "{}/{}".format(self.datastore_path, uuid)
-        fname = "{}/{}-{}.stripped.txt".format(output_path, result_obj['previous_md5'], str(time.time()))
-        with open(fname, 'w') as f:
+        output_path = "{}/{}".format(self.datastore_path, watch_uuid)
+        # Incase the operator deleted it, check and create.
+        if not os.path.isdir(output_path):
+            mkdir(output_path)
+
+        fname = "{}/{}.stripped.txt".format(output_path, uuid.uuid4())
+        with open(fname, 'wb') as f:
             f.write(contents)
             f.close()
 
-        # Update history with the stripped text for future reference, this will also mean we save the first
-        # Should always be keyed by string(timestamp)
-        self.update_watch(uuid, {"history": {str(result_obj["last_checked"]): fname}})
-
         return fname
 
     def sync_to_json(self):
-        print("Saving..")
-        data ={}
+        logging.info("Saving JSON..")
 
         try:
             data = deepcopy(self.__data)
-        except RuntimeError:
-            time.sleep(0.5)
-            print ("! Data changed when writing to JSON, trying again..")
+        except RuntimeError as e:
+            # Try again in 15 seconds
+            time.sleep(15)
+            logging.error ("! Data changed when writing to JSON, trying again.. %s", str(e))
             self.sync_to_json()
             return
         else:
-            with open(self.json_store_path, 'w') as json_file:
-                json.dump(data, json_file, indent=4)
-                logging.info("Re-saved index")
+
+            try:
+                # Re #286  - First write to a temp file, then confirm it looks OK and rename it
+                # This is a fairly basic strategy to deal with the case that the file is corrupted,
+                # system was out of memory, out of RAM etc
+                with open(self.json_store_path+".tmp", 'w') as json_file:
+                    json.dump(data, json_file, indent=4)
+
+            except Exception as e:
+                logging.error("Error writing JSON!! (Main JSON file save was skipped) : %s", str(e))
+
+            else:
+                os.rename(self.json_store_path+".tmp", self.json_store_path)
 
             self.needs_write = False
 
@@ -333,8 +415,30 @@ class ChangeDetectionStore:
             if self.stop_thread:
                 print("Shutting down datastore thread")
                 return
-            
+
             if self.needs_write:
                 self.sync_to_json()
-            time.sleep(3)
 
+            # Once per minute is enough, more and it can cause high CPU usage
+            # better here is to use something like self.app.config.exit.wait(1), but we cant get to 'app' from here
+            for i in range(30):
+                time.sleep(2)
+                if self.stop_thread:
+                    break
+
+    # Go through the datastore path and remove any snapshots that are not mentioned in the index
+    # This usually is not used, but can be handy.
+    def remove_unused_snapshots(self):
+        print ("Removing snapshots from datastore that are not in the index..")
+
+        index=[]
+        for uuid in self.data['watching']:
+            for id in self.data['watching'][uuid]['history']:
+                index.append(self.data['watching'][uuid]['history'][str(id)])
+
+        import pathlib
+        # Only in the sub-directories
+        for item in pathlib.Path(self.datastore_path).rglob("*/*txt"):
+            if not str(item) in index:
+                print ("Removing",item)
+                unlink(item)

changedetectionio/templates/_common_fields.jinja

@@ -0,0 +1,96 @@
+
+{% from '_helpers.jinja' import render_field %}
+
+{% macro render_common_settings_form(form, current_base_url) %}
+
+                        <div class="pure-control-group">
+                            {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
+    Gitter - gitter://token/room
+    Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+    AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
+    SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com")
+                            }}
+                            <div class="pure-form-message-inline">Use <a target=_new
+                                                                         href="https://github.com/caronc/apprise">AppRise
+                                URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>
+                            </div>
+                        </div>
+                        <div id="notification-customisation">
+                            <div class="pure-control-group">
+                                {{ render_field(form.notification_title, class="m-d") }}
+                                <span class="pure-form-message-inline">Title for all notifications</span>
+                            </div>
+                            <div class="pure-control-group">
+                                {{ render_field(form.notification_body , rows=5) }}
+                                <span class="pure-form-message-inline">Body for all notifications</span>
+                            </div>
+                            <div class="pure-control-group">
+                                {{ render_field(form.notification_format , rows=5) }}
+                                <span class="pure-form-message-inline">Format for all notifications</span>
+                            </div>
+                            <div class="pure-controls">
+                            <span class="pure-form-message-inline">
+                                These tokens can be used in the notification body and title to
+                                customise the notification text.
+                            </span>
+                                <table class="pure-table" id="token-table">
+                                    <thead>
+                                    <tr>
+                                        <th>Token</th>
+                                        <th>Description</th>
+                                    </tr>
+                                    </thead>
+                                    <tbody>
+                                    <tr>
+                                        <td><code>{base_url}</code></td>
+                                        <td>The URL of the changedetection.io instance you are running.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{watch_url}</code></td>
+                                        <td>The URL being watched.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{watch_uuid}</code></td>
+                                        <td>The UUID of the watch.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{watch_title}</code></td>
+                                        <td>The title of the watch.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{watch_tag}</code></td>
+                                        <td>The tag of the watch.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{preview_url}</code></td>
+                                        <td>The URL of the preview page generated by changedetection.io.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{diff}</code></td>
+                                        <td>The diff output - differences only</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{diff_full}</code></td>
+                                        <td>The diff output - full difference output</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{diff_url}</code></td>
+                                        <td>The URL of the diff page generated by changedetection.io.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{current_snapshot}</code></td>
+                                        <td>The current snapshot value, useful when combined with JSON or CSS filters
+                                        </td>
+                                    </tr>
+                                    </tbody>
+                                </table>
+                                <span class="pure-form-message-inline">
+                                URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
+                                Your <code>BASE_URL</code> var is currently "{{current_base_url}}"
+                            </span>
+                            </div>
+                        </div>
+                        <div class="pure-control-group">
+                            {{ render_field(form.trigger_check) }}
+                        </div>
+{% endmacro %}

changedetectionio/templates/_helpers.jinja

@@ -0,0 +1,27 @@
+{% macro render_field(field) %}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
+  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </div>
+{% endmacro %}
+
+{% macro render_simple_field(field) %}
+  <span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>
+  <span {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </span>
+{% endmacro %}
+
+

changedetectionio/templates/base.html

@@ -4,9 +4,9 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta name="description" content="Self hosted website change detection.">
-    <title>Change Detection</title>
-    <link rel="stylesheet" href="/static/css/pure-min.css">
-    <link rel="stylesheet" href="/static/css/styles.css?ver=1000">
+    <title>Change Detection{{extra_title}}</title>
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}">
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}">
     {% if extra_stylesheets %}
         {% for m in extra_stylesheets %}
         <link rel="stylesheet" href="{{ m }}?ver=1000">
@@ -16,40 +16,47 @@
 <body>
 
 <div class="header">
-    <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed">
-        {% if not current_user.is_authenticated %}
+
+    <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
+        {% if has_password and not current_user.is_authenticated %}
             <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener"><strong>Change</strong>Detection.io</a>
         {% else %}
-            <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
+            <a class="pure-menu-heading" href="{{url_for('index')}}"><strong>Change</strong>Detection.io</a>
         {% endif %}
         {% if current_diff_url %}
         <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</span></a>
         {% else %}
-        {% if new_version_available %}
+        {% if new_version_available and not (has_password and not current_user.is_authenticated) %}
         <span id="new-version-text" class="pure-menu-heading"><a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a></span>
         {% endif %}
         {% endif %}
 
-        <ul class="pure-menu-list">
-        {% if current_user.is_authenticated %}
+        <ul class="pure-menu-list"  id="top-right-menu">
+        {% if current_user.is_authenticated or not has_password %}
+            {% if not current_diff_url %}
             <li class="pure-menu-item">
-                <a href="/backup" class="pure-menu-link">BACKUP</a>
+                <a href="{{ url_for('get_backup')}}" class="pure-menu-link">BACKUP</a>
             </li>
             <li class="pure-menu-item">
-                <a href="/import" class="pure-menu-link">IMPORT</a>
+                <a href="{{ url_for('import_page')}}" class="pure-menu-link">IMPORT</a>
             </li>
             <li class="pure-menu-item">
-                <a href="/settings" class="pure-menu-link">SETTINGS</a>
+                <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
             </li>
+            {% else %}
+            <li class="pure-menu-item">
+                <a href="{{ url_for('edit_page', uuid=uuid, next='diff') }}" class="pure-menu-link">EDIT</a>
+            </li>
+            {% endif %}
         {% else %}
             <li class="pure-menu-item">
                 <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
             </li>
         {% endif %}
 
-            {% if current_user.is_authenticated %}
-                <li class="pure-menu-item"><a href="/logout" class="pure-menu-link">LOG OUT</a></li>
-            {% endif %}
+        {% if current_user.is_authenticated %}
+            <li class="pure-menu-item"><a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a></li>
+        {% endif %}
             <li class="pure-menu-item"><a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
                 <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16"
                      version="1.1"
@@ -58,28 +65,26 @@
                           d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
                 </svg>
             </a></li>
-            <!--
-            <li class="pure-menu-item"><a href="#" class="pure-menu-link">Tour</a></li>
-            <li class="pure-menu-item"><a href="#" class="pure-menu-link">Sign Up</a></li>
-            -->
         </ul>
     </div>
 </div>
-<div id="version">v{{ version }}</div>
+
+{% if left_sticky %}<div class="sticky-tab" id="left-sticky"><a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a></div> {% endif %}
+{% if right_sticky %}<div class="sticky-tab" id="right-sticky">{{ right_sticky }}</div> {% endif %}
 <section class="content">
     <header>
         {% block header %}{% endblock %}
     </header>
 
-    {% if messages %}
-    <div class="messages">
-        {% for message in messages %}
-        <div class="flash-message {{ message['class'] }}">{{ message['message'] }}</div>
+    {% with messages = get_flashed_messages(with_categories=true) %}
+      {% if messages %}
+        <ul class=messages>
+        {% for category, message in messages %}
+          <li class="{{ category }}">{{ message }}</li>
         {% endfor %}
-    </div>
-    {% endif %}
-
-
+        </ul>
+      {% endif %}
+    {% endwith %}
     {% block content %}
 
     {% endblock %}

changedetectionio/templates/diff.html

@@ -36,7 +36,6 @@
     <a onclick="next_diff();">Jump</a>
 </div>
 <div id="diff-ui">
-
     <table>
         <tbody>
         <tr>
@@ -53,7 +52,9 @@
 
 </div>
 
-<script src="/static/js/diff.js"></script>
+
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff.js')}}"></script>
+
 <script defer="">
 
 
@@ -130,7 +131,8 @@ if ('oninput' in a) {
 
 function onDiffTypeChange(radio) {
 	window.diffType = radio.value;
-	document.title = "Diff " + radio.value.slice(4);
+// Not necessary 
+//	document.title = "Diff " + radio.value.slice(4);
 }
 
 var radio = document.getElementsByName('diff_type');

changedetectionio/templates/edit.html

@@ -0,0 +1,152 @@
+{% extends 'base.html' %}
+{% block content %}
+{% from '_helpers.jinja' import render_field %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+
+<div class="edit-form monospaced-textarea">
+
+    <div class="tabs">
+        <ul>
+            <li class="tab" id="default-tab"><a href="#general">General</a></li>
+            <li class="tab"><a href="#request">Request</a></li>
+            <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
+            <li class="tab"><a href="#notifications">Notifications</a></li>
+        </ul>
+    </div>
+
+    <div class="box-wrap inner">
+        <form class="pure-form pure-form-stacked"
+              action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next') ) }}" method="POST">
+
+            <div class="tab-pane-inner" id="general">
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
+                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span>
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.title, class="m-d") }}
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.tag) }}
+                        <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.minutes_between_check) }}
+                        {% if using_default_minutes %}
+                        <span class="pure-form-message-inline">Currently using the <a
+                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
+                        {% else %}
+                        <span class="pure-form-message-inline">Set to blank to use the <a
+                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
+                        {% endif %}
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.extract_title_as_title) }}
+                    </div>
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="request">
+                    <div class="pure-control-group">
+                        {{ render_field(form.fetch_backend) }}
+                        <span class="pure-form-message-inline">
+                            <p>Use the <strong>Basic</strong> method (default) where your watched site doesn't need Javascript to render.</p>
+                            <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
+                        </span>
+                    </div>
+
+                <fieldset class="pure-group">
+                                    <div class="pure-control-group">
+                    {{ render_field(form.method) }}
+                </div>
+                    <strong>Note: <i>Request Headers and Body settings are ONLY used by Basic fast Plaintext/HTTP Client fetch method.</i></strong>
+                    {{ render_field(form.headers, rows=5, placeholder="Example
+Cookie: foobar
+User-Agent: wonderbra 1.0") }}
+                </fieldset>
+                <div class="pure-control-group">
+                    {{ render_field(form.body, rows=5, placeholder="Example
+{
+   \"name\":\"John\",
+   \"age\":30,
+   \"car\":null
+}") }}
+                </div>
+
+            </div>
+
+            <div class="tab-pane-inner" id="notifications">
+                <strong>Note: <i>These settings override the global settings for this watch.</i></strong>
+                <fieldset>
+                    <div class="field-group">
+                        {{ render_common_settings_form(form, current_base_url) }}
+                    </div>
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="filters-and-triggers">
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.",
+                        class="m-d") }}
+                        <span class="pure-form-message-inline">
+                    <ul>
+                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
+                        <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <b>"json:"</b>, <a
+                                href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
+                        <li>XPATH - Limit text to this XPath rule, simply start with a forward-slash, example  <b>//*[contains(@class, 'sametext')]</b>, <a
+                                href="http://xpather.com/" target="new">test your XPath here</a></li>
+                    </ul>
+                    Please be sure that you thoroughly understand how to write CSS or JSONPath, XPath selector rules before filing an issue on GitHub! <a
+                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
+                </span>
+                    </div>
+
+                </fieldset>
+                <fieldset class="pure-group">
+                    {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
+/some.regex\d{2}/ for case-INsensitive regex
+                    ") }}
+                    <span class="pure-form-message-inline">
+                        <ul>
+                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
+                            <li>Regular Expression support, wrap the line in forward slash <b>/regex/</b></li>
+                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
+                        </ul>
+                </span>
+
+            </fieldset>
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line
+/some.regex\d{2}/ for case-INsensitive regex
+                    ") }}
+                        <span class="pure-form-message-inline">
+                    <ul>
+                        <li>Text to wait for before triggering a change/notification, all text and regex are tested <i>case-insensitive</i>.</li>
+                        <li>Trigger text is processed from the result-text that comes out of any CSS/JSON Filters for this watch</li>
+                        <li>Each line is process separately (think of each line as "OR")</li>
+                        <li>Note: Wrap in forward slash / to use regex  example: <span style="font-family: monospace; background: #eee">/foo\d/</span></li>
+                    </ul>
+                        </span>
+                    </div>
+                </fieldset>
+            </div>
+
+            <div id="actions">
+                <div class="pure-control-group">
+
+                    <button type="submit" class="pure-button pure-button-primary">Save</button>
+                    <a href="{{url_for('api_delete', uuid=uuid)}}"
+                       class="pure-button button-small button-error ">Delete</a>
+                    <a href="{{url_for('api_clone', uuid=uuid)}}"
+                       class="pure-button button-small ">Create Copy</a>
+                </div>
+            </div>
+        </form>
+    </div>
+</div>
+
+{% endblock %}

changedetectionio/templates/import.html

@@ -0,0 +1,23 @@
+{% extends 'base.html' %}
+
+{% block content %}
+<div class="edit-form">
+     <div class="inner">
+        <form class="pure-form pure-form-aligned" action="{{url_for('import_page')}}" method="POST">
+            <fieldset class="pure-group">
+                <legend>One URL per line, URLs that do not pass validation will stay in the textarea.</legend>
+
+                <textarea name="urls" class="pure-input-1-2" placeholder="https://"
+                          style="width: 100%;
+                                font-family:monospace;
+                                white-space: pre;
+                                overflow-wrap: normal;
+                                overflow-x: scroll;" rows="25">{{ remaining }}</textarea>
+            </fieldset>
+            <button type="submit" class="pure-button pure-input-1-2 pure-button-primary">Import</button>
+        </form>
+     </div>
+</div>
+
+{% endblock %}
+

changedetectionio/templates/login.html

@@ -2,7 +2,9 @@
 
 {% block content %}
 <div class="edit-form">
-    <form class="pure-form pure-form-stacked" action="/login" method="POST">
+
+ <div class="inner">
+    <form class="pure-form pure-form-stacked" action="{{url_for('login')}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
                 <label for="password">Password</label>
@@ -11,10 +13,11 @@
                 <input type="hidden" id="email" name="email" value="defaultuser@changedetection.io" />
             </div>
             <div class="pure-control-group">
-                <button type="submit" class="pure-button pure-button-primary">Submit</button>
+                <button type="submit" class="pure-button pure-button-primary">Login</button>
             </div>
         </fieldset>
     </form>
-</div>
+  </div>
+ </div>
 
 {% endblock %}

changedetectionio/templates/scrub.html

@@ -2,47 +2,36 @@
 
 {% block content %}
 <div class="edit-form">
-
-
-    <form class="pure-form pure-form-stacked" action="/scrub" method="POST">
+    <div class="box-wrap inner">
+    <form class="pure-form pure-form-stacked" action="{{url_for('scrub_page')}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
                 This will remove all version snapshots/data, but keep your list of URLs. <br/>
                 You may like to use the <strong>BACKUP</strong> link first.<br/>
-
-                Type in the word <strong>scrub</strong> to confirm that you understand!
-                <br/>
-
             </div>
-
+            <br/>
             <div class="pure-control-group">
-                <br/>
-                <label for="confirmtext">Confirm text</label><br/>
+                <label for="confirmtext">Confirmation text</label>
                 <input type="text" id="confirmtext" required="" name="confirmtext" value="" size="10"/>
+                <span class="pure-form-message-inline">Type in the word <strong>scrub</strong> to confirm that you understand!</span>
             </div>
-
-
+            <br/>
             <div class="pure-control-group">
-                <br/>
-                <label for="confirmtext">Limit delete history including and after date</label><br/>
-                <input type="text" id="limit_date" required="" name="limit_date" value="" size="10"/>
-                <br/>
-
-
+                <label for="confirmtext">Optional: Limit deletion of snapshots to snapshots <i>newer</i> than date/time</label>
+                <input type="datetime-local" id="limit_date" name="limit_date"  />
+                <span class="pure-form-message-inline">dd/mm/yyyy hh:mm (24 hour format)</span>
             </div>
+            <br/>
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Scrub!</button>
             </div>
             <br/>
             <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Cancel</a>
+                <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Cancel</a>
             </div>
-
-
         </fieldset>
     </form>
-
-
+    </div>
 </div>
 
 {% endblock %}

changedetectionio/templates/settings.html

@@ -0,0 +1,110 @@
+{% extends 'base.html' %}
+
+{% block content %}
+{% from '_helpers.jinja' import render_field %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
+
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='settings.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+
+<div class="edit-form">
+    <div class="tabs">
+        <ul>
+            <li class="tab" id="default-tab"><a href="#general">General</a></li>
+            <li class="tab"><a href="#notifications">Notifications</a></li>
+            <li class="tab"><a href="#fetching">Fetching</a></li>
+            <li class="tab"><a href="#filters">Global Filters</a></li>
+        </ul>
+    </div>
+    <div class="box-wrap inner">
+        <form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
+            <div class="tab-pane-inner" id="general">
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.minutes_between_check) }}
+                        <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
+                    </div>
+                    <div class="pure-control-group">
+                        {% if current_user.is_authenticated %}
+                        <a href="{{url_for('settings_page', removepassword='yes')}}"
+                           class="pure-button pure-button-primary">Remove password</a>
+                        {% else %}
+                        {{ render_field(form.password) }}
+                        <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
+                        {% endif %}
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.base_url, placeholder="http://yoursite.com:5000/",
+                        class="m-d") }}
+                        <span class="pure-form-message-inline">
+                            Base URL used for the {base_url} token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{current_base_url}}"),
+                            <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
+                        </span>
+                    </div>
+
+                    <div class="pure-control-group">
+                        {{ render_field(form.extract_title_as_title) }}
+                        <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
+                    </div>
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="notifications">
+                <fieldset>
+                    <div class="field-group">
+                        {{ render_common_settings_form(form, current_base_url) }}
+                    </div>
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="fetching">
+                <div class="pure-control-group">
+                    {{ render_field(form.fetch_backend) }}
+                    <span class="pure-form-message-inline">
+                        <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
+                        <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
+                    </span>
+                </div>
+            </div>
+
+
+            <div class="tab-pane-inner" id="filters">
+
+                    <fieldset class="pure-group">
+                    {{ render_field(form.ignore_whitespace) }}
+                    <span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br/>
+                    <i>Note:</i> Changing this will change the status of your existing watches, possibily trigger alerts etc.
+                    </span>
+                    </fieldset>
+
+
+                    <fieldset class="pure-group">
+                    {{ render_field(form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
+/some.regex\d{2}/ for case-INsensitive regex
+                    ") }}
+                    <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br/>
+                    <span class="pure-form-message-inline">
+                        <ul>
+                            <li>Note: This is applied globally in addition to the per-watch rules.</li>
+                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
+                            <li>Regular Expression support, wrap the line in forward slash <b>/regex/</b></li>
+                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
+                        </ul>
+                     </span>
+                    </fieldset>
+           </div>
+
+            <div id="actions">
+                <div class="pure-control-group">
+                    <button type="submit" class="pure-button pure-button-primary">Save</button>
+                                           <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
+                        <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete
+                            History
+                            Snapshot Data</a>
+                </div>
+            </div>
+        </form>
+    </div>
+</div>
+
+{% endblock %}

changedetectionio/templates/watch-overview.html

@@ -1,24 +1,24 @@
 {% extends 'base.html' %}
-
 {% block content %}
+{% from '_helpers.jinja' import render_simple_field %}
 
 <div class="box">
 
-    <form class="pure-form" action="/api/add" method="POST" id="new-watch-form">
+    <form class="pure-form" action="{{ url_for('api_watch_add') }}" method="POST" id="new-watch-form">
         <fieldset>
             <legend>Add a new change detection watch</legend>
-            <input type="url" placeholder="https://..." name="url"/>
-            <input type="text" placeholder="tag" size="10" name="tag" value="{{active_tag if active_tag}}"/>
+                {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
+                {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="tag") }}
             <button type="submit" class="pure-button pure-button-primary">Watch</button>
         </fieldset>
         <!-- add extra stuff, like do a http POST and send headers -->
         <!-- user/pass r = requests.get('https://api.github.com/user', auth=('user', 'pass')) -->
     </form>
     <div>
-        <a href="/" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
+        <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
         {% for tag in tags %}
             {% if tag != "" %}
-                <a href="/?tag={{ tag}}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
+                <a href="{{url_for('index', tag=tag) }}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
             {% endif %}
         {% endfor %}
     </div>
@@ -44,10 +44,13 @@
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
                 {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
-                <td>{{ loop.index }}</td>
-                <td class="paused-state state-{{watch.paused}}"><a href="/?pause={{ watch.uuid}}{% if active_tag %}&tag={{active_tag}}{% endif %}"><img src="/static/images/pause.svg" alt="Pause"/></a></td>
-                <td class="title-col">{{watch.title if watch.title is not none else watch.url}}
+                <td class="inline">{{ loop.index }}</td>
+                <td class="inline paused-state state-{{watch.paused}}"><a href="{{url_for('index', pause=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause" title="Pause"/></a></td>
+
+                <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.url }}"></a>
+                    {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="static/images/Google-Chrome-icon.png" />{% endif %}
+
                     {% if watch.last_error is defined and watch.last_error != False %}
                     <div class="fetch-error">{{ watch.last_error }}</div>
                     {% endif %}
@@ -55,22 +58,22 @@
                     <span class="watch-tag-list">{{ watch.tag}}</span>
                     {% endif %}
                 </td>
-                <td>{{watch|format_last_checked_time}}</td>
-                <td>{% if watch.history|length >= 2 and watch.last_changed %}
+                <td class="last-checked">{{watch|format_last_checked_time}}</td>
+                <td class="last-changed">{% if watch.history|length >= 2 and watch.last_changed %}
                     {{watch.last_changed|format_timestamp_timeago}}
                     {% else %}
                     Not yet
                     {% endif %}
                 </td>
                 <td>
-                    <a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
+                    <a href="{{ url_for('api_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="pure-button button-small pure-button-primary">Recheck</a>
-                    <a href="/edit/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
                     {% if watch.history|length >= 2 %}
-                    <a href="/diff/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
+                    <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
                     {% else %}
                         {% if watch.history|length == 1 %}
-                            <a href="/preview/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
+                            <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
                         {% endif %}
                     {% endif %}
                 </td>
@@ -81,17 +84,17 @@
         <ul id="post-list-buttons">
             {% if has_unviewed %}
             <li>
-                <a href="/api/mark-all-viewed" class="pure-button button-tag ">Mark all viewed</a>
+                <a href="{{url_for('mark_all_viewed', tag=request.args.get('tag')) }}" class="pure-button button-tag ">Mark all viewed</a>
             </li>
             {% endif %}
             <li>
-               <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('api_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
                 all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
             </li>
             <li>
-                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="/static/images/Generic_Feed-icon.svg" height="15px"></a>
+                <a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>
             </li>
         </ul>
     </div>
 </div>
-{% endblock %}
+{% endblock %}

changedetectionio/tests/conftest.py

@@ -1,17 +1,31 @@
 #!/usr/bin/python3
 
 import pytest
-from backend import changedetection_app
-from backend import store
+from changedetectionio import changedetection_app
+from changedetectionio import store
 import os
 
-
 # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
 # Much better boilerplate than the docs
 # https://www.python-boilerplate.com/py3+flask+pytest/
 
 global app
 
+
+def cleanup(datastore_path):
+    # Unlink test output files
+    files = ['output.txt',
+             'url-watches.json',
+             'notification.txt',
+             'count.txt',
+             'endpoint-content.txt'
+                 ]
+    for file in files:
+        try:
+            os.unlink("{}/{}".format(datastore_path, file))
+        except FileNotFoundError:
+            pass
+
 @pytest.fixture(scope='session')
 def app(request):
     """Create application for the tests."""
@@ -22,12 +36,10 @@ def app(request):
     except FileExistsError:
         pass
 
-    try:
-        os.unlink("{}/url-watches.json".format(datastore_path))
-    except FileNotFoundError:
-        pass
+    cleanup(datastore_path)
 
     app_config = {'datastore_path': datastore_path}
+    cleanup(app_config['datastore_path'])
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
     app = changedetection_app(app_config, datastore)
     app.config['STOP_THREADS'] = True
@@ -35,14 +47,8 @@ def app(request):
     def teardown():
         datastore.stop_thread = True
         app.config.exit.set()
-        try:
-            os.unlink("{}/url-watches.json".format(datastore_path))
-        except FileNotFoundError:
-            # This is fine in the case of a failure.
-            pass
-
-        assert 1 == 1
+        cleanup(app_config['datastore_path'])
 
+       
     request.addfinalizer(teardown)
     yield app
-

changedetectionio/tests/test_access_control.py

@@ -0,0 +1,108 @@
+from flask import url_for
+
+
+def test_check_access_control(app, client):
+    # Still doesnt work, but this is closer.
+
+    with app.test_client() as c:
+        # Check we dont have any password protection enabled yet.
+        res = c.get(url_for("settings_page"))
+        assert b"Remove password" not in res.data
+
+        # Enable password check.
+        res = c.post(
+            url_for("settings_page"),
+            data={"password": "foobar",
+                  "minutes_between_check": 180,
+                  'fetch_backend': "html_requests"},
+            follow_redirects=True
+        )
+
+        assert b"Password protection enabled." in res.data
+        assert b"LOG OUT" not in res.data
+
+        # Check we hit the login
+        res = c.get(url_for("index"), follow_redirects=True)
+
+        assert b"Login" in res.data
+
+        # Menu should not be available yet
+        #        assert b"SETTINGS" not in res.data
+        #        assert b"BACKUP" not in res.data
+        #        assert b"IMPORT" not in res.data
+
+        # defaultuser@changedetection.io is actually hardcoded for now, we only use a single password
+        res = c.post(
+            url_for("login"),
+            data={"password": "foobar"},
+            follow_redirects=True
+        )
+
+        assert b"LOG OUT" in res.data
+        res = c.get(url_for("settings_page"))
+
+        # Menu should be available now
+        assert b"SETTINGS" in res.data
+        assert b"BACKUP" in res.data
+        assert b"IMPORT" in res.data
+        assert b"LOG OUT" in res.data
+
+        # Now remove the password so other tests function, @todo this should happen before each test automatically
+        res = c.get(url_for("settings_page", removepassword="yes"),
+              follow_redirects=True)
+        assert b"Password protection removed." in res.data
+
+        res = c.get(url_for("index"))
+        assert b"LOG OUT" not in res.data
+
+
+# There was a bug where saving the settings form would submit a blank password
+def test_check_access_control_no_blank_password(app, client):
+    # Still doesnt work, but this is closer.
+
+    with app.test_client() as c:
+        # Check we dont have any password protection enabled yet.
+        res = c.get(url_for("settings_page"))
+        assert b"Remove password" not in res.data
+
+        # Enable password check.
+        res = c.post(
+            url_for("settings_page"),
+            data={"password": "",
+                  "minutes_between_check": 180,
+                  'fetch_backend': "html_requests"},
+
+        follow_redirects=True
+        )
+
+        assert b"Password protection enabled." not in res.data
+        assert b"Login" not in res.data
+
+
+# There was a bug where saving the settings form would submit a blank password
+def test_check_access_no_remote_access_to_remove_password(app, client):
+    # Still doesnt work, but this is closer.
+
+    with app.test_client() as c:
+        # Check we dont have any password protection enabled yet.
+        res = c.get(url_for("settings_page"))
+        assert b"Remove password" not in res.data
+
+        # Enable password check.
+        res = c.post(
+            url_for("settings_page"),
+            data={"password": "password", "minutes_between_check": 180,
+                  'fetch_backend': "html_requests"},
+            follow_redirects=True
+        )
+
+        assert b"Password protection enabled." in res.data
+        assert b"Login" in res.data
+
+        res = c.get(url_for("settings_page", removepassword="yes"),
+              follow_redirects=True)
+        assert b"Password protection removed." not in res.data
+
+        res = c.get(url_for("index"),
+              follow_redirects=True)
+        assert b"watch-table-wrapper" not in res.data

changedetectionio/tests/test_api.py

@@ -0,0 +1,74 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def set_response_data(test_return_data):
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_snapshot_api_detects_change(client, live_server):
+
+    test_return_data = "Some initial text"
+
+    test_return_data_modified = "Some NEW nice initial text"
+
+    sleep_time_for_fetch_thread = 3
+
+    set_response_data(test_return_data)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(
+        url_for("api_snapshot", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert test_return_data.encode() == res.data
+
+    #  Make a change
+    set_response_data(test_return_data_modified)
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(
+        url_for("api_snapshot", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert test_return_data_modified.encode() == res.data
+
+def test_snapshot_api_invalid_uuid(client, live_server):
+
+    res = client.get(
+        url_for("api_snapshot", uuid="invalid"),
+        follow_redirects=True
+    )
+
+    assert res.status_code == 400
+

changedetectionio/tests/test_auth.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_basic_auth(client, live_server):
+
+    live_server_setup(live_server)
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@")
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Check form validation
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(1)
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b'myuser mypass basic' in res.data

changedetectionio/tests/test_backend.py

@@ -3,55 +3,14 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-import pytest
+from . util import set_original_response, set_modified_response, live_server_setup
 
 sleep_time_for_fetch_thread = 3
 
 
-def test_setup_liveserver(live_server):
-    @live_server.app.route('/test-endpoint')
-    def test_endpoint():
-        # Tried using a global var here but didn't seem to work, so reading from a file instead.
-        with open("test-datastore/output.txt", "r") as f:
-            return f.read()
-
-    live_server.start()
-
-    assert 1 == 1
-
-
-def set_original_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>Which is across multiple lines</p>
-     </br>
-     So let's see what happens.  </br>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/output.txt", "w") as f:
-        f.write(test_return_data)
-
-
-def set_modified_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>which has this one new line</p>
-     </br>
-     So let's see what happens.  </br>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/output.txt", "w") as f:
-        f.write(test_return_data)
-
-
 def test_check_basic_change_detection_functionality(client, live_server):
     set_original_response()
+    live_server_setup(live_server)
 
     # Add our URL to the import page
     res = client.post(
@@ -75,6 +34,12 @@ def test_check_basic_change_detection_functionality(client, live_server):
         assert b'unviewed' not in res.data
         assert b'test-endpoint' in res.data
 
+        # Default no password set, this stuff should be always available.
+
+        assert b"SETTINGS" in res.data
+        assert b"BACKUP" in res.data
+        assert b"IMPORT" in res.data
+
     #####################
 
     # Make a change
@@ -93,6 +58,12 @@ def test_check_basic_change_detection_functionality(client, live_server):
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
+    # #75, and it should be in the RSS feed
+    res = client.get(url_for("rss"))
+    expected_url = url_for('test_endpoint', _external=True)
+    assert b'<rss' in res.data
+    assert expected_url.encode('utf-8') in res.data
+
     # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
     res = client.get(url_for("diff_history_page", uuid="first"))
     assert b'Compare newest' in res.data
@@ -109,52 +80,28 @@ def test_check_basic_change_detection_functionality(client, live_server):
         # It should report nothing found (no new 'unviewed' class)
         res = client.get(url_for("index"))
         assert b'unviewed' not in res.data
+        assert b'head title' not in res.data # Should not be present because this is off by default
         assert b'test-endpoint' in res.data
 
     set_original_response()
 
+    # Enable auto pickup of <title> in settings
+    res = client.post(
+        url_for("settings_page"),
+        data={"extract_title_as_title": "1", "minutes_between_check": 180, 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
+
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
+    # It should have picked up the <title>
+    assert b'head title' in res.data
 
+    #
     # Cleanup everything
     res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
 
-
-def test_check_access_control(client):
-    return
-    # @note: does not seem to handle the last logout step correctly, we're still logged in.. but yet..
-    #        pytest team keep telling us that we have a new context.. i'm lost :(
-
-    # Add our URL to the import page
-    res = client.post(
-        url_for("settings_page"),
-        data={"password": "foobar"},
-        follow_redirects=True
-    )
-    assert b"LOG OUT" not in res.data
-
-    client.get(url_for("import_page"), follow_redirects=True)
-    assert b"Password" in res.data
-
-    #defaultuser@changedetection.io is actually hardcoded for now, we only use a single password
-    res = client.post(
-        url_for("login"),
-        data={"password": "foobar", "email": "defaultuser@changedetection.io"},
-        follow_redirects=True
-    )
-
-    assert b"LOG OUT" in res.data
-
-    client.get(url_for("settings_page"), follow_redirects=True)
-    assert b"LOG OUT" in res.data
-
-    # Now remove the password so other tests function, @todo this should happen before each test automatically
-
-    print(res.data)
-    client.get(url_for("settings_page", removepassword="true"), follow_redirects=True)
-
-    client.get(url_for("import_page", removepassword="true"), follow_redirects=True)
-    assert b"LOG OUT" not in res.data

changedetectionio/tests/test_backup.py

@@ -0,0 +1,25 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from urllib.request import urlopen
+from . util import set_original_response, set_modified_response, live_server_setup
+
+
+def test_backup(client, live_server):
+
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    res = client.get(
+        url_for("get_backup"),
+        follow_redirects=True
+    )
+
+    # Should get the right zip content type
+    assert res.content_type == "application/zip"
+    # Should be PK/ZIP stream
+    assert res.data.count(b'PK') >= 2
+

changedetectionio/tests/test_clone.py

@@ -0,0 +1,30 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+
+
+def test_trigger_functionality(client, live_server):
+
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": "https://changedetection.io"},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+
+    res = client.get(
+        url_for("api_clone", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"Cloned." in res.data

changedetectionio/tests/test_css_selector.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+from ..html_tools import *
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that will change</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that changes</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+# Test that the CSS extraction works how we expect, important here is the right placing of new lines \n's
+def test_css_filter_output():
+    from changedetectionio import fetch_site_status
+    from inscriptis import get_text
+
+    # Check text with sub-parts renders correctly
+    content = """<html> <body><div id="thingthing" >  Some really <b>bold</b> text  </div> </body> </html>"""
+    html_blob = css_filter(css_filter="#thingthing", html_content=content)
+    text = get_text(html_blob)
+    assert text == "  Some really bold text"
+
+    content = """<html> <body>
+    <p>foo bar blah</p>
+    <div class="parts">Block A</div> <div class="parts">Block B</div></body> 
+    </html>
+"""
+    html_blob = css_filter(css_filter=".parts", html_content=content)
+    text = get_text(html_blob)
+
+    # Divs are converted to 4 whitespaces by inscriptis
+    assert text == "    Block A\n    Block B"
+
+
+# Tests the whole stack works with the CSS Filter
+def test_check_markup_css_filter_restriction(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    css_filter = "#sametext"
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(css_filter.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+    #  Make a change
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should have 'unviewed' still
+    # Because it should be looking at only that 'sametext' id
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data

changedetectionio/tests/test_errorhandling.py

@@ -0,0 +1,62 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+from ..html_tools import *
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def test_error_handler(client, live_server):
+
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint_403_error', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'Status Code 403' in res.data
+    assert bytes("just now".encode('utf-8')) in res.data
+
+# Just to be sure error text is properly handled
+def test_error_text_handler(client, live_server):
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": "https://errorfuldomainthatnevereallyexists12356.com"},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    res = client.get(url_for("index"))
+    assert b'Name or service not known' in res.data
+    assert bytes("just now".encode('utf-8')) in res.data
+

changedetectionio/tests/test_ignore_regex_text.py

@@ -0,0 +1,31 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+# Unit test of the stripper
+# Always we are dealing in utf-8
+def test_strip_regex_text_func():
+    from changedetectionio import fetch_site_status
+
+    test_content = """
+    but sometimes we want to remove the lines.
+    
+    but 1 lines
+    but including 1234 lines
+    igNORe-cAse text we dont want to keep    
+    but not always."""
+
+    ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"]
+
+    fetcher = fetch_site_status.perform_site_check(datastore=False)
+    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
+
+    assert b"but 1 lines" in stripped_content
+    assert b"igNORe-cAse text" not in stripped_content
+    assert b"but 1234 lines" not in stripped_content
+

changedetectionio/tests/test_ignore_text.py

@@ -0,0 +1,238 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+# Unit test of the stripper
+# Always we are dealing in utf-8
+def test_strip_text_func():
+    from changedetectionio import fetch_site_status
+
+    test_content = """
+    Some content
+    is listed here
+
+    but sometimes we want to remove the lines.
+
+    but not always."""
+
+    ignore_lines = ["sometimes"]
+
+    fetcher = fetch_site_status.perform_site_check(datastore=False)
+    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
+
+    assert b"sometimes" not in stripped_content
+    assert b"Some content" in stripped_content
+
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_modified_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some NEW nice initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+# Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
+def set_modified_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     <P>ZZZZZ</P>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_check_ignore_text_functionality(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
+    set_original_ignore_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"ignore_text": ignore_text, "url": test_url, 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(ignore_text.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+    #  Make a change
+    set_modified_ignore_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+    # Just to be sure.. set a regular modified change..
+    set_modified_original_ignore_response()
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+def test_check_global_ignore_text_functionality(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
+    set_original_ignore_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the settings page, add our ignore text
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "minutes_between_check": 180,
+            "global_ignore_text": ignore_text,
+            'fetch_backend': "html_requests"
+        },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Goto the edit page of the item, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"ignore_text": "something irrelevent but just to check", "url": test_url, 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("settings_page"),
+    )
+    assert bytes(ignore_text.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+    #  Make a change
+    set_modified_ignore_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+    # Just to be sure.. set a regular modified change..
+    set_modified_original_ignore_response()
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetectionio/tests/test_ignorewhitespace.py

@@ -0,0 +1,96 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+# Should be the same as set_original_ignore_response() but with a little more whitespacing
+def set_original_ignore_response_but_with_whitespace():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>
+
+
+     Which is across multiple lines</p>
+     <br>
+     </br>
+
+         So let's see what happens.  </br>
+
+
+     </body>
+     </html>
+
+    """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+
+# If there was only a change in the whitespacing, then we shouldnt have a change detected
+def test_check_ignore_whitespace(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    set_original_ignore_response()
+
+    # Goto the settings page, add our ignore text
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "minutes_between_check": 180,
+            "ignore_whitespace": "y",
+            'fetch_backend': "html_requests"
+        },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    set_original_ignore_response_but_with_whitespace()
+    time.sleep(sleep_time_for_fetch_thread)
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data

changedetectionio/tests/test_jsonpath_selector.py

@@ -0,0 +1,373 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+import pytest
+
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def test_unittest_inline_html_extract():
+    # So lets pretend that the JSON we want is inside some HTML
+    content="""
+    <html>
+    
+    food and stuff and more
+    <script>
+    alert('nothing really good here');
+    </script>
+    
+    <script type="application/ld+json">
+  xx {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","description":"During the first year of life, nutrition is critical for your baby. NAN OPTIPRO 1 is tailored to ensure your formula fed infant receives balanced, high quality nutrition.<br />Starter infant formula. The age optimised protein source (whey dominant) is from cow’s milk.<br />Backed by more than 150 years of Nestlé expertise.<br />For hygiene and convenience, it is available in an innovative packaging format with a separate storage area for the scoop, and a semi-transparent window which allows you to see how much powder is left in the can without having to open it.","image":"https://cdn0.woolworths.media/content/wowproductimages/large/155536.jpg","brand":{"@context":"http://schema.org","@type":"Organization","name":"Nan"},"gtin13":"7613287517388","offers":{"@context":"http://schema.org","@type":"Offer","potentialAction":{"@context":"http://schema.org","@type":"BuyAction"},"availability":"http://schema.org/InStock","itemCondition":"http://schema.org/NewCondition","price":23.5,"priceCurrency":"AUD"},"review":[],"sku":"155536"}
+</script>
+<body>
+and it can also be repeated
+<script type="application/ld+json">
+  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","description":"During the first year of life, nutrition is critical for your baby. NAN OPTIPRO 1 is tailored to ensure your formula fed infant receives balanced, high quality nutrition.<br />Starter infant formula. The age optimised protein source (whey dominant) is from cow’s milk.<br />Backed by more than 150 years of Nestlé expertise.<br />For hygiene and convenience, it is available in an innovative packaging format with a separate storage area for the scoop, and a semi-transparent window which allows you to see how much powder is left in the can without having to open it.","image":"https://cdn0.woolworths.media/content/wowproductimages/large/155536.jpg","brand":{"@context":"http://schema.org","@type":"Organization","name":"Nan"},"gtin13":"7613287517388","offers":{"@context":"http://schema.org","@type":"Offer","potentialAction":{"@context":"http://schema.org","@type":"BuyAction"},"availability":"http://schema.org/InStock","itemCondition":"http://schema.org/NewCondition","price":23.5,"priceCurrency":"AUD"},"review":[],"sku":"155536"}
+</script>
+<h4>ok</h4>
+</body>
+</html>
+
+    """
+    from .. import html_tools
+
+    # See that we can find the second <script> one, which is not broken, and matches our filter
+    text = html_tools.extract_json_as_string(content, "$.offers.price")
+    assert text == "23.5"
+
+    text = html_tools.extract_json_as_string('{"id":5}', "$.id")
+    assert text == "5"
+
+    # When nothing at all is found, it should throw JSONNOTFound
+    # Which is caught and shown to the user in the watch-overview table
+    with pytest.raises(html_tools.JSONNotFound) as e_info:
+        html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "$.id")
+
+def set_original_ext_response():
+    data = """
+        [
+        {
+            "isPriceLowered": false,
+            "status": "ForSale",
+            "statusOrig": "for sale"
+        },
+        {
+            "_id": "5e7b3e1fb3262d306323ff1e",
+            "listingsType": "consumer",
+            "status": "ForSale",
+            "statusOrig": "for sale"
+        }
+    ]
+        """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(data)
+
+def set_modified_ext_response():
+    data = """
+    [
+    {
+        "isPriceLowered": false,
+        "status": "Sold",
+        "statusOrig": "sold"
+    },
+    {
+        "_id": "5e7b3e1fb3262d306323ff1e",
+        "listingsType": "consumer",
+        "isPriceLowered": false,
+        "status": "Sold"
+    }
+]
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(data)
+
+def set_original_response():
+    test_return_data = """
+    {
+      "employees": [
+        {
+          "id": 1,
+          "name": "Pankaj",
+          "salary": "10000"
+        },
+        {
+          "name": "David",
+          "salary": "5000",
+          "id": 2
+        }
+      ],
+      "boss": {
+        "name": "Fat guy"
+      },
+      "available": true
+    }
+    """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+
+def set_response_with_html():
+    test_return_data = """
+    {
+      "test": [
+        {
+          "html": "<b>"
+        }
+      ]
+    }
+    """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """
+    {
+      "employees": [
+        {
+          "id": 1,
+          "name": "Pankaj",
+          "salary": "10000"
+        },
+        {
+          "name": "David",
+          "salary": "5000",
+          "id": 2
+        }
+      ],
+      "boss": {
+        "name": "Foobar"
+      },
+      "available": false
+    }
+        """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+def test_check_json_without_filter(client, live_server):
+    # Request a JSON document from a application/json source containing HTML
+    # and be sure it doesn't get chewed up by instriptis
+    set_response_with_html()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint_json', _external=True)
+    client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b'&#34;&lt;b&gt;' in res.data
+    assert res.data.count(b'{\n') >= 2
+
+
+def test_check_json_filter(client, live_server):
+    json_filter = 'json:boss.name'
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": json_filter,
+              "url": test_url,
+              "tag": "",
+              "headers": "",
+              "fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(json_filter.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+    #  Make a change
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(4)
+
+    # It should have 'unviewed' still
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    # Should not see this, because its not in the JSONPath we entered
+    res = client.get(url_for("diff_history_page", uuid="first"))
+    # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
+    assert b'Foobar' in res.data
+
+
+def test_check_json_filter_bool_val(client, live_server):
+    json_filter = "json:$['available']"
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    test_url = url_for('test_endpoint', _external=True)
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": json_filter,
+              "url": test_url,
+              "tag": "",
+              "headers": "",
+              "fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+    #  Make a change
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    res = client.get(url_for("diff_history_page", uuid="first"))
+    # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
+    assert b'false' in res.data
+
+# Re #265 - Extended JSON selector test
+# Stuff to consider here
+# - Selector should be allowed to return empty when it doesnt match (people might wait for some condition)
+# - The 'diff' tab could show the old and new content
+# - Form should let us enter a selector that doesnt (yet) match anything
+def test_check_json_ext_filter(client, live_server):
+    json_filter = 'json:$[?(@.status==Sold)]'
+
+    set_original_ext_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": json_filter,
+              "url": test_url,
+              "tag": "",
+              "headers": "",
+              "fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(json_filter.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+    #  Make a change
+    set_modified_ext_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(4)
+
+    # It should have 'unviewed'
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    res = client.get(url_for("diff_history_page", uuid="first"))
+
+    # We should never see 'ForSale' because we are selecting on 'Sold' in the rule,
+    # But we should know it triggered ('unviewed' assert above)
+    assert b'ForSale' not in res.data
+    assert b'Sold' in res.data
+

changedetectionio/tests/test_notification.py

@@ -0,0 +1,227 @@
+import os
+import time
+import re
+from flask import url_for
+from . util import set_original_response, set_modified_response, live_server_setup
+import logging
+from changedetectionio.notification import default_notification_body, default_notification_title
+
+# Hard to just add more live server URLs when one test is already running (I think)
+# So we add our test here (was in a different file)
+def test_check_notification(client, live_server):
+
+    live_server_setup(live_server)
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(3)
+
+    # Re 360 - new install should have defaults set
+    res = client.get(url_for("settings_page"))
+    assert default_notification_body.encode() in res.data
+    assert default_notification_title.encode() in res.data
+
+    # When test mode is in BASE_URL env mode, we should see this already configured
+    env_base_url = os.getenv('BASE_URL', '').strip()
+    if len(env_base_url):
+        logging.debug(">>> BASE_URL enabled, looking for %s", env_base_url)
+        res = client.get(url_for("settings_page"))
+        assert bytes(env_base_url.encode('utf-8')) in res.data
+    else:
+        logging.debug(">>> SKIPPING BASE_URL check")
+
+    # re #242 - when you edited an existing new entry, it would not correctly show the notification settings
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("api_watch_add"),
+        data={"url": test_url, "tag": ''},
+        follow_redirects=True
+    )
+    assert b"Watch added" in res.data
+
+    # Give the thread time to pick up the first version
+    time.sleep(3)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    url = url_for('test_notification_endpoint', _external=True)
+    notification_url = url.replace('http', 'json')
+
+    print (">>>> Notification URL: "+notification_url)
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"notification_urls": notification_url,
+              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
+              "notification_body": "BASE URL: {base_url}\n"
+                                   "Watch URL: {watch_url}\n"
+                                   "Watch UUID: {watch_uuid}\n"
+                                   "Watch title: {watch_title}\n"
+                                   "Watch tag: {watch_tag}\n"
+                                   "Preview: {preview_url}\n"
+                                   "Diff URL: {diff_url}\n"
+                                   "Snapshot: {current_snapshot}\n"
+                                   "Diff: {diff}\n"
+                                   "Diff Full: {diff_full}\n"
+                                   ":-)",
+              "notification_format": "Text",
+              "url": test_url,
+              "tag": "my tag",
+              "title": "my title",
+              "headers": "",
+              "fetch_backend": "html_requests",
+              "trigger_check": "y"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+    assert b"Test notification queued" in res.data
+
+    # Hit the edit page, be sure that we saved it
+    res = client.get(
+        url_for("edit_page", uuid="first"))
+    assert bytes(notification_url.encode('utf-8')) in res.data
+
+    # Re #242 - wasnt saving?
+    assert bytes("New ChangeDetection.io Notification".encode('utf-8')) in res.data
+
+
+
+    # Because we hit 'send test notification on save'
+    time.sleep(3)
+
+    notification_submission = None
+
+    # Verify what was sent as a notification, this file should exist
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+        # Did we see the URL that had a change, in the notification?
+
+    assert test_url in notification_submission
+
+    os.unlink("test-datastore/notification.txt")
+
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Did the front end see it?
+    res = client.get(
+        url_for("index"))
+
+    assert bytes("just now".encode('utf-8')) in res.data
+
+    notification_submission=None
+    # Verify what was sent as a notification
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+        # Did we see the URL that had a change, in the notification?
+
+    assert test_url in notification_submission
+
+    # Diff was correctly executed
+    assert "Diff Full: (changed) Which is across multiple lines" in notification_submission
+    assert "(-> into) which has this one new line" in notification_submission
+
+
+    if env_base_url:
+        # Re #65 - did we see our BASE_URl ?
+        logging.debug (">>> BASE_URL checking in notification: %s", env_base_url)
+        assert env_base_url in notification_submission
+    else:
+        logging.debug(">>> Skipping BASE_URL check")
+
+    ##  Now configure something clever, we go into custom config (non-default) mode, this is returned by the endpoint
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(";jasdhflkjadshf kjhsdfkjl ahslkjf haslkjd hfaklsj hf\njl;asdhfkasj stuff we will detect\n")
+
+    res = client.post(
+        url_for("settings_page"),
+        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
+              "notification_urls": "json://foobar.com", #Re #143 should not see that it sent without [test checkbox]
+              "minutes_between_check": 180,
+              "fetch_backend": "html_requests",
+              },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+    # Re #143 - should not see this if we didnt hit the test box
+    assert b"Test notification queued" not in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Did the front end see it?
+    res = client.get(
+        url_for("index"))
+
+    assert bytes("just now".encode('utf-8')) in res.data
+
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+        print ("Notification submission was:", notification_submission)
+        # Re #342 - check for accidental python byte encoding of non-utf8/string
+        assert "b'" not in notification_submission
+
+        assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE)
+        assert "Watch title: my title" in notification_submission
+        assert "Watch tag: my tag" in notification_submission
+        assert "diff/" in notification_submission
+        assert "preview/" in notification_submission
+        assert ":-)" in notification_submission
+        assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
+        # This should insert the {current_snapshot}
+        assert "stuff we will detect" in notification_submission
+
+    # Prove that "content constantly being marked as Changed with no Updating causes notification" is not a thing
+    # https://github.com/dgtlmoon/changedetection.io/discussions/192
+    os.unlink("test-datastore/notification.txt")
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(3)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(3)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(3)
+    assert os.path.exists("test-datastore/notification.txt") == False
+
+
+    # Now adding a wrong token should give us an error
+    res = client.post(
+        url_for("settings_page"),
+        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
+              "notification_body": "Rubbish: {rubbish}\n",
+              "notification_format": "Text",
+              "notification_urls": "json://foobar.com",
+              "minutes_between_check": 180,
+              "fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+
+    assert bytes("is not a valid token".encode('utf-8')) in res.data
+
+    # Re #360 some validation
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"notification_urls": notification_url,
+              "notification_title": "",
+              "notification_body": "",
+              "notification_format": "Text",
+              "url": test_url,
+              "tag": "my tag",
+              "title": "my title",
+              "headers": "",
+              "fetch_backend": "html_requests",
+              "trigger_check": "y"},
+        follow_redirects=True
+    )
+    assert b"Notification Body and Title is required when a Notification URL is used" in res.data
+

changedetectionio/tests/test_request.py

@@ -0,0 +1,211 @@
+import json
+import time
+from flask import url_for
+from . util import set_original_response, set_modified_response, live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+# Hard to just add more live server URLs when one test is already running (I think)
+# So we add our test here (was in a different file)
+def test_headers_in_request(client, live_server):
+    # Add our URL to the import page
+    test_url = url_for('test_headers', _external=True)
+
+    # Add the test URL twice, we will check
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;'
+
+
+    # Add some headers to a request
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": test_url,
+              "tag": "",
+              "fetch_backend": "html_requests",
+              "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+
+    # Give the thread time to pick up the first version
+    time.sleep(5)
+
+    # The service should echo back the request headers
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Flask will convert the header key to uppercase
+    assert b"Xxx:ooo" in res.data
+    assert b"Cool:yeah" in res.data
+
+    # The test call service will return the headers as the body
+    from html import escape
+    assert escape(cookie_header).encode('utf-8') in res.data
+
+    time.sleep(5)
+
+    # Re #137 -  Examine the JSON index file, it should have only one set of headers entered
+    watches_with_headers = 0
+    with open('test-datastore/url-watches.json') as f:
+        app_struct = json.load(f)
+        for uuid in app_struct['watching']:
+            if (len(app_struct['watching'][uuid]['headers'])):
+                watches_with_headers += 1
+
+    # Should be only one with headers set
+    assert watches_with_headers==1
+
+def test_body_in_request(client, live_server):
+    # Add our URL to the import page
+    test_url = url_for('test_body', _external=True)
+
+    # Add the test URL twice, we will check
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    body_value = 'Test Body Value'
+
+    # Attempt to add a body with a GET method
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": test_url,
+              "tag": "",
+              "method": "GET",
+              "fetch_backend": "html_requests",
+              "body": "invalid"},
+        follow_redirects=True
+    )
+    assert b"Body must be empty when Request Method is set to GET" in res.data
+
+    # Add a properly formatted body with a proper method
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": test_url,
+              "tag": "",
+              "method": "POST",
+              "fetch_backend": "html_requests",
+              "body": body_value},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Give the thread time to pick up the first version
+    time.sleep(5)
+
+    # The service should echo back the body
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Check if body returned contains the specified data
+    assert str.encode(body_value) in res.data
+
+    watches_with_body = 0
+    with open('test-datastore/url-watches.json') as f:
+        app_struct = json.load(f)
+        for uuid in app_struct['watching']:
+            if app_struct['watching'][uuid]['body']==body_value:
+                watches_with_body += 1
+
+    # Should be only one with body set
+    assert watches_with_body==1
+
+def test_method_in_request(client, live_server):
+    # Add our URL to the import page
+    test_url = url_for('test_method', _external=True)
+
+    # Add the test URL twice, we will check
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Attempt to add a method which is not valid
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "tag": "",
+            "fetch_backend": "html_requests",
+            "method": "invalid"},
+        follow_redirects=True
+    )
+    assert b"Not a valid choice" in res.data
+
+    # Add a properly formatted body
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "tag": "",
+            "fetch_backend": "html_requests",
+            "method": "PATCH"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Give the thread time to pick up the first version
+    time.sleep(5)
+
+    # The service should echo back the request verb
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # The test call service will return the verb as the body
+    assert b"PATCH" in res.data
+
+    time.sleep(5)
+
+    watches_with_method = 0
+    with open('test-datastore/url-watches.json') as f:
+        app_struct = json.load(f)
+        for uuid in app_struct['watching']:
+            if app_struct['watching'][uuid]['method'] == 'PATCH':
+                watches_with_method += 1
+
+    # Should be only one with method set to PATCH
+    assert watches_with_method == 1
+

changedetectionio/tests/test_trigger.py

@@ -2,30 +2,7 @@
 
 import time
 from flask import url_for
-from urllib.request import urlopen
-import pytest
-
-
-# Unit test of the stripper
-# Always we are dealing in utf-8
-def test_strip_text_func():
-    from backend import fetch_site_status
-
-    test_content = """
-    Some content
-    is listed here
-
-    but sometimes we want to remove the lines.
-
-    but not always."""
-
-    ignore_lines = ["sometimes"]
-
-    fetcher = fetch_site_status.perform_site_check(datastore=False)
-    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
-
-    assert b"sometimes" not in stripped_content
-    assert b"Some content" in stripped_content
+from . util import live_server_setup
 
 
 def set_original_ignore_response():
@@ -40,7 +17,7 @@ def set_original_ignore_response():
 
     """
 
-    with open("test-datastore/output.txt", "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
@@ -56,32 +33,34 @@ def set_modified_original_ignore_response():
 
     """
 
-    with open("test-datastore/output.txt", "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-# Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
-def set_modified_ignore_response():
+def set_modified_with_trigger_text_response():
     test_return_data = """<html>
        <body>
-     Some initial text</br>
+     Some NEW nice initial text</br>
      <p>Which is across multiple lines</p>
-     <P>ZZZZZ</P>
      </br>
+     foobar123
+     <br/>
      So let's see what happens.  </br>
      </body>
      </html>
 
     """
 
-    with open("test-datastore/output.txt", "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
-def test_check_ignore_text_functionality(client, live_server):
-    sleep_time_for_fetch_thread = 3
+def test_trigger_functionality(client, live_server):
 
-    ignore_text = "XXXXX\nYYYYY\nZZZZZ"
+    live_server_setup(live_server)
+
+    sleep_time_for_fetch_thread = 3
+    trigger_text = "foobar123"
     set_original_ignore_response()
 
     # Give the endpoint time to spin up
@@ -106,7 +85,9 @@ def test_check_ignore_text_functionality(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"ignore-text": ignore_text, "url": test_url, "tag": "", "headers": ""},
+        data={"trigger_text": trigger_text,
+              "url": test_url,
+              "fetch_backend": "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -115,7 +96,7 @@ def test_check_ignore_text_functionality(client, live_server):
     res = client.get(
         url_for("edit_page", uuid="first"),
     )
-    assert bytes(ignore_text.encode('utf-8')) in res.data
+    assert bytes(trigger_text.encode('utf-8')) in res.data
 
     # Trigger a check
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
@@ -129,7 +110,7 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b'/test-endpoint' in res.data
 
     #  Make a change
-    set_modified_ignore_response()
+    set_modified_original_ignore_response()
 
     # Trigger a check
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
@@ -139,14 +120,12 @@ def test_check_ignore_text_functionality(client, live_server):
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
-    assert b'/test-endpoint' in res.data
 
     # Just to be sure.. set a regular modified change..
-    set_modified_original_ignore_response()
+    time.sleep(sleep_time_for_fetch_thread)
+    set_modified_with_trigger_text_response()
+
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
-
-    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_trigger_regex.py

@@ -0,0 +1,81 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+
+def test_trigger_regex_functionality(client, live_server):
+
+    live_server_setup(live_server)
+
+    sleep_time_for_fetch_thread = 3
+
+    set_original_ignore_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (just a new one shouldnt have anything)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+    ### test regex
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"trigger_text": '/something \d{3}/',
+              "url": test_url,
+              "fetch_backend": "html_requests"},
+        follow_redirects=True
+    )
+
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("some new noise")
+
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (nothing should match the regex)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("regex test123<br/>\nsomething 123")
+
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data

changedetectionio/tests/test_trigger_regex_with_filter.py

@@ -0,0 +1,84 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+
+def test_trigger_regex_functionality(client, live_server):
+
+    live_server_setup(live_server)
+
+    sleep_time_for_fetch_thread = 3
+
+    set_original_ignore_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (just a new one shouldnt have anything)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+    ### test regex with filter
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"trigger_text": "/cool.stuff\d/",
+              "url": test_url,
+              "css_filter": '#in-here',
+              "fetch_backend": "html_requests"},
+        follow_redirects=True
+    )
+
+    # Check that we have the expected text.. but it's not in the css filter we want
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("<html>some new noise with cool stuff2 ok</html>")
+
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (nothing should match the regex and filter)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("<html>some new noise with <span id=in-here>cool stuff6</span> ok</html>")
+
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+

changedetectionio/tests/test_watch_fields_storage.py

@@ -0,0 +1,150 @@
+import time
+from flask import url_for
+from urllib.request import urlopen
+from . util import set_original_response, set_modified_response, live_server_setup
+
+
+def test_check_watch_field_storage(client, live_server):
+    set_original_response()
+    live_server_setup(live_server)
+
+    test_url = "http://somerandomsitewewatch.com"
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={ "notification_urls": "json://myapi.com",
+               "minutes_between_check": 126,
+               "css_filter" : ".fooclass",
+               "title" : "My title",
+               "ignore_text" : "ignore this",
+               "url": test_url,
+               "tag": "woohoo",
+               "headers": "curl:foo",
+               'fetch_backend': "html_requests"
+               },
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"json://myapi.com" in res.data
+    assert b"126" in res.data
+    assert b".fooclass" in res.data
+    assert b"My title" in res.data
+    assert b"ignore this" in res.data
+    assert b"http://somerandomsitewewatch.com" in res.data
+    assert b"woohoo" in res.data
+    assert b"curl: foo" in res.data
+
+
+
+# Re https://github.com/dgtlmoon/changedetection.io/issues/110
+def test_check_recheck_global_setting(client, live_server):
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 1566,
+               'fetch_backend': "html_requests"
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Now add a record
+
+    test_url = "http://somerandomsitewewatch.com"
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Now visit the edit page, it should have the default minutes
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should show the default minutes
+    assert b"change to another value if you want to be specific" in res.data
+    assert b"1566" in res.data
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 222,
+                'fetch_backend': "html_requests"
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should show the default minutes
+    assert b"change to another value if you want to be specific" in res.data
+    assert b"222" in res.data
+
+    # Now change it specifically, it should show the new minutes
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url,
+              "minutes_between_check": 55,
+              'fetch_backend': "html_requests"
+              },
+        follow_redirects=True
+    )
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"55" in res.data
+
+    # Now submit an empty field, it should give back the default global minutes
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 666,
+                'fetch_backend': "html_requests"
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url,
+              "minutes_between_check": "",
+              'fetch_backend': "html_requests"
+              },
+        follow_redirects=True
+    )
+
+    assert b"Updated watch." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"666" in res.data
+

changedetectionio/tests/test_xpath_selector.py

@@ -0,0 +1,118 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+from ..html_tools import *
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div class="sametext">Some text thats the same</div>
+     <div class="changetext">Some text that will change</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  THIS CHANGES AND SHOULDNT TRIGGER A CHANGE</br>
+     <div class="sametext">Some text thats the same</div>
+     <div class="changetext">Some new text</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+def test_check_markup_xpath_filter_restriction(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    xpath_filter = "//*[contains(@class, 'sametext')]"
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # view it/reset state back to viewed
+    client.get(url_for("diff_history_page", uuid="first"), follow_redirects=True)
+
+    #  Make a change
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+def test_xpath_validation(client, live_server):
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"is not a valid XPath expression" in res.data

changedetectionio/tests/unit/__init__.py

@@ -0,0 +1 @@
+"""Unit tests for the app."""

changedetectionio/tests/unit/test-content/README.md

@@ -0,0 +1,5 @@
+# What is this?
+This is test content for the python diff engine, we use the JS interface for the front end, because you can explore 
+differences in words etc, but we use (at the moment) the python difflib engine.
+
+This content `before.txt` and `after.txt` is for unit testing

changedetectionio/tests/unit/test-content/after.txt

@@ -0,0 +1,6 @@
+After twenty years, as cursed as I may be
+for having learned computerese,
+I continue to examine bits, bytes and words
+xok
+and insure that I'm one of those computer nerds.
+and something new

changedetectionio/tests/unit/test-content/before.txt

@@ -0,0 +1,5 @@
+After twenty years, as cursed as I may be
+for having learned computerese,
+I continue to examine bits, bytes and words
+ok
+and insure that I'm one of those computer nerds.

changedetectionio/tests/unit/test_notification_diff.py

@@ -0,0 +1,25 @@
+#!/usr/bin/python3
+
+# run from dir above changedetectionio/ dir
+# python3 -m unittest changedetectionio.tests.unit.test_notification_diff
+
+import unittest
+import os
+
+from changedetectionio import diff
+
+# mostly
+class TestDiffBuilder(unittest.TestCase):
+
+    def test_expected_diff_output(self):
+        base_dir=os.path.dirname(__file__)
+        output = diff.render_diff(base_dir+"/test-content/before.txt", base_dir+"/test-content/after.txt")
+        output = output.split("\n")
+        self.assertIn("(changed) ok", output)
+        self.assertIn("(-> into) xok", output)
+        self.assertIn("(added) and something new", output)
+
+        # @todo test blocks of changed, blocks of added, blocks of removed
+
+if __name__ == '__main__':
+    unittest.main()

changedetectionio/tests/util.py

@@ -0,0 +1,116 @@
+#!/usr/bin/python3
+
+
+def set_original_response():
+    test_return_data = """<html>
+    <head><title>head title</title></head>
+    <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """<html>
+    <head><title>modified head title</title></head>
+    <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+def live_server_setup(live_server):
+
+
+    @live_server.app.route('/test-endpoint')
+    def test_endpoint():
+        # Tried using a global var here but didn't seem to work, so reading from a file instead.
+        with open("test-datastore/endpoint-content.txt", "r") as f:
+            return f.read()
+
+    @live_server.app.route('/test-endpoint-json')
+    def test_endpoint_json():
+
+        from flask import make_response
+
+        with open("test-datastore/endpoint-content.txt", "r") as f:
+            resp = make_response(f.read())
+            resp.headers['Content-Type'] = 'application/json'
+            return resp
+
+    @live_server.app.route('/test-403')
+    def test_endpoint_403_error():
+
+        from flask import make_response
+        resp = make_response('', 403)
+        return resp
+
+    # Just return the headers in the request
+    @live_server.app.route('/test-headers')
+    def test_headers():
+
+        from flask import request
+        output= []
+
+        for header in request.headers:
+             output.append("{}:{}".format(str(header[0]),str(header[1])   ))
+
+        return "\n".join(output)
+
+    # Just return the body in the request
+    @live_server.app.route('/test-body', methods=['POST', 'GET'])
+    def test_body():
+
+        from flask import request
+
+        return request.data
+
+    # Just return the verb in the request
+    @live_server.app.route('/test-method', methods=['POST', 'GET', 'PATCH'])
+    def test_method():
+
+        from flask import request
+
+        return request.method
+
+    # Where we POST to as a notification
+    @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET'])
+    def test_notification_endpoint():
+        from flask import request
+
+        with open("test-datastore/notification.txt", "wb") as f:
+            # Debug method, dump all POST to file also, used to prove #65
+            data = request.stream.read()
+            if data != None:
+                f.write(data)
+
+        print("\n>> Test notification endpoint was hit.\n")
+        return "Text was set"
+
+
+    # Just return the verb in the request
+    @live_server.app.route('/test-basicauth', methods=['GET'])
+    def test_basicauth_method():
+
+        from flask import request
+        auth = request.authorization
+        ret = " ".join([auth.username, auth.password, auth.type])
+        return ret
+
+    live_server.start()

changedetectionio/update_worker.py

@@ -0,0 +1,148 @@
+import threading
+import queue
+import time
+
+# A single update worker
+#
+# Requests for checking on a single site(watch) from a queue of watches
+# (another process inserts watches into the queue that are time-ready for checking)
+
+
+class update_worker(threading.Thread):
+    current_uuid = None
+
+    def __init__(self, q, notification_q, app, datastore, *args, **kwargs):
+        self.q = q
+        self.app = app
+        self.notification_q = notification_q
+        self.datastore = datastore
+        super().__init__(*args, **kwargs)
+
+    def run(self):
+        from changedetectionio import fetch_site_status
+
+        update_handler = fetch_site_status.perform_site_check(datastore=self.datastore)
+
+        while not self.app.config.exit.is_set():
+
+            try:
+                uuid = self.q.get(block=False)
+            except queue.Empty:
+                pass
+
+            else:
+                self.current_uuid = uuid
+                from changedetectionio import content_fetcher
+
+                if uuid in list(self.datastore.data['watching'].keys()):
+
+                    changed_detected = False
+                    contents = ""
+                    update_obj= {}
+                    now = time.time()
+
+                    try:
+
+                        changed_detected, update_obj, contents = update_handler.run(uuid)
+
+                        # Re #342
+                        # In Python 3, all strings are sequences of Unicode characters. There is a bytes type that holds raw bytes.
+                        # We then convert/.decode('utf-8') for the notification etc
+                        if not isinstance(contents, (bytes, bytearray)):
+                            raise Exception("Error - returned data from the fetch handler SHOULD be bytes")
+
+
+                    except PermissionError as e:
+                        self.app.logger.error("File permission error updating", uuid, str(e))
+                    except content_fetcher.EmptyReply as e:
+                        # Some kind of custom to-str handler in the exception handler that does this?
+                        err_text = "EmptyReply: Status Code {}".format(e.status_code)
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+                                                                           'last_check_status': e.status_code})
+                    except Exception as e:
+                        self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
+
+                    else:
+                        try:
+                            watch = self.datastore.data['watching'][uuid]
+                            fname = "" # Saved history text filename
+
+                            # For the FIRST time we check a site, or a change detected, save the snapshot.
+                            if changed_detected or not watch['last_checked']:
+                                # A change was detected
+                                fname = self.datastore.save_history_text(watch_uuid=uuid, contents=contents)
+                                # Should always be keyed by string(timestamp)
+                                self.datastore.update_watch(uuid, {"history": {str(round(time.time())): fname}})
+
+                            # Generally update anything interesting returned
+                            self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
+
+                            # A change was detected
+                            if changed_detected:
+                                n_object = {}
+                                print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))
+
+                                # Notifications should only trigger on the second time (first time, we gather the initial snapshot)
+                                if len(watch['history']) > 1:
+
+                                    dates = list(watch['history'].keys())
+                                    # Convert to int, sort and back to str again
+                                    # @todo replace datastore getter that does this automatically
+                                    dates = [int(i) for i in dates]
+                                    dates.sort(reverse=True)
+                                    dates = [str(i) for i in dates]
+
+                                    prev_fname = watch['history'][dates[1]]
+
+
+                                    # Did it have any notification alerts to hit?
+                                    if len(watch['notification_urls']):
+                                        print(">>> Notifications queued for UUID from watch {}".format(uuid))
+                                        n_object['notification_urls'] = watch['notification_urls']
+                                        n_object['notification_title'] = watch['notification_title']
+                                        n_object['notification_body'] = watch['notification_body']
+                                        n_object['notification_format'] = watch['notification_format']
+
+                                    # No? maybe theres a global setting, queue them all
+                                    elif len(self.datastore.data['settings']['application']['notification_urls']):
+                                        print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(uuid))
+                                        n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
+                                        n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title']
+                                        n_object['notification_body'] = self.datastore.data['settings']['application']['notification_body']
+                                        n_object['notification_format'] = self.datastore.data['settings']['application']['notification_format']
+                                    else:
+                                        print(">>> NO notifications queued, watch and global notification URLs were empty.")
+
+                                    # Only prepare to notify if the rules above matched
+                                    if 'notification_urls' in n_object:
+                                        # HTML needs linebreak, but MarkDown and Text can use a linefeed
+                                        if n_object['notification_format'] == 'HTML':
+                                            line_feed_sep = "</br>"
+                                        else:
+                                            line_feed_sep = "\n"
+
+                                        from changedetectionio import diff
+                                        n_object.update({
+                                            'watch_url': watch['url'],
+                                            'uuid': uuid,
+                                            'current_snapshot': contents.decode('utf-8'),
+                                            'diff_full': diff.render_diff(prev_fname, fname, line_feed_sep=line_feed_sep),
+                                            'diff': diff.render_diff(prev_fname, fname, True, line_feed_sep=line_feed_sep)
+                                        })
+
+                                        self.notification_q.put(n_object)
+
+                        except Exception as e:
+                            # Catch everything possible here, so that if a worker crashes, we don't lose it until restart!
+                            print("!!!! Exception in update_worker !!!\n", e)
+
+                    finally:
+                        # Always record that we atleast tried
+                        self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
+                                                                           'last_checked': round(time.time())})
+
+                self.current_uuid = None  # Done
+                self.q.task_done()
+
+            self.app.config.exit.wait(1)

docker-compose.yml

@@ -0,0 +1,65 @@
+version: '2'
+services:
+    changedetection.io:
+      image: ghcr.io/dgtlmoon/changedetection.io
+      container_name: changedetection.io
+      hostname: changedetection.io
+      volumes:
+        - changedetection-data:/datastore
+
+  #    environment:
+  #        Default listening port, can also be changed with the -p option
+  #      - PORT=5000
+
+  #      - PUID=1000
+  #      - PGID=1000
+  #
+  #       Alternative WebDriver/selenium URL, do not use "'s or 's!
+  #      - WEBDRIVER_URL=http://browser-chrome:4444/wd/hub
+  #
+  #       WebDriver proxy settings webdriver_proxyType, webdriver_ftpProxy, webdriver_httpProxy, webdriver_noProxy,
+  #                                webdriver_proxyAutoconfigUrl, webdriver_sslProxy, webdriver_autodetect,
+  #                                webdriver_socksProxy, webdriver_socksUsername, webdriver_socksVersion, webdriver_socksPassword
+  #
+  #             https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.proxy
+  #
+  #        Plain requsts - proxy support example.
+  #      - HTTP_PROXY=socks5h://10.10.1.10:1080
+  #      - HTTPS_PROXY=socks5h://10.10.1.10:1080
+  #
+  #        An exclude list (useful for notification URLs above) can be specified by with
+  #      - NO_PROXY="localhost,192.168.0.0/24"
+  #
+  #        Base URL of your changedetection.io install (Added to the notification alert)
+  #      - BASE_URL=https://mysite.com
+
+  #        Respect proxy_pass type settings, `proxy_set_header Host "localhost";` and `proxy_set_header X-Forwarded-Prefix /app;`
+  #        More here https://github.com/dgtlmoon/changedetection.io/wiki/Running-changedetection.io-behind-a-reverse-proxy-sub-directory
+  #      - USE_X_SETTINGS=1
+
+      # Comment out ports: when using behind a reverse proxy , enable networks: etc.
+      ports:
+        - 5000:5000
+      restart: unless-stopped
+
+     # Used for fetching pages via WebDriver+Chrome where you need Javascript support.
+     # Now working on arm64 (needs testing on rPi - tested on Oracle ARM instance)
+     # replace image with seleniarm/standalone-chromium:4.0.0-20211213
+
+#    browser-chrome:
+#        hostname: browser-chrome
+#        image: selenium/standalone-chrome-debug:3.141.59
+#        environment:
+#            - VNC_NO_PASSWORD=1
+#            - SCREEN_WIDTH=1920
+#            - SCREEN_HEIGHT=1080
+#            - SCREEN_DEPTH=24
+#        volumes:
+#            # Workaround to avoid the browser crashing inside a docker container
+#            # See https://github.com/SeleniumHQ/docker-selenium#quick-start
+#            - /dev/shm:/dev/shm
+#        restart: unless-stopped
+
+volumes:
+  changedetection-data:
+

heroku.yml

@@ -0,0 +1,2 @@
+run:
+  changedetection: python3 ./changedetection.py -C -d ./datastore -p $PORT

requirements-dev.txt

@@ -0,0 +1,2 @@
+pytest ~=6.2
+pytest-flask ~=1.2

requirements.txt

@@ -1,13 +1,36 @@
-chardet==2.3.0
-flask~= 1.0
-pytest ~=6.2
-pytest-flask ~=1.1
-eventlet ~= 0.30
-requests ~= 2.15
+flask~= 2.0
+
+eventlet>=0.31.0
 validators
 timeago ~=1.0
-inscriptis ~= 1.1
+inscriptis ~= 1.2
 feedgen ~= 0.9
 flask-login ~= 0.5
 pytz
-urllib3
+
+# Set these versions together to avoid a RequestsDependencyWarning
+requests[socks] ~= 2.26
+urllib3 > 1.26
+chardet > 2.3.0
+
+wtforms ~= 2.3.3
+jsonpath-ng ~= 1.5.3
+
+# Notification library
+apprise ~= 0.9
+
+# apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
+paho-mqtt
+
+# Pinned version of cryptography otherwise
+# ERROR: Could not build wheels for cryptography which use PEP 517 and cannot be installed directly
+cryptography ~= 3.4
+
+# Used for CSS filtering
+bs4
+
+# XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.
+lxml
+
+# 3.141 was missing socksVersion, 3.150 was not in pypi, so we try 4.1.0
+selenium ~= 4.1.0

runtime.txt

@@ -0,0 +1 @@
+python-3.8.12

setup.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+import codecs
+import os.path
+import re
+import sys
+
+from setuptools import setup, find_packages
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+def read(*parts):
+    return codecs.open(os.path.join(here, *parts), 'r').read()
+
+
+def find_version(*file_paths):
+    version_file = read(*file_paths)
+    version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
+                              version_file, re.M)
+    if version_match:
+        return version_match.group(1)
+    raise RuntimeError("Unable to find version string.")
+
+
+install_requires = open('requirements.txt').readlines()
+
+setup(
+    name='changedetection.io',
+    version=find_version("changedetectionio", "__init__.py"),
+    description='Website change detection and monitoring service',
+    long_description=open('README-pip.md').read(),
+    long_description_content_type='text/markdown',
+    keywords='website change monitor for changes notification change detection '
+             'alerts tracking website tracker change alert website and monitoring',
+    zip_safe=False,
+    entry_points={"console_scripts": ["changedetection.io=changedetection:main"]},
+    author='dgtlmoon',
+    url='https://changedetection.io',
+    scripts=['changedetection.py'],
+    packages=['changedetectionio'],
+    include_package_data=True,
+    install_requires=install_requires,
+    license="Apache License 2.0",
+    python_requires=">= 3.6",
+    classifiers=['Intended Audience :: Customer Service',
+                 'Intended Audience :: Developers',
+                 'Intended Audience :: Education',
+                 'Intended Audience :: End Users/Desktop',
+                 'Intended Audience :: Financial and Insurance Industry',
+                 'Intended Audience :: Healthcare Industry',
+                 'Intended Audience :: Information Technology',
+                 'Intended Audience :: Legal Industry',
+                 'Intended Audience :: Manufacturing',
+                 'Intended Audience :: Other Audience',
+                 'Intended Audience :: Religion',
+                 'Intended Audience :: Science/Research',
+                 'Intended Audience :: System Administrators',
+                 'Intended Audience :: Telecommunications Industry',
+                 'Topic :: Education',
+                 'Topic :: Internet',
+                 'Topic :: Internet :: WWW/HTTP :: Indexing/Search',
+                 'Topic :: Internet :: WWW/HTTP :: Site Management',
+                 'Topic :: Internet :: WWW/HTTP :: Site Management :: Link Checking',
+                 'Topic :: Internet :: WWW/HTTP :: Browsers',
+                 'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
+                 'Topic :: Office/Business',
+                 'Topic :: Other/Nonlisted Topic',
+                 'Topic :: Scientific/Engineering :: Information Analysis',
+                 'Topic :: Text Processing :: Markup :: HTML',
+                 'Topic :: Utilities'
+                 ],
+)