Simplify scrub operation

Co-authored-by: Ara Hayrabedian <ara.hayrabedian@gmail.com>

.dockerignore

@@ -0,0 +1,2 @@
+.git
+.github

.github/FUNDING.yml

@@ -1,12 +1,3 @@
 # These are supported funding model platforms
 
 github: dgtlmoon
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,41 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Version**
+In the top right area: 0....
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS] 
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,23 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+**Version and OS**
+For example, 0.123 on linux/docker
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe the use-case and give concrete real-world examples**
+Attach any HTML/JSON, give links to sites, screenshots etc, we are not mind readers
+
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,62 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  schedule:
+    - cron: '27 9 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/containers.yml

@@ -0,0 +1,129 @@
+name: Build and push containers
+
+on:
+  # Automatically triggered by a testing workflow passing, but this is only checked when it lands in the `master`/default branch
+#  workflow_run:
+#    workflows: ["ChangeDetection.io Test"]
+#    branches: [master]
+#    tags: ['0.*']
+#    types: [completed]
+
+  # Or a new tagged release
+  release:
+    types: [published, edited]
+
+  push:
+    branches:
+      - master
+
+jobs:
+  metadata:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Show metadata
+      run: |
+        echo SHA ${{ github.sha }}
+        echo github.ref:  ${{ github.ref }}
+        echo github_ref: $GITHUB_REF
+        echo Event name: ${{ github.event_name }}
+        echo Ref ${{ github.ref }}
+        echo c: ${{ github.event.workflow_run.conclusion }}
+        echo r: ${{ github.event.workflow_run }}
+        echo tname: "${{ github.event.release.tag_name }}"
+        echo headbranch: -${{ github.event.workflow_run.head_branch }}-
+        set
+
+  build-push-containers:
+    runs-on: ubuntu-latest
+    # If the testing workflow has a success, then we build to :latest
+    # Or if we are in a tagged release scenario.
+    if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+      - name: Create release metadata
+        run: |
+          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > changedetectionio/source.txt
+          echo ${{ github.ref }} > changedetectionio/tag.txt
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: all
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to Docker Hub Container Registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+          version: latest
+          driver-opts: image=moby/buildkit:master
+
+      # master always builds :latest
+      - name: Build and push :latest
+        id: docker_build
+        if: ${{ github.ref }} == "refs/heads/master"
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest,ghcr.io/${{ github.repository }}:latest
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+
+      # A new tagged release is required, which builds :tag
+      - name: Build and push :tag
+        id: docker_build_tag_release
+        if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }},ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+
+      - name: Image digest
+        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+

.github/workflows/pypi.yml

@@ -0,0 +1,44 @@
+name: PyPi Test and Push tagged release
+
+# Triggers the workflow on push or pull request events
+on:
+  workflow_run:
+    workflows: ["ChangeDetection.io Test"]
+    tags: '*.*'
+    types: [completed]
+
+
+jobs:
+  test-build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+#      - name: Install dependencies
+#        run: |
+#          python -m pip install --upgrade pip
+#          pip install flake8 pytest
+#          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+#          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+      - name: Test that pip builds without error
+        run: |
+          pip3 --version
+          python3 -m pip install wheel
+          python3 setup.py bdist_wheel
+          python3 -m pip install dist/changedetection.io-*-none-any.whl --force
+          changedetection.io -d /tmp -p 10000 &
+          sleep 3
+          curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+          killall -9 changedetection.io
+
+      # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
+      # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
+
+      # https://github.com/docker/buildx/issues/495#issuecomment-918925854
+#if: ${{ github.event_name == 'release'}}

.github/workflows/test-only.yml

@@ -0,0 +1,45 @@
+name: ChangeDetection.io Test
+
+# Triggers the workflow on push or pull request events
+on: [push, pull_request]
+
+jobs:
+  test-build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Show env vars
+        run: set
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Unit tests
+        run: |
+          python3 -m unittest changedetectionio.tests.unit.test_notification_diff
+
+      - name: Test with pytest
+        run: |
+          # Each test is totally isolated and performs its own cleanup/reset
+          cd changedetectionio; ./run_all_tests.sh
+
+      # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
+      # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
+
+      # https://github.com/docker/buildx/issues/495#issuecomment-918925854

.gitignore

@@ -2,4 +2,11 @@ __pycache__
 .idea
 *.pyc
 datastore/url-watches.json
-datastore/*
+datastore/*
+__pycache__
+.pytest_cache
+build
+dist
+venv
+*.egg-info*
+.vscode/settings.json

CONTRIBUTING.md

@@ -0,0 +1,15 @@
+Contributing is always welcome!
+
+I am no professional flask developer, if you know a better way that something can be done, please let me know!
+
+Otherwise, it's always best to PR into the `dev` branch.
+
+Please be sure that all new functionality has a matching test!
+
+Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notifications.py` for example
+
+```
+pip3 install -r requirements-dev
+```
+
+this is from https://github.com/dgtlmoon/changedetection.io/blob/master/requirements-dev.txt

Dockerfile

@@ -1,17 +1,61 @@
-FROM python:3.8-slim
-COPY requirements.txt /tmp/requirements.txt
-RUN pip3 install -r /tmp/requirements.txt
+# pip dependencies install stage
+FROM python:3.8-slim as builder
 
-COPY backend /app
-WORKDIR /app
+# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
+ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl-dev \
+    libffi-dev \
+    gcc \
+    libc-dev \
+    libxslt-dev \
+    zlib1g-dev \
+    g++
+
+RUN mkdir /install
+WORKDIR /install
+
+COPY requirements.txt /requirements.txt
+
+RUN pip install --target=/dependencies -r /requirements.txt
+
+# Final image stage
+FROM python:3.8-slim
+
+# Actual packages needed at runtime, usually due to the notification (apprise) backend
+# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
+ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
+
+# Re #93, #73, excluding rustc (adds another 430Mb~)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl-dev \
+    libffi-dev \
+    gcc \
+    libc-dev \
+    libxslt-dev \
+    zlib1g-dev \
+    g++
 
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
-
 ENV PYTHONUNBUFFERED=1
 
 RUN [ ! -d "/datastore" ] && mkdir /datastore
 
-CMD [ "python", "./backend.py" ]
+# Re #80, sets SECLEVEL=1 in openssl.conf to allow monitoring sites with weak/old cipher suites
+RUN sed -i 's/^CipherString = .*/CipherString = DEFAULT@SECLEVEL=1/' /etc/ssl/openssl.cnf
 
+# Copy modules over to the final image and add their dir to PYTHONPATH
+COPY --from=builder /dependencies /usr/local
+ENV PYTHONPATH=/usr/local
 
+EXPOSE 5000
 
+# The actual flask app
+COPY changedetectionio /app/changedetectionio
+# The eventlet server wrapper
+COPY changedetection.py /app/changedetection.py
+
+WORKDIR /app
+
+CMD [ "python", "./changedetection.py" , "-d", "/datastore"]

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

MANIFEST.in

@@ -0,0 +1,6 @@
+recursive-include changedetectionio/templates *
+recursive-include changedetectionio/static *
+include changedetection.py
+global-exclude *.pyc
+global-exclude node_modules
+global-exclude venv

Procfile

@@ -0,0 +1 @@
+web: python3 ./changedetection.py -C -d ./datastore -p $PORT

README-pip.md

@@ -0,0 +1,71 @@
+#  changedetection.io
+![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
+<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
+  <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
+</a>
+<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
+  <img src="https://img.shields.io/github/v/release/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
+</a>
+
+## Self-hosted open source change monitoring of web pages.
+
+_Know when web pages change! Stay ontop of new information!_ 
+
+Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
+
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
+
+#### Example use cases
+
+Know when ...
+
+- Government department updates (changes are often only on their websites)
+- Local government news (changes are often only on their websites)
+- New software releases, security advisories when you're not on their mailing list.
+- Festivals with changes
+- Realestate listing changes
+- COVID related news from government websites
+- Detect and monitor changes in JSON API responses 
+- API monitoring and alerting
+
+**Get monitoring now!**
+
+```bash
+$ pip3 install changedetection.io   
+```
+
+Specify a target for the *datastore path* with `-d` (required) and a *listening port* with `-p` (defaults to `5000`)
+
+```bash
+$ changedetection.io -d /path/to/empty/data/dir -p 5000
+```
+
+
+Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
+
+### Features
+- Website monitoring
+- Change detection of content and analyses
+- Filters on change (Select by CSS or JSON)
+- Triggers (Wait for text, wait for regex)
+- Notification support
+- JSON API Monitoring
+- Parse JSON embedded in HTML
+- (Reverse) Proxy support
+- Javascript support via WebDriver
+- RaspberriPi (arm v6/v7/64 support)
+
+See https://github.com/dgtlmoon/changedetection.io for more information.
+
+
+
+### Support us
+
+Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
+
+Please support us, even small amounts help a LOT.
+
+BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />

README.md

@@ -1,42 +1,193 @@
 #  changedetection.io
+[![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)
 
-## Self-hosted change monitoring of web pages.
+![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
 
-_Know when web pages change! Stay ontop of new information!_
+## Self-Hosted, Open Source, Change Monitoring of Web Pages
+
+_Know when web pages change! Stay ontop of new information!_ 
+
+Live your data-life *pro-actively* instead of *re-actively*.
+
+Free, Open-source web page monitoring, notification and change detection. Don't have time? [**Try our $6.99/month subscription - unlimited checks and watches!**](https://lemonade.changedetection.io/start)
+
+
+[<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start)
+
+
+**Get your own private instance now! Let us host it for you!**
+
+[**Try our $6.99/month subscription - unlimited checks and watches!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
+
+
+
+- Automatic Updates, Automatic Backups, No Heroku "paused application", don't miss a change!
+- Javascript browser included
+- Unlimited checks and watches!
 
 
 #### Example use cases
 
-Know when ...
-
-- Government department updates (changes are often only on their websites)
-- Local government news (changes are often only on their websites)
-- New software releases 
+- Products and services have a change in pricing
+- Governmental department updates (changes are often only on their websites)
+- New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
 - Realestate listing changes
+- COVID related news from government websites
+- University/organisation news from their website
+- Detect and monitor changes in JSON API responses 
+- JSON API monitoring and alerting
+- Changes in legal and other documents
+- Trigger API calls via notifications when text appears on a website
+- Glue together APIs using the JSON filter and JSON notifications
+- Create RSS feeds based on changes in web content
+- Monitor HTML source code for unexpected changes, strengthen your PCI compliance
+- You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
+
+_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver!</a>_
+
+## Screenshots
+
+Examining differences in content.
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
+
+Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 
 
-Get monitoring now! super simple, one command!
+## Installation
+
+### Docker
+
+With Docker composer, just clone this repository and..
+```bash
+$ docker-compose up -d
+```
+Docker standalone
+```bash
+$ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
+```
+
+### Windows
+
+See the install instructions at the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Microsoft-Windows
+
+### Python Pip
+
+Check out our pypi page https://pypi.org/project/changedetection.io/
+
+```bash
+$ pip3 install changedetection.io
+$ changedetection.io -d /path/to/empty/data/dir -p 5000
+```
+
+Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
+
+_Now with per-site configurable support for using a fast built in HTTP fetcher or use a Chrome based fetcher for monitoring of JavaScript websites!_
+
+## Updating changedetection.io
+
+### Docker
+```
+docker pull dgtlmoon/changedetection.io
+docker kill $(docker ps -a|grep changedetection.io|awk '{print $1}')
+docker rm $(docker ps -a|grep changedetection.io|awk '{print $1}')
+docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
+```
+
+### docker-compose
+
+```bash
+docker-compose pull && docker-compose up -d
+```
+
+See the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki
+
+
+## Filters
+XPath, JSONPath and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools.
+
+(We support LXML re:test, re:math and re:replace.)
+
+## Notifications
+
+ChangeDetection.io supports a massive amount of notifications (including email, office365, custom APIs, etc) when a web-page has a change detected thanks to the <a href="https://github.com/caronc/apprise">apprise</a> library.
+Simply set one or more notification URL's in the _[edit]_ tab of that watch.
+
+Just some examples
+
+    discord://webhook_id/webhook_token
+    flock://app_token/g:channel_id
+    gitter://token/room
+    gchat://workspace/key/token
+    msteams://TokenA/TokenB/TokenC/
+    o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+    rocket://user:password@hostname/#Channel
+    mailto://user:pass@example.com?to=receivingAddress@example.com
+    json://someserver.com/custom-api
+    syslog://
+ 
+<a href="https://github.com/caronc/apprise#popular-notification-services">And everything else in this list!</a>
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
+
+Now you can also customise your notification content!
+
+## JSON API Monitoring
+
+Detect changes and monitor data in JSON API's by using the built-in JSONPath selectors as a filter / selector.
+
+![image](https://user-images.githubusercontent.com/275001/125165842-0ce01980-e1dc-11eb-9e73-d8137dd162dc.png)
+
+This will re-parse the JSON and apply formatting to the text, making it super easy to monitor and detect changes in JSON API results
+
+![image](https://user-images.githubusercontent.com/275001/125165995-d9ea5580-e1dc-11eb-8030-f0deced2661a.png)
+
+### Parse JSON embedded in HTML!
+
+When you enable a `json:` filter, you can even automatically extract and parse embedded JSON inside a HTML page! Amazingly handy for sites that build content based on JSON, such as many e-commerce websites. 
 
 ```
-$ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore dgtlmoon/changedetection.io
+<html>
+...
+<script type="application/ld+json">
+  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","price": 23.50 }
+</script>
 ```  
 
-Now visit http://127.0.0.1:5000 , You should now be able to access the UI.
+`json:$.price` would give `23.50`, or you can extract the whole structure
 
-  
+## Proxy configuration
 
-![Self-hoster web page change monitoring application screenshot](screenshot.png?raw=true "Self-hosted web page change monitoring screenshot")
+See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration
+
+## Raspberry Pi support?
+
+Raspberry Pi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! See the wiki for [details](https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver)
 
 
-### Future plans
+## Support us
 
-- Greater configuration of check interval times, page request headers.
-- ~~General options for timeout, default headers~~
-- On change detection, callout to another API (handy for notices/issue trackers)
-- ~~Explore the differences that were detected~~ 
-- Add more options to explore versions of differences
-- Use a graphic/rendered page difference instead of text (see the experimental `selenium-screenshot-diff` branch)
+Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 
- 
-Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
+
+Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://lemonade.changedetection.io/start) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
+
+Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)
+
+Or BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />
+
+## Commercial Support
+
+I offer commercial support, this software is depended on by network security, aerospace , data-science and data-journalist professionals just to name a few, please reach out at dgtlmoon@gmail.com for any enquiries, I am more than glad to work with your organisation to further the possibilities of what can be done with changedetection.io
+
+
+[release-shield]: https://img.shields.io:/github/v/release/dgtlmoon/changedetection.io?style=for-the-badge
+[docker-pulls]: https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io?style=for-the-badge
+[test-shield]: https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master
+
+[license-shield]: https://img.shields.io/github/license/dgtlmoon/changedetection.io.svg?style=for-the-badge
+[release-link]: https://github.com/dgtlmoon.com/changedetection.io/releases
+[docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io

app.json

@@ -0,0 +1,21 @@
+{
+  "name": "ChangeDetection.io",
+  "description": "The best and simplest self-hosted open source website change detection monitoring and notification service.",
+  "keywords": [
+    "changedetection",
+    "website monitoring"
+  ],
+  "repository": "https://github.com/dgtlmoon/changedetection.io",
+  "success_url": "/",
+  "scripts": {
+  },
+  "env": {
+  },
+  "formation": {
+    "web": {
+      "quantity": 1,
+      "size": "free"
+    }
+  },
+  "image": "heroku/python"
+}

backend/backend.py

@@ -1,365 +0,0 @@
-#!/usr/bin/python3
-
-
-# @todo logging
-# @todo sort by last_changed
-# @todo extra options for url like , verify=False etc.
-# @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option?
-# @todo maybe a button to reset all 'last-changed'.. so you can see it clearly when something happens since your last visit
-# @todo option for interval day/6 hour/etc
-# @todo on change detected, config for calling some API
-# @todo make tables responsive!
-# @todo fetch title into json
-# https://distill.io/features
-# proxy per check
-#i
-import json
-import eventlet
-import eventlet.wsgi
-
-import time
-import os
-import getopt
-import sys
-import datetime
-import timeago
-
-import threading
-
-from flask import Flask, render_template, request, send_file, send_from_directory, safe_join, abort, redirect, url_for
-
-# Local
-import store
-import fetch_site_status
-
-ticker_thread = None
-
-datastore = store.ChangeDetectionStore()
-messages = []
-extra_stylesheets = []
-running_update_threads = {}
-
-app = Flask(__name__, static_url_path='/static')
-app.config['STATIC_RESOURCES'] = "/app/static"
-
-# app.config['SECRET_KEY'] = 'secret!'
-
-# Disables caching of the templates
-app.config['TEMPLATES_AUTO_RELOAD'] = True
-
-
-# We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
-# running or something similar.
-@app.template_filter('format_last_checked_time')
-def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
-    global running_update_threads
-    if watch_obj['uuid'] in running_update_threads:
-        if running_update_threads[watch_obj['uuid']].is_alive():
-            return "Checking now.."
-
-    if watch_obj['last_checked'] == 0:
-        return 'Not yet'
-
-    return timeago.format(int(watch_obj['last_checked']), time.time())
-
-
-# @app.context_processor
-# def timeago():
-#    def _timeago(lower_time, now):
-#        return timeago.format(lower_time, now)
-#    return dict(timeago=_timeago)
-
-@app.template_filter('format_timestamp_timeago')
-def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
-    if timestamp == 0:
-        return 'Not yet'
-    return timeago.format(timestamp, time.time())
-    # return timeago.format(timestamp, time.time())
-    # return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
-
-
-@app.route("/", methods=['GET'])
-def main_page():
-    global messages
-
-    limit_tag = request.args.get('tag')
-
-    # Sort by last_changed and add the uuid which is usually the key..
-    sorted_watches = []
-    for uuid, watch in datastore.data['watching'].items():
-        if limit_tag != None:
-            # Support for comma separated list of tags.
-            for tag_in_watch in watch['tag'].split(','):
-                tag_in_watch = tag_in_watch.strip()
-                if tag_in_watch == limit_tag:
-                    watch['uuid'] = uuid
-                    sorted_watches.append(watch)
-
-        else:
-            watch['uuid'] = uuid
-            sorted_watches.append(watch)
-
-    sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)
-
-    existing_tags = datastore.get_all_tags()
-    output = render_template("watch-overview.html",
-                             watches=sorted_watches,
-                             messages=messages,
-                             tags=existing_tags,
-                             active_tag=limit_tag)
-
-    # Show messages but once.
-    messages = []
-    return output
-
-
-@app.route("/edit", methods=['GET'])
-def edit_page():
-    global messages
-
-    uuid = request.args.get('uuid')
-    output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], messages=messages)
-    return output
-
-
-@app.route("/settings", methods=['GET', "POST"])
-def settings_page():
-    global messages
-    if request.method == 'POST':
-        try:
-            minutes = int(request.values.get('minutes').strip())
-        except ValueError:
-            messages.append({'class': 'error', 'message': "Invalid value given, use an integer."})
-
-        else:
-            if minutes >= 5 and minutes <= 600:
-                datastore.data['settings']['requests']['minutes_between_check'] = minutes
-                datastore.needs_write = True
-
-                messages.append({'class': 'ok', 'message': "Updated"})
-            else:
-                messages.append({'class': 'error', 'message': "Must be equal to or greater than 5 and less than 600 minutes"})
-
-    output = render_template("settings.html", messages=messages, minutes=datastore.data['settings']['requests']['minutes_between_check'])
-    messages =[]
-
-    return output
-
-@app.route("/import", methods=['GET', "POST"])
-def import_page():
-    import validators
-    global messages
-    remaining_urls=[]
-
-    good = 0
-
-    if request.method == 'POST':
-        urls = request.values.get('urls').split("\n")
-        for url in urls:
-            url = url.strip()
-            if len(url) and validators.url(url):
-                datastore.add_watch(url=url.strip(), tag="")
-                good += 1
-            else:
-                if len(url):
-                    remaining_urls.append(url)
-
-        messages.append({'class': 'ok', 'message': "{} Imported, {} Skipped.".format(good, len(remaining_urls))})
-
-        launch_checks()
-
-    output = render_template("import.html",
-                             messages=messages,
-                             remaining="\n".join(remaining_urls)
-                             )
-    messages = []
-    return output
-
-
-@app.route("/diff/<string:uuid>", methods=['GET'])
-def diff_history_page(uuid):
-    global messages
-    global extra_stylesheets
-    extra_stylesheets.append('/static/css/diff.css')
-
-    watch = datastore.data['watching'][uuid]
-
-    dates = list(watch['history'].keys())
-    dates = [int(i) for i in dates]
-    dates.sort(reverse=True)
-
-    left_file_contents = right_file_contents = ""
-    l_file = watch['history'][str(dates[-1])]
-    with open(l_file, 'r') as f:
-        left_file_contents = f.read()
-
-    r_file  = watch['history'][str(dates[-2])]
-    with open(r_file, 'r') as f:
-        right_file_contents = f.read()
-
-    output = render_template("diff.html", watch_a=watch, messages=messages, left=left_file_contents,
-                             right=right_file_contents, extra_stylesheets=extra_stylesheets)
-    return output
-
-@app.route("/favicon.ico", methods=['GET'])
-def favicon():
-    return send_from_directory("/app/static/images", filename="favicon.ico")
-
-
-@app.route("/static/<string:group>/<string:filename>", methods=['GET'])
-def static_content(group, filename):
-    try:
-        return send_from_directory("/app/static/{}".format(group), filename=filename)
-    except FileNotFoundError:
-        abort(404)
-
-
-@app.route("/api/add", methods=['POST'])
-def api_watch_add():
-    global messages
-
-    # @todo add_watch should throw a custom Exception for validation etc
-    datastore.add_watch(url=request.form.get('url').strip(), tag=request.form.get('tag').strip())
-    messages.append({'class': 'ok', 'message': 'Watch added.'})
-    launch_checks()
-    return redirect(url_for('main_page'))
-
-
-@app.route("/api/delete", methods=['GET'])
-def api_delete():
-    global messages
-    uuid = request.args.get('uuid')
-    datastore.delete(uuid)
-    messages.append({'class': 'ok', 'message': 'Deleted.'})
-
-    return redirect(url_for('main_page'))
-
-
-@app.route("/api/update", methods=['POST'])
-def api_update():
-    global messages
-    import validators
-
-    uuid = request.args.get('uuid')
-
-    url = request.form.get('url').strip()
-    tag = request.form.get('tag').strip()
-
-    form_headers = request.form.get('headers').strip().split("\n")
-    extra_headers = {}
-    if form_headers:
-        for header in form_headers:
-            if len(header):
-                parts = header.split(':', 1)
-                extra_headers.update({parts[0].strip(): parts[1].strip()})
-
-
-
-    validators.url(url) #@todo switch to prop/attr/observer
-    datastore.data['watching'][uuid].update({'url': url,
-                                             'tag': tag,
-                                             'headers':extra_headers})
-    datastore.needs_write = True
-
-    messages.append({'class': 'ok', 'message': 'Updated watch.'})
-
-    return redirect(url_for('main_page'))
-
-@app.route("/api/checknow", methods=['GET'])
-def api_watch_checknow():
-    global messages
-
-    uuid = request.args.get('uuid')
-
-    running_update_threads[uuid] = fetch_site_status.perform_site_check(uuid=uuid,
-                                                                        datastore=datastore)
-    running_update_threads[uuid].start()
-
-    return redirect(url_for('main_page'))
-
-
-@app.route("/api/recheckall", methods=['GET'])
-def api_watch_recheckall():
-    import fetch_site_status
-
-    global running_update_threads
-    i = 0
-    for uuid, watch in datastore.data['watching'].items():
-        i = i + 1
-
-        running_update_threads[watch['uuid']] = fetch_site_status.perform_site_check(uuid=uuid,
-                                                                                     datastore=datastore)
-        running_update_threads[watch['uuid']].start()
-
-    return "{} triggered recheck of {} watches.".format(i, len(datastore.data['watching']))
-
-
-# Can be used whenever, launch threads that need launching to update the stored information
-def launch_checks():
-    import fetch_site_status
-    global running_update_threads
-
-
-    minutes = datastore.data['settings']['requests']['minutes_between_check']
-    for uuid, watch in datastore.data['watching'].items():
-
-
-        if watch['last_checked'] <= time.time() - (minutes * 60):
-            running_update_threads[watch['uuid']] = fetch_site_status.perform_site_check(uuid=uuid,
-                                                                                         datastore=datastore)
-            running_update_threads[watch['uuid']].start()
-
-
-# Thread runner to check every minute
-def ticker_thread_check_time_launch_checks():
-    while True:
-        launch_checks()
-        time.sleep(60)
-
-# Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON
-# by just running periodically in one thread.
-def save_datastore():
-    while True:
-        if datastore.needs_write:
-            datastore.sync_to_json()
-        time.sleep(5)
-
-def main(argv):
-    ssl_mode = False
-    port = 5000
-
-    try:
-        opts, args = getopt.getopt(argv, "sp:", "purge")
-    except getopt.GetoptError:
-        print('backend.py -s SSL enable -p [port]')
-        sys.exit(2)
-
-    for opt, arg in opts:
-        if opt == '--purge':
-            # Remove history, the actual files you need to delete manually.
-            for uuid, watch in datastore.data['watching'].items():
-                watch.update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': None})
-
-        if opt == '-s':
-            ssl_mode = True
-
-        if opt == '-p':
-            port = arg
-
-    # @todo handle ctrl break
-    ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
-    save_data_thread = threading.Thread(target=save_datastore).start()
-
-    # @todo finalise SSL config, but this should get you in the right direction if you need it.
-    if ssl_mode:
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen(('', port)),
-                                               certfile='cert.pem',
-                                               keyfile='privkey.pem',
-                                               server_side=True), app)
-
-    else:
-        eventlet.wsgi.server(eventlet.listen(('', port)), app)
-
-
-if __name__ == '__main__':
-    main(sys.argv[1:])

backend/dev-docker/requirements.txt

@@ -1,20 +0,0 @@
-aiohttp
-async-timeout
-chardet==2.3.0
-multidict
-python-engineio
-six==1.10.0
-yarl
-flask
-
-eventlet
-requests
-validators
-
-bleach==3.2.1
-html5lib==0.9999999       # via bleach
-timeago
-html2text
-
-# @notes
-# - Dont install socketio, it interferes with flask_socketio

backend/fetch_site_status.py

@@ -1,134 +0,0 @@
-from threading import Thread
-import time
-import requests
-import hashlib
-import os
-import re
-import html2text
-# Not needed due to inscriptis being way better.
-#from urlextract import URLExtract
-from inscriptis import get_text
-
-# Hmm Polymorphism datastore, thread, etc
-class perform_site_check(Thread):
-    def __init__(self, *args, uuid=False, datastore, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.timestamp = int(time.time())  # used for storage etc too
-        self.uuid = uuid
-        self.datastore = datastore
-        self.url = datastore.get_val(uuid, 'url')
-        self.current_md5 = datastore.get_val(uuid, 'previous_md5')
-        self.output_path = "/datastore/{}".format(self.uuid)
-
-    def save_firefox_screenshot(self, uuid, output):
-        # @todo call selenium or whatever
-        return
-
-    def ensure_output_path(self):
-
-        try:
-            os.stat(self.output_path)
-        except:
-            os.mkdir(self.output_path)
-
-    def save_response_html_output(self, output):
-        # @todo maybe record a history.json, [timestamp, md5, filename]
-        with open("{}/{}.txt".format(self.output_path, self.timestamp), 'w') as f:
-            f.write(output)
-            f.close()
-
-    def save_response_stripped_output(self, output):
-        fname = "{}/{}.stripped.txt".format(self.output_path, self.timestamp)
-        with open(fname, 'w') as f:
-            f.write(output)
-            f.close()
-
-        return fname
-
-    def run(self):
-
-        extra_headers = self.datastore.get_val(self.uuid, 'headers')
-
-        # Tweak the base config with the per-watch ones
-        request_headers = self.datastore.data['settings']['headers'].copy()
-        request_headers.update(extra_headers)
-
-        print("Checking", self.url)
-        #print(request_headers)
-
-        self.ensure_output_path()
-
-        try:
-            timeout = self.datastore.data['settings']['requests']['timeout']
-        except KeyError:
-            # @todo yeah this should go back to the default value in store.py, but this whole object should abstract off it
-            timeout = 15
-
-        try:
-            r = requests.get(self.url,
-                             headers=request_headers,
-                             timeout=timeout,
-                             verify=False)
-
-            stripped_text_from_html = get_text(r.text)
-
-
-            # @todo This should be a config option.
-            # Many websites include junk in the links, trackers, etc.. Since we are really a service all about text changes..
-
-# inscriptis handles this much cleaner, probably not needed..
-#            extractor = URLExtract()
-#            urls = extractor.find_urls(stripped_text_from_html)
-            # Remove the urls, longest first so that we dont end up chewing up bigger links with parts of smaller ones.
-#            if urls:
-#                urls.sort(key=len, reverse=True)
-#                for url in urls:
-#                    # Sometimes URLExtract will consider something like 'foobar.com' as a link when that was just text.
-#                    if "://" in url:
-#                        # print ("Stripping link", url)
-#                        stripped_text_from_html = stripped_text_from_html.replace(url, '')
-
-
-
-        # Usually from networkIO/requests level
-        except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
-            self.datastore.update_watch(self.uuid, 'last_error', str(e))
-            print(str(e))
-
-        except requests.exceptions.MissingSchema:
-            print("Skipping {} due to missing schema/bad url".format(self.uuid))
-
-        # Usually from html2text level
-        except UnicodeDecodeError as e:
-            self.datastore.update_watch(self.uuid, 'last_error', str(e))
-            print(str(e))
-            # figure out how to deal with this cleaner..
-            # 'utf-8' codec can't decode byte 0xe9 in position 480: invalid continuation byte
-
-        else:
-
-            # We rely on the actual text in the html output.. many sites have random script vars etc
-            self.datastore.update_watch(self.uuid, 'last_error', False)
-            self.datastore.update_watch(self.uuid, 'last_check_status', r.status_code)
-
-            fetched_md5 = hashlib.md5(stripped_text_from_html.encode('utf-8')).hexdigest()
-
-            if self.current_md5 != fetched_md5:
-
-                # Dont confuse people by putting last-changed, when it actually just changed from nothing..
-                if self.datastore.get_val(self.uuid, 'previous_md5') is not None:
-                    self.datastore.update_watch(self.uuid, 'last_changed', self.timestamp)
-
-                self.datastore.update_watch(self.uuid, 'previous_md5', fetched_md5)
-                self.save_response_html_output(r.text)
-                output_filepath = self.save_response_stripped_output(stripped_text_from_html)
-
-                # Update history with the stripped text for future reference, this will also mean we save the first
-                # attempt because 'self.current_md5 != fetched_md5'  (current_md5 will be None when not run)
-                # need to learn more about attr/setters/getters
-                history = self.datastore.get_val(self.uuid, 'history')
-                history.update(dict([(str(self.timestamp), output_filepath)]))
-                self.datastore.update_watch(self.uuid, 'history', history)
-
-        self.datastore.update_watch(self.uuid, 'last_checked', int(time.time()))
-        pass

backend/nocache.py

@@ -1,14 +0,0 @@
-
-from flask import make_response
-from functools import wraps, update_wrapper
-from datetime import datetime
-
-def nocache(view):
-    @wraps(view)
-    def no_cache(*args, **kwargs):
-        response = make_response(view(*args, **kwargs))
-        response.headers['hmm'] = datetime.now()
-
-        return response
-
-    return update_wrapper(no_cache, view)

backend/requirements.txt

@@ -1,20 +0,0 @@
-aiohttp
-async-timeout
-chardet==2.3.0
-multidict
-python-engineio
-six==1.10.0
-yarl
-flask
-
-eventlet
-requests
-validators
-
-bleach==3.2.1
-html5lib==0.9999999       # via bleach
-timeago
-html2text
-
-# @notes
-# - Dont install socketio, it interferes with flask_socketio

backend/static/css/diff.css

@@ -1,61 +0,0 @@
-table {
-	table-layout: fixed;
-	width: 100%;
-}
-td {
-	width: 33%;
-	padding: 3px 4px;
-	border: 1px solid transparent;
-	vertical-align: top;
-	font: 1em monospace;
-	text-align: left;
-	white-space: pre-wrap;
-}
-h1 {
-	display: inline;
-	font-size: 100%;
-}
-del {
-	text-decoration: none;
-	color: #b30000;
-	background: #fadad7;
-}
-
-ins {
-	background: #eaf2c2;
-	color: #406619;
-	text-decoration: none;
-}
-
-#result {
-	white-space: pre-wrap;
-}
-
-#settings {
-
-
-	line-height: 2em;
-}
-#settings label {
-	margin-left: 1em;
-}
-
-.source {
-	position: absolute;
-	right: 1%;
-	top: .2em;
-}
-
-@-moz-document url-prefix() {
-	body {
-		height: 99%; /* Hide scroll bar in Firefox */
-	}
-}
-
-#diff-ui {
-background: #fff;
-    padding: 2em;
-    margin: 1em;
-    border-radius: 5px;
-    font-size: 9px;
-}

backend/static/css/styles.css

@@ -1,201 +0,0 @@
-/*
- * -- BASE STYLES --
- * Most of these are inherited from Base, but I want to change a few.
- */
-body {
-  color: #333;
-  background: #262626;
-}
-
-.pure-table-even {
-  background: #fff;
-}
-/* Some styles from https://css-tricks.com/ */
-
-a {
-  text-decoration: none;
-  color: #1b98f8;
-}
-a.github-link {
-  color: #fff;
-}
-
-
-.pure-menu-horizontal {
-  background: #fff;
-  padding: 5px;
-  display: flex;
-  justify-content: space-between;
-  border-bottom: 2px solid #ed5900;
-
-  align-items: center;
-}
-
-section.content {
-
-  padding-top: 5em;
-  padding-bottom: 5em;
-  flex-direction: column;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-}
-
-.pure-table.watch-table td {
-  font-size: 80%;
-}
-
-/* table related */
-.watch-table {
-  width: 100%;
-}
-.watch-tag-list {
-  color: #e70069;
-  white-space: nowrap;
-}
-
-.box {
-  max-width: 80%;
-  flex-direction: column;
-  display: flex;
-  justify-content: center;
-}
-
-.watch-table .error {
-  color: #a00;
-}
-
-.watch-table td {
-  white-space: nowrap;
-}
-
-.watch-table td.title-col {
-  word-break: break-all;
-  white-space: normal;
-}
-.watch-table th {
-  white-space: nowrap;
-}
-
-.watch-table .title-col a[target="_blank"]::after {
-  content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
-  margin: 0 3px 0 5px;
-}
-
-/* hotovo */
-
-
-body:after {
-  content: "";
-  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)
-}
-
-body:after, body:before {
-  display: block;
-  height: 600px;
-  position: absolute;
-  top: 0;
-  left: 0;
-  width: 100%;
-  z-index: -1;
-}
-body::after {
-  opacity: 0.91;
-}
-body::before {
-  content: "";
-  background-image: url(/static/images/gradient-border.png);
-}
-body:before {
-  background-size: cover
-}
-
-body:after, body:before {
-  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
-  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
-}
-
-
-.button-small {
-  font-size: 85%;
-}
-
-
-.fetch-error {
-  padding-top: 1em;
-  font-size: 60%;
-  max-width: 400px;
-  display: block;
-}
-
-.edit-form {
-   background: #fff;
-   padding: 2em;
-   margin: 1em;
-   border-radius: 5px;
-}
-.button-secondary {
-    color: white;
-    border-radius: 4px;
-    text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
-}
-
-.button-success {
-    background: rgb(28, 184, 65);
-    /* this is a green */
-}
-
-.button-tag {
-    background: rgb(99, 99, 99);
-    color: #fff;
-    font-size: 65%;
-    border-bottom-left-radius: initial;
-    border-bottom-right-radius: initial;
-}
-.button-tag.active {
-    background: #9c9c9c;
-    font-weight: bold;
-}
-.button-error {
-    background: rgb(202, 60, 60);
-    /* this is a maroon */
-}
-
-.button-warning {
-    background: rgb(223, 117, 20);
-    /* this is an orange */
-}
-
-.button-secondary {
-    background: rgb(66, 184, 221);
-    /* this is a light blue */
-}
-
-
-.button-cancel {
-    background: rgb(200, 200, 200);
-    /* this is a green */
-}
-
-.messages {
-    padding: 1em;
-    background: rgba(255,255,255,.2);
-    border-radius: 10px;
-    color: #fff;
-    font-weight: bold;
-}
-
-.pure-form label  {
-  font-weight: bold;
-}
-
-#new-watch-form {
-    background: rgba(0,0,0,.05);
-    padding: 1em;
-    border-radius: 10px;
-    margin-bottom: 1em;
-}
-#new-watch-form legend {
-    color: #fff;
-}
-

backend/store.py

@@ -1,147 +0,0 @@
-import json
-import uuid as uuid_builder
-import validators
-
-
-# Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
-# Open a github issue if you know something :)
-# https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
-class ChangeDetectionStore:
-
-    def __init__(self):
-        self.needs_write = False
-
-        self.__data = {
-            'note' : "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
-            'watching': {},
-            'settings': {
-                'headers': {
-                    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
-                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
-                    'Accept-Encoding': 'gzip, deflate, br',
-                    'Accept-Language': 'en-GB,en-US;q=0.9,en;'
-                },
-                'requests': {
-                    'timeout': 15, # Default 15 seconds
-                    'minutes_between_check': 3 * 60 # Default 3 hours
-                }
-            }
-        }
-
-
-        # Base definition for all watchers
-        self.generic_definition = {
-            'url': None,
-            'tag': None,
-            'last_checked': 0,
-            'last_changed': 0,
-            'title': None,
-            'previous_md5': None,
-            'uuid': str(uuid_builder.uuid4()),
-            'headers' : {}, # Extra headers to send
-            'history' : {} # Dict of timestamp and output stripped filename
-        }
-
-
-        try:
-            with open('/datastore/url-watches.json') as json_file:
-                from_disk = json.load(json_file)
-
-                # @todo isnt there a way todo this dict.update recursively?
-                # Problem here is if the one on the disk is missing a sub-struct, it wont be present anymore.
-                if 'watching' in from_disk:
-                    self.__data['watching'].update(from_disk['watching'])
-
-                if 'settings' in from_disk:
-                    if 'headers' in from_disk['settings']:
-                        self.__data['settings']['headers'].update(from_disk['settings']['headers'])
-
-                    if 'requests' in from_disk['settings']:
-                        self.__data['settings']['requests'].update(from_disk['settings']['requests'])
-
-
-                # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
-                # @todo pretty sure theres a python we todo this with an abstracted(?) object!
-                i = 0
-                for uuid, watch in self.data['watching'].items():
-                    _blank = self.generic_definition.copy()
-                    _blank.update(watch)
-                    self.__data['watching'].update({uuid: _blank})
-                    print("Watching:", uuid, _blank['url'])
-
-        # First time ran, doesnt exist.
-        except (FileNotFoundError, json.decoder.JSONDecodeError):
-            print("Creating JSON store")
-            self.add_watch(url='http://www.quotationspage.com/random.php', tag='test')
-            self.add_watch(url='https://news.ycombinator.com/', tag='Tech news')
-            self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
-            self.add_watch(url='https://changedetection.io', tag='Tech news')
-
-            
-#        self.entryVariable.get()
-    def update_watch(self, uuid, val, var):
-
-        self.__data['watching'][uuid].update({val: var})
-        self.needs_write = True
-
-
-    @property
-    def data(self):
-        return self.__data
-
-    def get_all_tags(self):
-        tags=[]
-        for uuid, watch in self.data['watching'].items():
-
-            # Support for comma separated list of tags.
-            for tag in watch['tag'].split(','):
-                tag = tag.strip()
-                if not tag in tags:
-                    tags.append(tag)
-
-        tags.sort()
-        return tags
-
-    def delete(self, uuid):
-        # Probably their should be dict...
-        del(self.__data['watching'][uuid])
-        self.needs_write = True
-
-
-    def url_exists(self, url):
-
-        # Probably their should be dict...
-        for watch in self.data['watching']:
-            if watch['url'] == url:
-                return True
-
-        return False
-
-    def get_val(self, uuid, val):
-        # Probably their should be dict...
-        return self.data['watching'][uuid].get(val)
-
-    def add_watch(self, url, tag):
-
-        # @todo deal with exception
-        validators.url(url)
-
-        # @todo use a common generic version of this
-
-        _blank = self.generic_definition.copy()
-        _blank.update({
-            'url': url,
-            'tag': tag,
-            'uuid': str(uuid_builder.uuid4())
-        })
-
-        self.data['watching'].update({_blank['uuid']: _blank})
-
-
-    def sync_to_json(self):
-        print ("Saving index")
-        with open('/datastore/url-watches.json', 'w') as json_file:
-            json.dump(self.data, json_file, indent=4)
-        self.needs_write = False
-
-# body of the constructor

backend/templates/base.html

@@ -1,64 +0,0 @@
-<!doctype html>
-<html lang="en">
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <meta name="description" content="Self hosted website change detection.">
-    <title>Change Detection</title>
-    <link rel="stylesheet" href="/static/css/pure-min.css">
-    <link rel="stylesheet" href="/static/css/styles.css">
-    {% if extra_stylesheets %}
-        {% for m in extra_stylesheets %}
-        <link rel="stylesheet" href="{{ m }}">
-        {% endfor %}
-    {% endif %}
-</head>
-<body>
-
-<div class="header">
-    <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed">
-        <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
-
-        <ul class="pure-menu-list">
-
-            <li class="pure-menu-item">
-                <a href="/import" class="pure-menu-link">IMPORT</a>
-            </li>
-            <li class="pure-menu-item">
-                <a href="/settings" class="pure-menu-link">SETTINGS</a>
-            </li>
-            <li class="pure-menu-item"><a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
-                <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16" version="1.1"
-                     width="32" aria-hidden="true">
-                    <path fill-rule="evenodd"
-                          d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
-                </svg>
-            </a></li>
-            <!--
-            <li class="pure-menu-item"><a href="#" class="pure-menu-link">Tour</a></li>
-            <li class="pure-menu-item"><a href="#" class="pure-menu-link">Sign Up</a></li>
-            -->
-        </ul>
-    </div>
-</div>
-
-<section class="content">
-    <header>
-        {% block header %}{% endblock %}
-    </header>
-
-    {% if messages %}
-    <div class="messages">
-        {% for message in messages %}
-        <div class="flash-message {{ message['class'] }}">{{ message['message'] }}</div>
-        {% endfor %}
-    </div>
-    {% endif %}
-
-
-    {% block content %}
-
-    {% endblock %}
-</section>
-</body>
-</html>

backend/templates/diff.html

@@ -1,98 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-
-<div id="diff-ui">
-    <del>Removed text</del>
-    <ins>Inserted Text</ins>
-
-<div id="settings">
-    <h3>Diff</h3>
-    <label><input type="radio" name="diff_type" value="diffChars"> Chars</label>
-    <label><input type="radio" name="diff_type" value="diffWords" > Words</label>
-    <label><input type="radio" name="diff_type" value="diffLines" checked=""> Lines</label>
-</div>
-
-    <table>
-        <tbody>
-        <tr>
-			<!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
-            <td id="a" style="display: none;">{{left}}</td>
-            <td id="b" style="display: none;">{{right}}</td>
-            <td>
-                <pre id="result"></pre>
-            </td>
-        </tr>
-        </tbody>
-    </table>
-Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff" >github.com/kpdecker/jsdiff</a>
-</div>
-
-<script src="/static/js/diff.js"></script>
-<script defer="">
-var a = document.getElementById('a');
-var b = document.getElementById('b');
-var result = document.getElementById('result');
-
-function changed() {
-	var diff = JsDiff[window.diffType](a.textContent, b.textContent);
-	var fragment = document.createDocumentFragment();
-	for (var i=0; i < diff.length; i++) {
-
-		if (diff[i].added && diff[i + 1] && diff[i + 1].removed) {
-			var swap = diff[i];
-			diff[i] = diff[i + 1];
-			diff[i + 1] = swap;
-		}
-
-		var node;
-		if (diff[i].removed) {
-			node = document.createElement('del');
-			node.appendChild(document.createTextNode(diff[i].value));
-		} else if (diff[i].added) {
-			node = document.createElement('ins');
-			node.appendChild(document.createTextNode(diff[i].value));
-		} else {
-			node = document.createTextNode(diff[i].value);
-		}
-		fragment.appendChild(node);
-	}
-
-	result.textContent = '';
-	result.appendChild(fragment);
-}
-
-window.onload = function() {
-	onDiffTypeChange(document.querySelector('#settings [name="diff_type"]:checked'));
-	changed();
-};
-
-a.onpaste = a.onchange =
-b.onpaste = b.onchange = changed;
-
-if ('oninput' in a) {
-	a.oninput = b.oninput = changed;
-} else {
-	a.onkeyup = b.onkeyup = changed;
-}
-
-function onDiffTypeChange(radio) {
-	window.diffType = radio.value;
-	document.title = "Diff " + radio.value.slice(4);
-}
-
-var radio = document.getElementsByName('diff_type');
-for (var i = 0; i < radio.length; i++) {
-	radio[i].onchange = function(e) {
-		onDiffTypeChange(e.target);
-		changed();
-	}
-}
-
-</script>
-
-
-
-
-
-{% endblock %}

backend/templates/edit.html

@@ -1,55 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-<div class="edit-form">
-
-
-    <form class="pure-form pure-form-stacked" action="/api/update?uuid={{uuid}}" method="POST">
-        <fieldset>
-            <div class="pure-control-group">
-                <label for="url">URL</label>
-                <input type="url" id="url" required="" placeholder="https://..." name="url" value="{{ watch.url}}"
-                       size="50"/>
-                <span class="pure-form-message-inline">This is a required field.</span>
-            </div>
-            <div class="pure-control-group">
-                <label for="tag">Tag</label>
-                <input type="text" placeholder="tag" size="10" id="tag" name="tag" value="{{ watch.tag}}"/>
-                <span class="pure-form-message-inline">Grouping tags, can be a comma separated list.</span>
-            </div>
-
-            <fieldset class="pure-group">
-                <label for="headers">Extra request headers</label>
-
-                <textarea id=headers name="headers" class="pure-input-1-2" placeholder="Example
-Cookie: foobar
-User-Agent: wonderbra 1.0"
-                          style="width: 100%;
-                            font-family:monospace;
-                            white-space: pre;
-                            overflow-wrap: normal;
-                            overflow-x: scroll;" rows="5">{% for key, value in watch.headers.items() %}{{ key }}: {{ value }}
-{% endfor %}</textarea>
-                <br/>
-
-            </fieldset>
-            <div class="pure-control-group">
-                <button type="submit" class="pure-button pure-button-primary">Save</button>
-            </div>
-            <br/>
-
-            <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Cancel</a>
-                <a href="/api/delete?uuid={{uuid}}"
-                   class="pure-button button-small button-error ">Delete</a>
-
-            </div>
-
-
-        </fieldset>
-    </form>
-
-
-</div>
-
-{% endblock %}

backend/templates/import.html

@@ -1,26 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-<div class="edit-form">
-
-
-    <form class="pure-form pure-form-aligned" action="/import" method="POST">
-
-        <fieldset class="pure-group">
-            <legend>One URL per line, URLs that do not pass validation will stay in the textarea.</legend>
-
-            <textarea name="urls" class="pure-input-1-2" placeholder="https://"
-                      style="width: 100%;
-                            font-family:monospace;
-                            white-space: pre;
-                            overflow-wrap: normal;
-                            overflow-x: scroll;" rows="25">{{ remaining }}</textarea>
-        </fieldset>
-        <button type="submit" class="pure-button pure-input-1-2 pure-button-primary">Import</button>
-
-    </form>
-
-</div>
-
-{% endblock %}
-

backend/templates/settings.html

@@ -1,33 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-<div class="edit-form">
-
-
-    <form class="pure-form pure-form-stacked" action="/settings" method="POST">
-        <fieldset>
-            <div class="pure-control-group">
-                <label for="minutes">Maximum time in minutes until recheck.</label>
-                <input type="text" id="minutes" required="" name="minutes" value="{{minutes}}"
-                       size="5"/>
-                <span class="pure-form-message-inline">This is a required field.</span>
-            </div>
-
-
-            <div class="pure-control-group">
-                <button type="submit" class="pure-button pure-button-primary">Save</button>
-            </div>
-            <br/>
-
-            <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Cancel</a>
-            </div>
-
-
-        </fieldset>
-    </form>
-
-
-</div>
-
-{% endblock %}

backend/templates/watch-overview.html

@@ -1,72 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-
-<div class="box">
-
-    <form class="pure-form" action="/api/add" method="POST" id="new-watch-form">
-        <fieldset>
-            <legend>Add a new change detection watch</legend>
-            <input type="url" placeholder="https://..." name="url"/>
-            <input type="text" placeholder="tag" size="10" name="tag" value="{{active_tag if active_tag}}"/>
-            <button type="submit" class="pure-button pure-button-primary">Watch</button>
-        </fieldset>
-        <!-- add extra stuff, like do a http POST and send headers -->
-        <!-- user/pass r = requests.get('https://api.github.com/user', auth=('user', 'pass')) -->
-    </form>
-    <div>
-
-        {% for tag in tags %}
-            {% if tag == "" %}
-            <a  href="/" class="pure-button button-tag {{'active' if active_tag == tag }}">All</a>
-            {% else %}
-            <a  href="/?tag={{ tag}}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
-            {% endif %}
-        {% endfor %}
-    </div>
-
-    <div id="watch-table-wrapper">
-
-        <table class="pure-table pure-table-striped watch-table">
-            <thead>
-            <tr>
-                <th>#</th>
-                <th></th>
-                <th>Last Checked</th>
-                <th>Last Changed</th>
-                <th></th>
-            </tr>
-            </thead>
-            <tbody>
-
-
-            {% for watch in watches %}
-            <tr id="{{ watch.uuid }}"
-                class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}">
-                <td>{{ loop.index }}</td>
-                <td class="title-col">{{watch.title if watch.title is not none else watch.url}}
-                    <a class="external" target=_blank href="{{ watch.url }}"></a>
-                    {% if watch.last_error is defined and watch.last_error != False %}
-                    <div class="fetch-error">{{ watch.last_error }}</div>
-                    {% endif %}
-                    {% if not active_tag %}
-                    <span class="watch-tag-list">{{ watch.tag}}</span>
-                    {% endif %}
-                </td>
-                <td>{{watch|format_last_checked_time}}</td>
-                <td>{{watch.last_changed|format_timestamp_timeago}}</td>
-                <td><a href="/api/checknow?uuid={{ watch.uuid}}" class="pure-button button-small pure-button-primary">Recheck</a>
-                    <a href="/edit?uuid={{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
-                    {% if watch.history|length >= 2 %}
-                    <a href="/diff/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
-                    {% endif %}
-                </td>
-            </tr>
-            {% endfor %}
-
-
-            </tbody>
-        </table>
-    </div>
-</div>
-{% endblock %}

changedetection.py

@@ -0,0 +1,11 @@
+#!/usr/bin/python3
+
+# Entry-point for running from the CLI when not installed via Pip, Pip will handle the console_scripts entry_points's from setup.py
+# It's recommended to use `pip3 install changedetection.io` and start with `changedetection.py` instead, it will be linkd to your global path.
+# or Docker.
+# Read more https://github.com/dgtlmoon/changedetection.io/wiki
+
+from changedetectionio import changedetection
+
+if __name__ == '__main__':
+    changedetection.main()

changedetectionio/.gitignore

@@ -0,0 +1 @@
+test-datastore

changedetectionio/changedetection.py

@@ -0,0 +1,114 @@
+#!/usr/bin/python3
+
+# Launch as a eventlet.wsgi server instance.
+
+import getopt
+import os
+import sys
+
+import eventlet
+import eventlet.wsgi
+from . import store, changedetection_app
+from . import __version__
+
+def main():
+    ssl_mode = False
+    host = ''
+    port = os.environ.get('PORT') or 5000
+    do_cleanup = False
+    datastore_path = None
+
+    # On Windows, create and use a default path.
+    if os.name == 'nt':
+        datastore_path = os.path.expandvars(r'%APPDATA%\changedetection.io')
+        os.makedirs(datastore_path, exist_ok=True)
+    else:
+        # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
+        datastore_path = os.path.join(os.getcwd(), "../datastore")
+
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "Ccsd:h:p:", "port")
+    except getopt.GetoptError:
+        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
+        sys.exit(2)
+
+    create_datastore_dir = False
+
+    for opt, arg in opts:
+        #        if opt == '--purge':
+        # Remove history, the actual files you need to delete manually.
+        #            for uuid, watch in datastore.data['watching'].items():
+        #                watch.update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': None})
+
+        if opt == '-s':
+            ssl_mode = True
+
+        if opt == '-h':
+            host = arg
+
+        if opt == '-p':
+            port = int(arg)
+
+        if opt == '-d':
+            datastore_path = arg
+
+        # Cleanup (remove text files that arent in the index)
+        if opt == '-c':
+            do_cleanup = True
+
+        # Create the datadir if it doesnt exist
+        if opt == '-C':
+            create_datastore_dir = True
+
+    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
+    app_config = {'datastore_path': datastore_path}
+
+    if not os.path.isdir(app_config['datastore_path']):
+        if create_datastore_dir:
+            os.mkdir(app_config['datastore_path'])
+        else:
+            print(
+                "ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists or specify a directory with the -d option.\n"
+                "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
+            sys.exit(2)
+
+    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
+    app = changedetection_app(app_config, datastore)
+
+    # Go into cleanup mode
+    if do_cleanup:
+        datastore.remove_unused_snapshots()
+
+    app.config['datastore_path'] = datastore_path
+
+
+    @app.context_processor
+    def inject_version():
+        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
+                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
+                    has_password=datastore.data['settings']['application']['password'] != False
+                    )
+
+    # Proxy sub-directory support
+    # Set environment var USE_X_SETTINGS=1 on this script
+    # And then in your proxy_pass settings
+    #
+    #         proxy_set_header Host "localhost";
+    #         proxy_set_header X-Forwarded-Prefix /app;
+
+    if os.getenv('USE_X_SETTINGS'):
+        print ("USE_X_SETTINGS is ENABLED\n")
+        from werkzeug.middleware.proxy_fix import ProxyFix
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
+
+    if ssl_mode:
+        # @todo finalise SSL config, but this should get you in the right direction if you need it.
+        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port)),
+                                               certfile='cert.pem',
+                                               keyfile='privkey.pem',
+                                               server_side=True), app)
+
+    else:
+        eventlet.wsgi.server(eventlet.listen((host, int(port))), app)
+
+

changedetectionio/content_fetcher.py

@@ -0,0 +1,209 @@
+from abc import ABC, abstractmethod
+import chardet
+import os
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
+from selenium.common.exceptions import WebDriverException
+import requests
+import time
+import urllib3.exceptions
+
+
+class EmptyReply(Exception):
+    def __init__(self, status_code, url):
+        # Set this so we can use it in other parts of the app
+        self.status_code = status_code
+        self.url = url
+        return
+
+    pass
+
+class Fetcher():
+    error = None
+    status_code = None
+    content = None
+    headers = None
+
+    fetcher_description ="No description"
+
+    @abstractmethod
+    def get_error(self):
+        return self.error
+
+    @abstractmethod
+    def run(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False):
+        # Should set self.error, self.status_code and self.content
+        pass
+
+    @abstractmethod
+    def quit(self):
+        return
+
+    @abstractmethod
+    def screenshot(self):
+        return
+
+    @abstractmethod
+    def get_last_status_code(self):
+        return self.status_code
+
+    @abstractmethod
+    # Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
+    def is_ready(self):
+        return True
+
+#   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
+#   the current one would return javascript output (as we use JS to generate the diff)
+#
+#   Returns tuple(mime_type, stream)
+#    @abstractmethod
+#    def return_diff(self, stream_a, stream_b):
+#        return
+
+def available_fetchers():
+        import inspect
+        from changedetectionio import content_fetcher
+        p=[]
+        for name, obj in inspect.getmembers(content_fetcher):
+            if inspect.isclass(obj):
+                # @todo html_ is maybe better as fetcher_ or something
+                # In this case, make sure to edit the default one in store.py and fetch_site_status.py
+                if "html_" in name:
+                    t=tuple([name,obj.fetcher_description])
+                    p.append(t)
+
+        return p
+
+class html_webdriver(Fetcher):
+    if os.getenv("WEBDRIVER_URL"):
+        fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))
+    else:
+        fetcher_description = "WebDriver Chrome/Javascript"
+
+    command_executor = ''
+
+    # Configs for Proxy setup
+    # In the ENV vars, is prefixed with "webdriver_", so it is for example "webdriver_sslProxy"
+    selenium_proxy_settings_mappings = ['proxyType', 'ftpProxy', 'httpProxy', 'noProxy',
+                                        'proxyAutoconfigUrl', 'sslProxy', 'autodetect',
+                                        'socksProxy', 'socksVersion', 'socksUsername', 'socksPassword']
+
+
+
+    proxy=None
+
+    def __init__(self):
+        # .strip('"') is going to save someone a lot of time when they accidently wrap the env value
+        self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub').strip('"')
+
+        # If any proxy settings are enabled, then we should setup the proxy object
+        proxy_args = {}
+        for k in self.selenium_proxy_settings_mappings:
+            v = os.getenv('webdriver_' + k, False)
+            if v:
+                proxy_args[k] = v.strip('"')
+
+        if proxy_args:
+            self.proxy = SeleniumProxy(raw=proxy_args)
+
+    def run(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False):
+
+        # request_body, request_method unused for now, until some magic in the future happens.
+
+        # check env for WEBDRIVER_URL
+        self.driver = webdriver.Remote(
+            command_executor=self.command_executor,
+            desired_capabilities=DesiredCapabilities.CHROME,
+            proxy=self.proxy)
+
+        try:
+            self.driver.get(url)
+        except WebDriverException as e:
+            # Be sure we close the session window
+            self.quit()
+            raise
+
+        # @todo - how to check this? is it possible?
+        self.status_code = 200
+        # @todo somehow we should try to get this working for WebDriver
+        # raise EmptyReply(url=url, status_code=r.status_code)
+
+        # @todo - dom wait loaded?
+        time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)))
+        self.content = self.driver.page_source
+        self.headers = {}
+
+    def screenshot(self):
+        return self.driver.get_screenshot_as_png()
+
+    # Does the connection to the webdriver work? run a test connection.
+    def is_ready(self):
+        from selenium import webdriver
+        from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+        from selenium.common.exceptions import WebDriverException
+
+        self.driver = webdriver.Remote(
+            command_executor=self.command_executor,
+            desired_capabilities=DesiredCapabilities.CHROME)
+
+        # driver.quit() seems to cause better exceptions
+        self.quit()
+        return True
+
+    def quit(self):
+        if self.driver:
+            try:
+                self.driver.quit()
+            except Exception as e:
+                print("Exception in chrome shutdown/quit" + str(e))
+
+# "html_requests" is listed as the default fetcher in store.py!
+class html_requests(Fetcher):
+    fetcher_description = "Basic fast Plaintext/HTTP Client"
+
+    def run(self,
+            url,
+            timeout,
+            request_headers,
+            request_body,
+            request_method,
+            ignore_status_codes=False):
+
+        r = requests.request(method=request_method,
+                         data=request_body,
+                         url=url,
+                         headers=request_headers,
+                         timeout=timeout,
+                         verify=False)
+
+        # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
+        # For example - some sites don't tell us it's utf-8, but return utf-8 content
+        # This seems to not occur when using webdriver/selenium, it seems to detect the text encoding more reliably.
+        # https://github.com/psf/requests/issues/1604 good info about requests encoding detection
+        if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
+            encoding = chardet.detect(r.content)['encoding']
+            if encoding:
+                r.encoding = encoding
+
+        # @todo test this
+        # @todo maybe you really want to test zero-byte return pages?
+        if (not ignore_status_codes and not r) or not r.content or not len(r.content):
+            raise EmptyReply(url=url, status_code=r.status_code)
+
+        self.status_code = r.status_code
+        self.content = r.text
+        self.headers = r.headers
+

changedetectionio/dev-docker/Dockerfile

@@ -3,9 +3,7 @@ FROM python:3.8-slim
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1
 
-COPY requirements.txt /tmp/requirements.txt
-RUN pip3 install -r /tmp/requirements.txt
-
+WORKDIR /app
 
 RUN [ ! -d "/datastore" ] && mkdir /datastore
 

changedetectionio/dev-docker/sleep.py

@@ -1,9 +1,7 @@
 import time
-import sys
 
 print ("Sleep loop, you should run your script from the console")
 
 while True: 
     # Wait for 5 seconds
-
-    time.sleep(2)
+    time.sleep(2)

changedetectionio/diff.py

@@ -0,0 +1,52 @@
+# used for the notifications, the front-end is using a JS library
+
+import difflib
+
+
+def same_slicer(l, a, b):
+    if a == b:
+        return [l[a]]
+    else:
+        return l[a:b]
+
+# like .compare but a little different output
+def customSequenceMatcher(before, after, include_equal=False):
+    cruncher = difflib.SequenceMatcher(isjunk=lambda x: x in " \\t", a=before, b=after)
+
+    # @todo Line-by-line mode instead of buncghed, including `after` that is not in `before` (maybe unset?)
+    for tag, alo, ahi, blo, bhi in cruncher.get_opcodes():
+        if include_equal and tag == 'equal':
+            g = before[alo:ahi]
+            yield g
+        elif tag == 'delete':
+            g = ["(removed) " + i for i in same_slicer(before, alo, ahi)]
+            yield g
+        elif tag == 'replace':
+            g = ["(changed) " + i for i in same_slicer(before, alo, ahi)]
+            g += ["(into   ) " + i for i in same_slicer(after, blo, bhi)]
+            yield g
+        elif tag == 'insert':
+            g = ["(added  ) " + i for i in same_slicer(after, blo, bhi)]
+            yield g
+
+# only_differences - only return info about the differences, no context
+# line_feed_sep could be "<br/>" or "<li>" or "\n" etc
+def render_diff(previous_file, newest_file, include_equal=False, line_feed_sep="\n"):
+    with open(newest_file, 'r') as f:
+        newest_version_file_contents = f.read()
+        newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]
+
+    if previous_file:
+        with open(previous_file, 'r') as f:
+            previous_version_file_contents = f.read()
+            previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
+    else:
+        previous_version_file_contents = ""
+
+    rendered_diff = customSequenceMatcher(previous_version_file_contents,
+                                          newest_version_file_contents,
+                                          include_equal)
+
+    # Recursively join lists
+    f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
+    return f(rendered_diff)

changedetectionio/fetch_site_status.py

@@ -0,0 +1,200 @@
+import hashlib
+import os
+import re
+import time
+import urllib3
+
+from changedetectionio import content_fetcher, html_tools
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+# Some common stuff here that can be moved to a base class
+class perform_site_check():
+
+    def __init__(self, *args, datastore, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.datastore = datastore
+
+    def run(self, uuid):
+        timestamp = int(time.time())  # used for storage etc too
+
+        changed_detected = False
+        screenshot = False  # as bytes
+        stripped_text_from_html = ""
+
+        watch = self.datastore.data['watching'][uuid]
+
+        # Protect against file:// access
+        if re.search(r'^file', watch['url'], re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
+            raise Exception(
+                "file:// type access is denied for security reasons."
+            )
+
+        # Unset any existing notification error
+        update_obj = {'last_notification_error': False, 'last_error': False}
+
+        extra_headers = self.datastore.get_val(uuid, 'headers')
+
+        # Tweak the base config with the per-watch ones
+        request_headers = self.datastore.data['settings']['headers'].copy()
+        request_headers.update(extra_headers)
+
+        # https://github.com/psf/requests/issues/4525
+        # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
+        # do this by accident.
+        if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
+            request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
+
+        timeout = self.datastore.data['settings']['requests']['timeout']
+        url = self.datastore.get_val(uuid, 'url')
+        request_body = self.datastore.get_val(uuid, 'body')
+        request_method = self.datastore.get_val(uuid, 'method')
+        ignore_status_code = self.datastore.get_val(uuid, 'ignore_status_codes')
+
+        # source: support
+        is_source = False
+        if url.startswith('source:'):
+            url = url.replace('source:', '')
+            is_source = True
+
+        # Pluggable content fetcher
+        prefer_backend = watch['fetch_backend']
+        if hasattr(content_fetcher, prefer_backend):
+            klass = getattr(content_fetcher, prefer_backend)
+        else:
+            # If the klass doesnt exist, just use a default
+            klass = getattr(content_fetcher, "html_requests")
+
+        fetcher = klass()
+        fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code)
+        # Fetching complete, now filters
+        # @todo move to class / maybe inside of fetcher abstract base?
+
+        # @note: I feel like the following should be in a more obvious chain system
+        #  - Check filter text
+        #  - Is the checksum different?
+        #  - Do we convert to JSON?
+        # https://stackoverflow.com/questions/41817578/basic-method-chaining ?
+        # return content().textfilter().jsonextract().checksumcompare() ?
+
+        is_json = 'application/json' in fetcher.headers.get('Content-Type', '')
+        is_html = not is_json
+
+        # source: support, basically treat it as plaintext
+        if is_source:
+            is_html = False
+            is_json = False
+
+        css_filter_rule = watch['css_filter']
+        subtractive_selectors = watch.get(
+            "subtractive_selectors", []
+        ) + self.datastore.data["settings"]["application"].get(
+            "global_subtractive_selectors", []
+        )
+
+        has_filter_rule = css_filter_rule and len(css_filter_rule.strip())
+        has_subtractive_selectors = subtractive_selectors and len(subtractive_selectors[0].strip())
+
+        if is_json and not has_filter_rule:
+            css_filter_rule = "json:$"
+            has_filter_rule = True
+
+        if has_filter_rule:
+            if 'json:' in css_filter_rule:
+                stripped_text_from_html = html_tools.extract_json_as_string(content=fetcher.content, jsonpath_filter=css_filter_rule)
+                is_html = False
+
+        if is_html or is_source:
+            # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+            html_content = fetcher.content
+
+            # If not JSON,  and if it's not text/plain..
+            if 'text/plain' in fetcher.headers.get('Content-Type', '').lower():
+                # Don't run get_text or xpath/css filters on plaintext
+                stripped_text_from_html = html_content
+            else:
+                # Then we assume HTML
+                if has_filter_rule:
+                    # For HTML/XML we offer xpath as an option, just start a regular xPath "/.."
+                    if css_filter_rule[0] == '/':
+                        html_content = html_tools.xpath_filter(xpath_filter=css_filter_rule, html_content=fetcher.content)
+                    else:
+                        # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+                        html_content = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
+                if has_subtractive_selectors:
+                    html_content = html_tools.element_removal(subtractive_selectors, html_content)
+
+                if not is_source:
+                    # extract text
+                    stripped_text_from_html = \
+                        html_tools.html_to_text(
+                            html_content,
+                            render_anchor_tag_content=self.datastore.data["settings"][
+                                "application"].get(
+                                "render_anchor_tag_content", False)
+                        )
+
+                elif is_source:
+                    stripped_text_from_html = html_content
+
+            # Re #340 - return the content before the 'ignore text' was applied
+            text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
+
+
+        # Re #340 - return the content before the 'ignore text' was applied
+        text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8')
+
+        # We rely on the actual text in the html output.. many sites have random script vars etc,
+        # in the future we'll implement other mechanisms.
+
+        update_obj["last_check_status"] = fetcher.get_last_status_code()
+
+        # If there's text to skip
+        # @todo we could abstract out the get_text() to handle this cleaner
+        text_to_ignore = watch.get('ignore_text', []) + self.datastore.data['settings']['application'].get('global_ignore_text', [])
+        if len(text_to_ignore):
+            stripped_text_from_html = html_tools.strip_ignore_text(stripped_text_from_html, text_to_ignore)
+        else:
+            stripped_text_from_html = stripped_text_from_html.encode('utf8')
+
+        # Re #133 - if we should strip whitespaces from triggering the change detected comparison
+        if self.datastore.data['settings']['application'].get('ignore_whitespace', False):
+            fetched_md5 = hashlib.md5(stripped_text_from_html.translate(None, b'\r\n\t ')).hexdigest()
+        else:
+            fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest()
+
+        # On the first run of a site, watch['previous_md5'] will be None, set it the current one.
+        if not watch.get('previous_md5'):
+            watch['previous_md5'] = fetched_md5
+            update_obj["previous_md5"] = fetched_md5
+
+        blocked_by_not_found_trigger_text = False
+
+        if len(watch['trigger_text']):
+            # Yeah, lets block first until something matches
+            blocked_by_not_found_trigger_text = True
+            # Filter and trigger works the same, so reuse it
+            result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),
+                                                  wordlist=watch['trigger_text'],
+                                                  mode="line numbers")
+            if result:
+                blocked_by_not_found_trigger_text = False
+
+        if not blocked_by_not_found_trigger_text and watch['previous_md5'] != fetched_md5:
+            changed_detected = True
+            update_obj["previous_md5"] = fetched_md5
+            update_obj["last_changed"] = timestamp
+
+        # Extract title as title
+        if is_html:
+            if self.datastore.data['settings']['application']['extract_title_as_title'] or watch['extract_title_as_title']:
+                if not watch['title'] or not len(watch['title']):
+                    update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
+
+        if self.datastore.data['settings']['application'].get('real_browser_save_screenshot', True):
+            screenshot = fetcher.screenshot()
+
+        fetcher.quit()
+
+        return changed_detected, update_obj, text_content_before_ignored_filter, screenshot

changedetectionio/forms.py

@@ -0,0 +1,385 @@
+import re
+
+from wtforms import (
+    BooleanField,
+    Field,
+    Form,
+    IntegerField,
+    PasswordField,
+    RadioField,
+    SelectField,
+    StringField,
+    SubmitField,
+    TextAreaField,
+    fields,
+    validators,
+    widgets,
+)
+from wtforms.validators import ValidationError
+
+from changedetectionio import content_fetcher
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+    valid_notification_formats,
+)
+
+from wtforms.fields import FormField
+
+valid_method = {
+    'GET',
+    'POST',
+    'PUT',
+    'PATCH',
+    'DELETE',
+}
+
+default_method = 'GET'
+
+
+class StringListField(StringField):
+    widget = widgets.TextArea()
+
+    def _value(self):
+        if self.data:
+            # ignore empty lines in the storage
+            data = list(filter(lambda x: len(x.strip()), self.data))
+            # Apply strip to each line
+            data = list(map(lambda x: x.strip(), data))
+            return "\r\n".join(data)
+        else:
+            return u''
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist and len(valuelist[0].strip()):
+            # Remove empty strings, stripping and splitting \r\n, only \n etc.
+            self.data = valuelist[0].splitlines()
+            # Remove empty lines from the final data
+            self.data = list(filter(lambda x: len(x.strip()), self.data))
+        else:
+            self.data = []
+
+
+class SaltyPasswordField(StringField):
+    widget = widgets.PasswordInput()
+    encrypted_password = ""
+
+    def build_password(self, password):
+        import base64
+        import hashlib
+        import secrets
+
+        # Make a new salt on every new password and store it with the password
+        salt = secrets.token_bytes(32)
+
+        key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
+        store = base64.b64encode(salt + key).decode('ascii')
+
+        return store
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            # Be really sure it's non-zero in length
+            if len(valuelist[0].strip()) > 0:
+                self.encrypted_password = self.build_password(valuelist[0])
+                self.data = ""
+        else:
+            self.data = False
+
+class TimeBetweenCheckForm(Form):
+    weeks = IntegerField('Weeks', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
+    days = IntegerField('Days', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
+    hours = IntegerField('Hours', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
+    minutes = IntegerField('Minutes', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
+    seconds = IntegerField('Seconds', validators=[validators.Optional(), validators.NumberRange(min=0, message="Should contain zero or more seconds")])
+    # @todo add total seconds minimum validatior = minimum_seconds_recheck_time
+
+# Separated by  key:value
+class StringDictKeyValue(StringField):
+    widget = widgets.TextArea()
+
+    def _value(self):
+        if self.data:
+            output = u''
+            for k in self.data.keys():
+                output += "{}: {}\r\n".format(k, self.data[k])
+
+            return output
+        else:
+            return u''
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            self.data = {}
+            # Remove empty strings
+            cleaned = list(filter(None, valuelist[0].split("\n")))
+            for s in cleaned:
+                parts = s.strip().split(':', 1)
+                if len(parts) == 2:
+                    self.data.update({parts[0].strip(): parts[1].strip()})
+
+        else:
+            self.data = {}
+
+class ValidateContentFetcherIsReady(object):
+    """
+    Validates that anything that looks like a regex passes as a regex
+    """
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        import urllib3.exceptions
+        from changedetectionio import content_fetcher
+
+        # Better would be a radiohandler that keeps a reference to each class
+        if field.data is not None:
+            klass = getattr(content_fetcher, field.data)
+            some_object = klass()
+            try:
+                ready = some_object.is_ready()
+
+            except urllib3.exceptions.MaxRetryError as e:
+                driver_url = some_object.command_executor
+                message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data))
+                message += '<br/>' + field.gettext(
+                    'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.')
+                message += '<br/>' + field.gettext('Did you follow the instructions in the wiki?')
+                message += '<br/><br/>' + field.gettext('WebDriver Host: %s' % (driver_url))
+                message += '<br/><a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">Go here for more information</a>'
+                message += '<br/>'+field.gettext('Content fetcher did not respond properly, unable to use it.\n %s' % (str(e)))
+
+                raise ValidationError(message)
+
+            except Exception as e:
+                message = field.gettext('Content fetcher \'%s\' did not respond properly, unable to use it.\n %s')
+                raise ValidationError(message % (field.data, e))
+
+
+class ValidateNotificationBodyAndTitleWhenURLisSet(object):
+    """
+       Validates that they entered something in both notification title+body when the URL is set
+       Due to https://github.com/dgtlmoon/changedetection.io/issues/360
+       """
+
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        if len(field.data):
+            if not len(form.notification_title.data) or not len(form.notification_body.data):
+                message = field.gettext('Notification Body and Title is required when a Notification URL is used')
+                raise ValidationError(message)
+
+class ValidateAppRiseServers(object):
+    """
+       Validates that each URL given is compatible with AppRise
+       """
+
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        import apprise
+        apobj = apprise.Apprise()
+
+        for server_url in field.data:
+            if not apobj.add(server_url):
+                message = field.gettext('\'%s\' is not a valid AppRise URL.' % (server_url))
+                raise ValidationError(message)
+
+class ValidateTokensList(object):
+    """
+    Validates that a {token} is from a valid set
+    """
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        from changedetectionio import notification
+        regex = re.compile('{.*?}')
+        for p in re.findall(regex, field.data):
+            if not p.strip('{}') in notification.valid_tokens:
+                message = field.gettext('Token \'%s\' is not a valid token.')
+                raise ValidationError(message % (p))
+            
+class validateURL(object):
+    
+    """
+       Flask wtform validators wont work with basic auth
+    """
+
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        import validators
+        try:
+            validators.url(field.data.strip())
+        except validators.ValidationFailure:
+            message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
+            raise ValidationError(message)
+        
+class ValidateListRegex(object):
+    """
+    Validates that anything that looks like a regex passes as a regex
+    """
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+
+        for line in field.data:
+            if line[0] == '/' and line[-1] == '/':
+                # Because internally we dont wrap in /
+                line = line.strip('/')
+                try:
+                    re.compile(line)
+                except re.error:
+                    message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
+                    raise ValidationError(message % (line))
+
+class ValidateCSSJSONXPATHInput(object):
+    """
+    Filter validation
+    @todo CSS validator ;)
+    """
+
+    def __init__(self, message=None, allow_xpath=True, allow_json=True):
+        self.message = message
+        self.allow_xpath = allow_xpath
+        self.allow_json = allow_json
+
+    def __call__(self, form, field):
+
+        if isinstance(field.data, str):
+            data = [field.data]
+        else:
+            data = field.data
+
+        for line in data:
+        # Nothing to see here
+            if not len(line.strip()):
+                return
+
+            # Does it look like XPath?
+            if line.strip()[0] == '/':
+                if not self.allow_xpath:
+                    raise ValidationError("XPath not permitted in this field!")
+                from lxml import etree, html
+                tree = html.fromstring("<html></html>")
+
+                try:
+                    tree.xpath(line.strip())
+                except etree.XPathEvalError as e:
+                    message = field.gettext('\'%s\' is not a valid XPath expression. (%s)')
+                    raise ValidationError(message % (line, str(e)))
+                except:
+                    raise ValidationError("A system-error occurred when validating your XPath expression")
+
+            if 'json:' in line:
+                if not self.allow_json:
+                    raise ValidationError("JSONPath not permitted in this field!")
+
+                from jsonpath_ng.exceptions import (
+                    JsonPathLexerError,
+                    JsonPathParserError,
+                )
+                from jsonpath_ng.ext import parse
+
+                input = line.replace('json:', '')
+
+                try:
+                    parse(input)
+                except (JsonPathParserError, JsonPathLexerError) as e:
+                    message = field.gettext('\'%s\' is not a valid JSONPath expression. (%s)')
+                    raise ValidationError(message % (input, str(e)))
+                except:
+                    raise ValidationError("A system-error occurred when validating your JSONPath expression")
+
+                # Re #265 - maybe in the future fetch the page and offer a
+                # warning/notice that its possible the rule doesnt yet match anything?
+
+
+class quickWatchForm(Form):
+    url = fields.URLField('URL', validators=[validateURL()])
+    tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)])
+
+# Common to a single watch and the global settings
+class commonSettingsForm(Form):
+
+    notification_urls = StringListField('Notification URL list', validators=[validators.Optional(), ValidateNotificationBodyAndTitleWhenURLisSet(), ValidateAppRiseServers()])
+    notification_title = StringField('Notification title', default=default_notification_title, validators=[validators.Optional(), ValidateTokensList()])
+    notification_body = TextAreaField('Notification body', default=default_notification_body, validators=[validators.Optional(), ValidateTokensList()])
+    notification_format = SelectField('Notification format', choices=valid_notification_formats.keys(), default=default_notification_format)
+    fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
+
+class watchForm(commonSettingsForm):
+
+    url = fields.URLField('URL', validators=[validateURL()])
+    tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)], default='')
+
+    time_between_check = FormField(TimeBetweenCheckForm)
+
+    css_filter = StringField('CSS/JSON/XPATH Filter', [ValidateCSSJSONXPATHInput()], default='')
+
+    subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
+    title = StringField('Title', default='')
+
+    ignore_text = StringListField('Ignore text', [ValidateListRegex()])
+    headers = StringDictKeyValue('Request headers')
+    body = TextAreaField('Request body', [validators.Optional()])
+    method = SelectField('Request method', choices=valid_method, default=default_method)
+    ignore_status_codes = BooleanField('Ignore status codes (process non-2xx status codes as normal)', default=False)
+    trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
+
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+    save_and_preview_button = SubmitField('Save & Preview', render_kw={"class": "pure-button pure-button-primary"})
+
+    def validate(self, **kwargs):
+        if not super().validate():
+            return False
+
+        result = True
+
+        # Fail form validation when a body is set for a GET
+        if self.method.data == 'GET' and self.body.data:
+            self.body.errors.append('Body must be empty when Request Method is set to GET')
+            result = False
+
+        return result
+
+
+# datastore.data['settings']['requests']..
+class globalSettingsRequestForm(Form):
+    time_between_check = FormField(TimeBetweenCheckForm)
+
+
+# datastore.data['settings']['application']..
+class globalSettingsApplicationForm(commonSettingsForm):
+
+    base_url = StringField('Base URL', validators=[validators.Optional()])
+    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
+    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
+    ignore_whitespace = BooleanField('Ignore whitespace')
+    real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome?')
+    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
+    render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
+    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    password = SaltyPasswordField()
+
+
+class globalSettingsForm(Form):
+    # Define these as FormFields/"sub forms", this way it matches the JSON storage
+    # datastore.data['settings']['application']..
+    # datastore.data['settings']['requests']..
+
+    requests = FormField(globalSettingsRequestForm)
+    application = FormField(globalSettingsApplicationForm)
+    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
+

changedetectionio/html_tools.py

@@ -0,0 +1,204 @@
+import json
+import re
+from typing import List
+
+from bs4 import BeautifulSoup
+from jsonpath_ng.ext import parse
+import re
+from inscriptis import get_text
+from inscriptis.model.config import ParserConfig
+
+
+class JSONNotFound(ValueError):
+    def __init__(self, msg):
+        ValueError.__init__(self, msg)
+
+# Given a CSS Rule, and a blob of HTML, return the blob of HTML that matches
+def css_filter(css_filter, html_content):
+    soup = BeautifulSoup(html_content, "html.parser")
+    html_block = ""
+    for item in soup.select(css_filter, separator=""):
+        html_block += str(item)
+
+    return html_block + "\n"
+
+def subtractive_css_selector(css_selector, html_content):
+    soup = BeautifulSoup(html_content, "html.parser")
+    for item in soup.select(css_selector):
+        item.decompose()
+    return str(soup)
+
+
+def element_removal(selectors: List[str], html_content):
+    """Joins individual filters into one css filter."""
+    selector = ",".join(selectors)
+    return subtractive_css_selector(selector, html_content)
+
+
+# Return str Utf-8 of matched rules
+def xpath_filter(xpath_filter, html_content):
+    from lxml import etree, html
+
+    tree = html.fromstring(html_content)
+    html_block = ""
+
+    for item in tree.xpath(xpath_filter.strip(), namespaces={'re':'http://exslt.org/regular-expressions'}):
+        html_block+= etree.tostring(item, pretty_print=True).decode('utf-8')+"<br/>"
+
+    return html_block
+
+
+# Extract/find element
+def extract_element(find='title', html_content=''):
+
+    #Re #106, be sure to handle when its not found
+    element_text = None
+
+    soup = BeautifulSoup(html_content, 'html.parser')
+    result = soup.find(find)
+    if result and result.string:
+        element_text = result.string.strip()
+
+    return element_text
+
+#
+def _parse_json(json_data, jsonpath_filter):
+    s=[]
+    jsonpath_expression = parse(jsonpath_filter.replace('json:', ''))
+    match = jsonpath_expression.find(json_data)
+
+    # More than one result, we will return it as a JSON list.
+    if len(match) > 1:
+        for i in match:
+            s.append(i.value)
+
+    # Single value, use just the value, as it could be later used in a token in notifications.
+    if len(match) == 1:
+        s = match[0].value
+
+    # Re #257 - Better handling where it does not exist, in the case the original 's' value was False..
+    if not match:
+        # Re 265 - Just return an empty string when filter not found
+        return ''
+
+    # Ticket #462 - allow the original encoding through, usually it's UTF-8 or similar
+    stripped_text_from_html = json.dumps(s, indent=4, ensure_ascii=False)
+
+    return stripped_text_from_html
+
+def extract_json_as_string(content, jsonpath_filter):
+
+    stripped_text_from_html = False
+
+    # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded <script type=ldjson>
+    try:
+        stripped_text_from_html = _parse_json(json.loads(content), jsonpath_filter)
+    except json.JSONDecodeError:
+
+        # Foreach <script json></script> blob.. just return the first that matches jsonpath_filter
+        s = []
+        soup = BeautifulSoup(content, 'html.parser')
+        bs_result = soup.findAll('script')
+
+        if not bs_result:
+            raise JSONNotFound("No parsable JSON found in this document")
+
+        for result in bs_result:
+            # Skip empty tags, and things that dont even look like JSON
+            if not result.string or not '{' in result.string:
+                continue
+                
+            try:
+                json_data = json.loads(result.string)
+            except json.JSONDecodeError:
+                # Just skip it
+                continue
+            else:
+                stripped_text_from_html = _parse_json(json_data, jsonpath_filter)
+                if stripped_text_from_html:
+                    break
+
+    if not stripped_text_from_html:
+        # Re 265 - Just return an empty string when filter not found
+        return ''
+
+    return stripped_text_from_html
+
+# Mode     - "content" return the content without the matches (default)
+#          - "line numbers" return a list of line numbers that match (int list)
+#
+# wordlist - list of regex's (str) or words (str)
+def strip_ignore_text(content, wordlist, mode="content"):
+    ignore = []
+    ignore_regex = []
+
+    # @todo check this runs case insensitive
+    for k in wordlist:
+
+        # Is it a regex?
+        if k[0] == '/':
+            ignore_regex.append(k.strip(" /"))
+        else:
+            ignore.append(k)
+
+    i = 0
+    output = []
+    ignored_line_numbers = []
+    for line in content.splitlines():
+        i += 1
+        # Always ignore blank lines in this mode. (when this function gets called)
+        if len(line.strip()):
+            regex_matches = False
+
+            # if any of these match, skip
+            for regex in ignore_regex:
+                try:
+                    if re.search(regex, line, re.IGNORECASE):
+                        regex_matches = True
+                except Exception as e:
+                    continue
+
+            if not regex_matches and not any(skip_text.lower() in line.lower() for skip_text in ignore):
+                output.append(line.encode('utf8'))
+            else:
+                ignored_line_numbers.append(i)
+
+
+
+    # Used for finding out what to highlight
+    if mode == "line numbers":
+        return ignored_line_numbers
+
+    return "\n".encode('utf8').join(output)
+
+
+def html_to_text(html_content: str, render_anchor_tag_content=False) -> str:
+    """Converts html string to a string with just the text. If ignoring
+    rendering anchor tag content is enable, anchor tag content are also
+    included in the text
+
+    :param html_content: string with html content
+    :param render_anchor_tag_content: boolean flag indicating whether to extract
+    hyperlinks (the anchor tag content) together with text. This refers to the
+    'href' inside 'a' tags.
+    Anchor tag content is rendered in the following manner:
+    '[ text ](anchor tag content)'
+    :return: extracted text from the HTML
+    """
+    #  if anchor tag content flag is set to True define a config for
+    #  extracting this content
+    if render_anchor_tag_content:
+
+        parser_config = ParserConfig(
+            annotation_rules={"a": ["hyperlink"]}, display_links=True
+        )
+
+    # otherwise set config to None
+    else:
+        parser_config = None
+
+    # get text and annotations via inscriptis
+    text_content = get_text(html_content, config=parser_config)
+
+    return text_content
+

changedetectionio/model/App.py

@@ -0,0 +1,50 @@
+import collections
+import os
+
+import uuid as uuid_builder
+
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+)
+
+class model(dict):
+    base_config = {
+            'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
+            'watching': {},
+            'settings': {
+                'headers': {
+                    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
+                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+                    'Accept-Encoding': 'gzip, deflate',  # No support for brolti in python requests yet.
+                    'Accept-Language': 'en-GB,en-US;q=0.9,en;'
+                },
+                'requests': {
+                    'timeout': 15,  # Default 15 seconds
+                    'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},
+                    'workers': 10  # Number of threads, lower is better for slow connections
+                },
+                'application': {
+                    'password': False,
+                    'base_url' : None,
+                    'extract_title_as_title': False,
+                    'fetch_backend': os.getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
+                    'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
+                    'global_subtractive_selectors': [],
+                    'ignore_whitespace': False,
+                    'render_anchor_tag_content': False,
+                    'notification_urls': [], # Apprise URL list
+                    # Custom notification content
+                    'notification_title': default_notification_title,
+                    'notification_body': default_notification_body,
+                    'notification_format': default_notification_format,
+                    'real_browser_save_screenshot': True,
+                    'schema_version' : 0
+                }
+            }
+        }
+
+    def __init__(self, *arg, **kw):
+        super(model, self).__init__(*arg, **kw)
+        self.update(self.base_config)

changedetectionio/model/Watch.py

@@ -0,0 +1,68 @@
+import os
+
+import uuid as uuid_builder
+
+minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
+
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+)
+
+
+class model(dict):
+    base_config = {
+            'url': None,
+            'tag': None,
+            'last_checked': 0,
+            'last_changed': 0,
+            'paused': False,
+            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
+            'newest_history_key': 0,
+            'title': None,
+            'previous_md5': False,
+#           UUID not needed, should be generated only as a key
+#            'uuid':
+            'headers': {},  # Extra headers to send
+            'body': None,
+            'method': 'GET',
+            'history': {},  # Dict of timestamp and output stripped filename
+            'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
+            # Custom notification content
+            'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
+            'notification_title': default_notification_title,
+            'notification_body': default_notification_body,
+            'notification_format': default_notification_format,
+            'css_filter': "",
+            'subtractive_selectors': [],
+            'trigger_text': [],  # List of text or regex to wait for until a change is detected
+            'fetch_backend': None,
+            'extract_title_as_title': False,
+            # Re #110, so then if this is set to None, we know to use the default value instead
+            # Requires setting to None on submit if it's the same as the default
+            # Should be all None by default, so we use the system default in this case.
+            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None}
+        }
+
+    def __init__(self, *arg, **kw):
+        self.update(self.base_config)
+        # goes at the end so we update the default object with the initialiser
+        super(model, self).__init__(*arg, **kw)
+
+
+    @property
+    def has_empty_checktime(self):
+        # using all() + dictionary comprehension
+        # Check if all values are 0 in dictionary
+        res = all(x == None or x == False or x==0 for x in self.get('time_between_check', {}).values())
+        return res
+
+    def threshold_seconds(self):
+        seconds = 0
+        mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
+        for m, n in mtable.items():
+            x = self.get('time_between_check', {}).get(m, None)
+            if x:
+                seconds += x * n
+        return seconds

changedetectionio/notification.py

@@ -0,0 +1,145 @@
+import apprise
+from apprise import NotifyFormat
+
+valid_tokens = {
+    'base_url': '',
+    'watch_url': '',
+    'watch_uuid': '',
+    'watch_title': '',
+    'watch_tag': '',
+    'diff': '',
+    'diff_full': '',
+    'diff_url': '',
+    'preview_url': '',
+    'current_snapshot': ''
+}
+
+valid_notification_formats = {
+    'Text': NotifyFormat.TEXT,
+    'Markdown': NotifyFormat.MARKDOWN,
+    'HTML': NotifyFormat.HTML,
+}
+
+default_notification_format = 'Text'
+default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n'
+default_notification_title = 'ChangeDetection.io Notification - {watch_url}'
+
+def process_notification(n_object, datastore):
+
+    # Get the notification body from datastore
+    n_body = n_object.get('notification_body', default_notification_body)
+    n_title = n_object.get('notification_title', default_notification_title)
+    n_format = valid_notification_formats.get(
+        n_object['notification_format'],
+        valid_notification_formats[default_notification_format],
+    )
+
+
+    # Insert variables into the notification content
+    notification_parameters = create_notification_parameters(n_object, datastore)
+
+    for n_k in notification_parameters:
+        token = '{' + n_k + '}'
+        val = notification_parameters[n_k]
+        n_title = n_title.replace(token, val)
+        n_body = n_body.replace(token, val)
+
+    # https://github.com/caronc/apprise/wiki/Development_LogCapture
+    # Anything higher than or equal to WARNING (which covers things like Connection errors)
+    # raise it as an exception
+    apobjs=[]
+    for url in n_object['notification_urls']:
+
+        apobj = apprise.Apprise(debug=True)
+        url = url.strip()
+        if len(url):
+            print(">> Process Notification: AppRise notifying {}".format(url))
+            with apprise.LogCapture(level=apprise.logging.DEBUG) as logs:
+                # Re 323 - Limit discord length to their 2000 char limit total or it wont send.
+                # Because different notifications may require different pre-processing, run each sequentially :(
+                # 2000 bytes minus -
+                #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers
+                #     Length of URL - Incase they specify a longer custom avatar_url
+
+                # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
+                k = '?' if not '?' in url else '&'
+                if not 'avatar_url' in url:
+                    url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
+
+                if url.startswith('tgram://'):
+                    # real limit is 4096, but minus some for extra metadata
+                    payload_max_size = 3600
+                    body_limit = max(0, payload_max_size - len(n_title))
+                    n_title = n_title[0:payload_max_size]
+                    n_body = n_body[0:body_limit]
+
+                elif url.startswith('discord://'):
+                    # real limit is 2000, but minus some for extra metadata
+                    payload_max_size = 1700
+                    body_limit = max(0, payload_max_size - len(n_title))
+                    n_title = n_title[0:payload_max_size]
+                    n_body = n_body[0:body_limit]
+
+                apobj.add(url)
+
+                apobj.notify(
+                    title=n_title,
+                    body=n_body,
+                    body_format=n_format)
+
+                apobj.clear()
+
+                # Incase it needs to exist in memory for a while after to process(?)
+                apobjs.append(apobj)
+
+                # Returns empty string if nothing found, multi-line string otherwise
+                log_value = logs.getvalue()
+                if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
+                    raise Exception(log_value)
+
+# Notification title + body content parameters get created here.
+def create_notification_parameters(n_object, datastore):
+    from copy import deepcopy
+
+    # in the case we send a test notification from the main settings, there is no UUID.
+    uuid = n_object['uuid'] if 'uuid' in n_object else ''
+
+    if uuid != '':
+        watch_title = datastore.data['watching'][uuid]['title']
+        watch_tag = datastore.data['watching'][uuid]['tag']
+    else:
+        watch_title = 'Change Detection'
+        watch_tag = ''
+
+    # Create URLs to customise the notification with
+    base_url = datastore.data['settings']['application']['base_url']
+
+    watch_url = n_object['watch_url']
+
+    # Re #148 - Some people have just {base_url} in the body or title, but this may break some notification services
+    #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.
+    if base_url == '':
+        base_url = "<base-url-env-var-not-set>"
+
+    diff_url = "{}/diff/{}".format(base_url, uuid)
+    preview_url = "{}/preview/{}".format(base_url, uuid)
+
+    # Not sure deepcopy is needed here, but why not
+    tokens = deepcopy(valid_tokens)
+
+    # Valid_tokens also used as a field validator
+    tokens.update(
+        {
+            'base_url': base_url if base_url is not None else '',
+            'watch_url': watch_url,
+            'watch_uuid': uuid,
+            'watch_title': watch_title if watch_title is not None else '',
+            'watch_tag': watch_tag if watch_tag is not None else '',
+            'diff_url': diff_url,
+            'diff': n_object.get('diff', ''),  # Null default in the case we use a test
+            'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
+            'preview_url': preview_url,
+            'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
+        })
+
+    return tokens

changedetectionio/pytest.ini

@@ -0,0 +1,12 @@
+[pytest]
+addopts = --no-start-live-server --live-server-port=5005
+#testpaths = tests pytest_invenio
+#live_server_scope = function
+
+filterwarnings =
+    ignore::DeprecationWarning:urllib3.*:
+
+; logging options
+log_cli = 1
+log_cli_level = DEBUG
+log_cli_format = %(asctime)s %(name)s: %(levelname)s %(message)s

changedetectionio/run_all_tests.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+
+# live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions
+# and I like to restart the server for each test (and have the test cleanup after each test)
+# merge request welcome :)
+
+
+# exit when any command fails
+set -e
+
+find tests/test_*py -type f|while read test_name
+do
+  echo "TEST RUNNING $test_name"
+  pytest $test_name
+done
+
+echo "RUNNING WITH BASE_URL SET"
+
+# Now re-run some tests with BASE_URL enabled
+# Re #65 - Ability to include a link back to the installation, in the notification.
+export BASE_URL="https://really-unique-domain.io"
+pytest tests/test_notification.py
+

changedetectionio/static/images/Generic_Feed-icon.svg

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg"
+     id="RSSicon"
+     viewBox="0 0 8 8" width="256" height="256">
+
+  <title>RSS feed icon</title>
+
+  <style type="text/css">
+    .button {stroke: none; fill: orange;}
+    .symbol {stroke: none; fill: white;}
+  </style>
+
+  <rect   class="button" width="8" height="8" rx="1.5" />
+  <circle class="symbol" cx="2" cy="6" r="1" />
+  <path   class="symbol" d="m 1,4 a 3,3 0 0 1 3,3 h 1 a 4,4 0 0 0 -4,-4 z" />
+  <path   class="symbol" d="m 1,2 a 5,5 0 0 1 5,5 h 1 a 6,6 0 0 0 -6,-6 z" />
+
+</svg>

changedetectionio/static/images/copy.svg

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   version="1.1"
+   id="Layer_1"
+   x="0px"
+   y="0px"
+   viewBox="0 0 115.77 122.88"
+   style="enable-background:new 0 0 115.77 122.88"
+   xml:space="preserve"
+   sodipodi:docname="copy.svg"
+   inkscape:version="1.1.1 (1:1.1+202109281949+c3084ef5ed)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg"><defs
+     id="defs11" /><sodipodi:namedview
+     id="namedview9"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     inkscape:zoom="5.5501303"
+     inkscape:cx="57.83648"
+     inkscape:cy="61.439999"
+     inkscape:window-width="1920"
+     inkscape:window-height="1056"
+     inkscape:window-x="1920"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="g6" /><style
+     type="text/css"
+     id="style2">.st0{fill-rule:evenodd;clip-rule:evenodd;}</style><g
+     id="g6"><path
+       class="st0"
+       d="M89.62,13.96v7.73h12.19h0.01v0.02c3.85,0.01,7.34,1.57,9.86,4.1c2.5,2.51,4.06,5.98,4.07,9.82h0.02v0.02 v73.27v0.01h-0.02c-0.01,3.84-1.57,7.33-4.1,9.86c-2.51,2.5-5.98,4.06-9.82,4.07v0.02h-0.02h-61.7H40.1v-0.02 c-3.84-0.01-7.34-1.57-9.86-4.1c-2.5-2.51-4.06-5.98-4.07-9.82h-0.02v-0.02V92.51H13.96h-0.01v-0.02c-3.84-0.01-7.34-1.57-9.86-4.1 c-2.5-2.51-4.06-5.98-4.07-9.82H0v-0.02V13.96v-0.01h0.02c0.01-3.85,1.58-7.34,4.1-9.86c2.51-2.5,5.98-4.06,9.82-4.07V0h0.02h61.7 h0.01v0.02c3.85,0.01,7.34,1.57,9.86,4.1c2.5,2.51,4.06,5.98,4.07,9.82h0.02V13.96L89.62,13.96z M79.04,21.69v-7.73v-0.02h0.02 c0-0.91-0.39-1.75-1.01-2.37c-0.61-0.61-1.46-1-2.37-1v0.02h-0.01h-61.7h-0.02v-0.02c-0.91,0-1.75,0.39-2.37,1.01 c-0.61,0.61-1,1.46-1,2.37h0.02v0.01v64.59v0.02h-0.02c0,0.91,0.39,1.75,1.01,2.37c0.61,0.61,1.46,1,2.37,1v-0.02h0.01h12.19V35.65 v-0.01h0.02c0.01-3.85,1.58-7.34,4.1-9.86c2.51-2.5,5.98-4.06,9.82-4.07v-0.02h0.02H79.04L79.04,21.69z M105.18,108.92V35.65v-0.02 h0.02c0-0.91-0.39-1.75-1.01-2.37c-0.61-0.61-1.46-1-2.37-1v0.02h-0.01h-61.7h-0.02v-0.02c-0.91,0-1.75,0.39-2.37,1.01 c-0.61,0.61-1,1.46-1,2.37h0.02v0.01v73.27v0.02h-0.02c0,0.91,0.39,1.75,1.01,2.37c0.61,0.61,1.46,1,2.37,1v-0.02h0.01h61.7h0.02 v0.02c0.91,0,1.75-0.39,2.37-1.01c0.61-0.61,1-1.46,1-2.37h-0.02V108.92L105.18,108.92z"
+       id="path4"
+       style="fill:#ffffff;fill-opacity:1" /></g></svg>

changedetectionio/static/images/pause.svg

@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   version="1.1"
+   id="Capa_1"
+   x="0px"
+   y="0px"
+   viewBox="0 0 15 14.998326"
+   xml:space="preserve"
+   width="15"
+   height="14.998326"><metadata
+   id="metadata39"><rdf:RDF><cc:Work
+       rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
+         rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
+   id="defs37" />
+<path
+   id="path2"
+   style="fill:#1b98f8;fill-opacity:1;stroke-width:0.0292893"
+   d="M 7.4975161,6.5052867e-4 C 4.549072,-0.04028702 1.7055675,1.8548221 0.58868606,4.5801341 -0.57739762,7.2574642 0.02596981,10.583326 2.069916,12.671949 4.0364753,14.788409 7.2763651,15.56067 9.989207,14.57284 12.801145,13.617602 14.87442,10.855325 14.985833,7.8845744 15.172496,4.9966544 13.49856,2.1100704 10.911002,0.8209349 9.8598067,0.28073592 8.6791261,-0.00114855 7.4975161,6.5052867e-4 Z M 6.5602569,10.251923 c -0.00509,0.507593 -0.5693885,0.488472 -0.9352002,0.468629 -0.3399386,0.0018 -0.8402048,0.07132 -0.9297965,-0.374189 -0.015842,-1.8973128 -0.015872,-3.7979649 0,-5.6952784 0.1334405,-0.5224315 0.7416869,-0.3424086 1.1377562,-0.374189 0.3969969,-0.084515 0.8245634,0.1963256 0.7272405,0.6382917 0,1.7789118 0,3.5578239 0,5.3367357 z m 3.7490371,0 c -0.0051,0.507593 -0.5693888,0.488472 -0.9352005,0.468629 -0.3399386,0.0018 -0.8402048,0.07132 -0.9297965,-0.374189 -0.015842,-1.8973128 -0.015872,-3.7979649 0,-5.6952784 0.1334405,-0.5224315 0.7416869,-0.3424086 1.1377562,-0.374189 0.3969969,-0.084515 0.8245638,0.1963256 0.7272408,0.6382917 0,1.7789118 0,3.5578239 0,5.3367357 z" />
+<g
+   id="g4"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g6"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g8"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g10"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g12"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g14"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g16"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g18"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g20"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g22"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g24"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g26"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g28"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g30"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g32"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+</svg>

changedetectionio/static/images/spread.svg

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   width="18"
+   height="19.92"
+   viewBox="0 0 18 19.92"
+   version="1.1"
+   id="svg6"
+   sodipodi:docname="spread.svg"
+   inkscape:version="1.1.1 (1:1.1+202109281949+c3084ef5ed)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs10" />
+  <sodipodi:namedview
+     id="namedview8"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0"
+     inkscape:zoom="28.416667"
+     inkscape:cx="9.0087975"
+     inkscape:cy="9.9941348"
+     inkscape:window-width="1920"
+     inkscape:window-height="1056"
+     inkscape:window-x="1920"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg6" />
+  <path
+     d="M -3,-2 H 21 V 22 H -3 Z"
+     fill="none"
+     id="path2" />
+  <path
+     d="m 15,14.08 c -0.76,0 -1.44,0.3 -1.96,0.77 L 5.91,10.7 C 5.96,10.47 6,10.24 6,10 6,9.76 5.96,9.53 5.91,9.3 L 12.96,5.19 C 13.5,5.69 14.21,6 15,6 16.66,6 18,4.66 18,3 18,1.34 16.66,0 15,0 c -1.66,0 -3,1.34 -3,3 0,0.24 0.04,0.47 0.09,0.7 L 5.04,7.81 C 4.5,7.31 3.79,7 3,7 1.34,7 0,8.34 0,10 c 0,1.66 1.34,3 3,3 0.79,0 1.5,-0.31 2.04,-0.81 l 7.12,4.16 c -0.05,0.21 -0.08,0.43 -0.08,0.65 0,1.61 1.31,2.92 2.92,2.92 1.61,0 2.92,-1.31 2.92,-2.92 0,-1.61 -1.31,-2.92 -2.92,-2.92 z"
+     id="path4"
+     style="fill:#0078e7;fill-opacity:1" />
+</svg>

changedetectionio/static/js/notifications.js

@@ -0,0 +1,53 @@
+$(document).ready(function() {
+
+  $('#add-email-helper').click(function (e) {
+    e.preventDefault();
+    email = prompt("Destination email");
+    if(email) {
+      var n = $(".notification-urls");
+      var p=email_notification_prefix;
+      $(n).val( $.trim( $(n).val() )+"\n"+email_notification_prefix+email );
+    }
+  });
+
+  $('#send-test-notification').click(function (e) {
+    e.preventDefault();
+
+    // this can be global
+    var csrftoken = $('input[name=csrf_token]').val();
+    $.ajaxSetup({
+        beforeSend: function(xhr, settings) {
+            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken)
+            }
+        }
+    })
+
+    data = {
+        window_url : window.location.href,
+        notification_urls : $('.notification-urls').val(),
+        notification_title : $('.notification-title').val(),
+        notification_body : $('.notification-body').val(),
+        notification_format : $('.notification-format').val(),
+    }
+    for (key in data) {
+      if (!data[key].length) {
+        alert(key+" is empty, cannot send test.")
+        return;
+      }
+    }
+
+    $.ajax({
+      type: "POST",
+      url: notification_base_url,
+      data : data
+    }).done(function(data){
+      console.log(data);
+      alert('Sent');
+    }).fail(function(data){
+      console.log(data);
+      alert('Error: '+data.responseJSON.error);
+    })
+  });
+});
+

changedetectionio/static/js/settings.js

@@ -0,0 +1,13 @@
+window.addEventListener("load", (event) => {
+  // just an example for now
+  function toggleVisible(elem) {
+    // theres better ways todo this
+    var x = document.getElementById(elem);
+    if (x.style.display === "block") {
+      x.style.display = "none";
+    } else {
+      x.style.display = "block";
+    }
+  }
+});
+

changedetectionio/static/js/tabs.js

@@ -0,0 +1,55 @@
+// Rewrite this is a plugin.. is all this JS really 'worth it?'
+
+
+if(!window.location.hash) {
+  var tab=document.querySelectorAll("#default-tab a");
+  tab[0].click();
+}
+
+window.addEventListener('hashchange', function() {
+  var tabs = document.getElementsByClassName('active');
+  while (tabs[0]) {
+    tabs[0].classList.remove('active')
+  }
+  set_active_tab();
+}, false);
+
+var has_errors=document.querySelectorAll(".messages .error");
+if (!has_errors.length) {
+    if (document.location.hash == "" ) {
+        document.location.hash = "#general";
+        document.getElementById("default-tab").className = "active";
+    } else {
+        set_active_tab();
+    }
+} else {
+  focus_error_tab();
+}
+
+function set_active_tab() {
+  var tab=document.querySelectorAll("a[href='"+location.hash+"']");
+  if (tab.length) {
+    tab[0].parentElement.className="active";
+  }
+    // hash could move the page down
+    window.scrollTo(0, 0);
+}
+
+function focus_error_tab() {
+  // time to use jquery or vuejs really,
+  // activate the tab with the error
+    var tabs = document.querySelectorAll('.tabs li a'),i;
+    for (i = 0; i < tabs.length; ++i) {
+      var tab_name=tabs[i].hash.replace('#','');
+      var pane_errors=document.querySelectorAll('#'+tab_name+' .error')
+      if (pane_errors.length) {
+        document.location.hash = '#'+tab_name;
+        return true;
+      }
+    }
+    return false;
+}
+
+
+
+

changedetectionio/static/js/watch-overview.js

@@ -0,0 +1,24 @@
+$(function () {
+  // Remove unviewed status when normally clicked
+  $('.diff-link').click(function () {
+    $(this).closest('.unviewed').removeClass('unviewed');
+  });
+
+  $('.with-share-link > *').click(function () {
+      $("#copied-clipboard").remove();
+
+      var range = document.createRange();
+      var n=$("#share-link")[0];
+      range.selectNode(n);
+      window.getSelection().removeAllRanges();
+      window.getSelection().addRange(range);
+      document.execCommand("copy");
+      window.getSelection().removeAllRanges();
+
+      $('.with-share-link').append('<span style="font-size: 80%; color: #fff;" id="copied-clipboard">Copied to clipboard</span>');
+      $("#copied-clipboard").fadeOut(2500, function() {
+       $(this).remove();
+      });
+  });
+});
+

changedetectionio/static/styles/.gitignore

@@ -0,0 +1 @@
+node_modules

changedetectionio/static/styles/diff.css

@@ -0,0 +1,78 @@
+#diff-ui {
+  background: #fff;
+  padding: 2em;
+  margin-left: 1em;
+  margin-right: 1em;
+  border-radius: 5px;
+  font-size: 11px; }
+  #diff-ui table {
+    table-layout: fixed;
+    width: 100%; }
+  #diff-ui td {
+    padding: 3px 4px;
+    border: 1px solid transparent;
+    vertical-align: top;
+    font: 1em monospace;
+    text-align: left; }
+  #diff-ui pre {
+    white-space: pre-wrap; }
+
+h1 {
+  display: inline;
+  font-size: 100%; }
+
+del {
+  text-decoration: none;
+  color: #b30000;
+  background: #fadad7; }
+
+ins {
+  background: #eaf2c2;
+  color: #406619;
+  text-decoration: none; }
+
+#result {
+  white-space: pre-wrap; }
+
+#settings {
+  background: rgba(0, 0, 0, 0.05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+  color: #fff;
+  font-size: 80%; }
+  #settings label {
+    margin-left: 1em;
+    display: inline-block;
+    font-weight: normal; }
+
+.source {
+  position: absolute;
+  right: 1%;
+  top: .2em; }
+
+@-moz-document url-prefix() {
+  body {
+    height: 99%;
+    /* Hide scroll bar in Firefox */ } }
+
+td#diff-col div {
+  text-align: justify;
+  white-space: pre-wrap; }
+
+.ignored {
+  background-color: #ccc;
+  /*  border: #0d91fa 1px solid; */
+  opacity: 0.7; }
+
+.triggered {
+  background-color: #1b98f8; }
+
+/* ignored and triggered? make it obvious error */
+.ignored.triggered {
+  background-color: #ff0000; }
+
+.tab-pane-inner#screenshot {
+  text-align: center; }
+  .tab-pane-inner#screenshot img {
+    max-width: 99%; }

changedetectionio/static/styles/diff.scss

@@ -0,0 +1,96 @@
+#diff-ui {
+
+    background: #fff;
+    padding: 2em;
+    margin-left: 1em;
+    margin-right: 1em;
+    border-radius: 5px;
+    font-size: 11px;
+
+    table {
+        table-layout: fixed;
+        width: 100%;
+    }
+    td {
+        padding: 3px 4px;
+        border: 1px solid transparent;
+        vertical-align: top;
+        font: 1em monospace;
+        text-align: left;
+    }
+    pre {
+            white-space: pre-wrap;
+    }
+}
+h1 {
+	display: inline;
+	font-size: 100%;
+}
+del {
+	text-decoration: none;
+	color: #b30000;
+	background: #fadad7;
+}
+
+ins {
+	background: #eaf2c2;
+	color: #406619;
+	text-decoration: none;
+}
+
+#result {
+	white-space: pre-wrap;
+}
+
+#settings {
+    background: rgba(0,0,0,.05);
+    padding: 1em;
+    border-radius: 10px;
+    margin-bottom: 1em;
+    color: #fff;
+    font-size: 80%;
+    label {
+	    margin-left: 1em;
+	    display: inline-block;
+	    font-weight: normal;
+    }
+}
+
+.source {
+	position: absolute;
+	right: 1%;
+	top: .2em;
+}
+
+@-moz-document url-prefix() {
+	body {
+		height: 99%; /* Hide scroll bar in Firefox */
+	}
+}
+
+td#diff-col div {
+    text-align: justify;
+    white-space: pre-wrap;
+}
+
+.ignored {
+    background-color: #ccc;
+   /*  border: #0d91fa 1px solid; */
+    opacity: 0.7;
+}
+
+.triggered {
+    background-color: #1b98f8;
+}
+
+/* ignored and triggered? make it obvious error */
+.ignored.triggered {
+  background-color: #ff0000;
+}
+
+.tab-pane-inner#screenshot {
+  text-align: center;
+  img {
+    max-width: 99%;
+  }
+}

changedetectionio/static/styles/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "changedetection.io-theme",
+  "version": "0.0.3",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "build": "node-sass styles.scss -o .;node-sass diff.scss -o ."
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "node-sass": "^7.0.0",
+    "tar": "^6.1.9",
+    "trim-newlines": "^3.0.1"
+  }
+}

changedetectionio/static/styles/styles.css

@@ -0,0 +1,450 @@
+/*
+ * -- BASE STYLES --
+ * Most of these are inherited from Base, but I want to change a few.
+ * nvm use v14.18.1
+ * npm install
+ * npm run build
+ * or npm run watch
+ */
+body {
+  color: #333;
+  background: #262626; }
+
+.pure-table-even {
+  background: #fff; }
+
+/* Some styles from https://css-tricks.com/ */
+a {
+  text-decoration: none;
+  color: #1b98f8; }
+
+a.github-link {
+  color: #fff; }
+
+.pure-menu-horizontal {
+  background: #fff;
+  padding: 5px;
+  display: flex;
+  justify-content: space-between;
+  border-bottom: 2px solid #ed5900;
+  align-items: center; }
+
+section.content {
+  padding-top: 5em;
+  padding-bottom: 1em;
+  flex-direction: column;
+  display: flex;
+  align-items: center;
+  justify-content: center; }
+
+code {
+  background: #eee; }
+
+/* table related */
+.watch-table {
+  width: 100%;
+  font-size: 80%; }
+  .watch-table tr.unviewed {
+    font-weight: bold; }
+  .watch-table .error {
+    color: #a00; }
+  .watch-table td {
+    white-space: nowrap; }
+  .watch-table td.title-col {
+    word-break: break-all;
+    white-space: normal; }
+  .watch-table th {
+    white-space: nowrap; }
+  .watch-table .title-col a[target="_blank"]::after, .watch-table .current-diff-url::after {
+    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+    margin: 0 3px 0 5px; }
+
+.watch-tag-list {
+  color: #e70069;
+  white-space: nowrap; }
+
+.box {
+  max-width: 80%;
+  flex-direction: column;
+  display: flex;
+  justify-content: center; }
+
+#post-list-buttons {
+  text-align: right;
+  padding: 0px;
+  margin: 0px; }
+  #post-list-buttons li {
+    display: inline-block; }
+  #post-list-buttons a {
+    border-top-left-radius: initial;
+    border-top-right-radius: initial;
+    border-bottom-left-radius: 5px;
+    border-bottom-right-radius: 5px; }
+
+body:after {
+  content: "";
+  background: linear-gradient(130deg, #5ad8f7, #2f50af 41.07%, #9150bf 84.05%); }
+
+body:after, body:before {
+  display: block;
+  height: 650px;
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  z-index: -1; }
+
+body::after {
+  opacity: 0.91; }
+
+body::before {
+  content: "";
+  background-size: cover; }
+
+body:after, body:before {
+  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
+  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%); }
+
+.arrow {
+  border: solid black;
+  border-width: 0 3px 3px 0;
+  display: inline-block;
+  padding: 3px; }
+  .arrow.right {
+    transform: rotate(-45deg);
+    -webkit-transform: rotate(-45deg); }
+  .arrow.left {
+    transform: rotate(135deg);
+    -webkit-transform: rotate(135deg); }
+  .arrow.up {
+    transform: rotate(-135deg);
+    -webkit-transform: rotate(-135deg); }
+  .arrow.down {
+    transform: rotate(45deg);
+    -webkit-transform: rotate(45deg); }
+
+.button-small {
+  font-size: 85%; }
+
+.fetch-error {
+  padding-top: 1em;
+  font-size: 60%;
+  max-width: 400px;
+  display: block; }
+
+.button-secondary {
+  color: white;
+  border-radius: 4px;
+  text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2); }
+
+.button-success {
+  background: #1cb841;
+  /* this is a green */ }
+
+.button-tag {
+  background: #636363;
+  color: #fff;
+  font-size: 65%;
+  border-bottom-left-radius: initial;
+  border-bottom-right-radius: initial; }
+  .button-tag.active {
+    background: #9c9c9c;
+    font-weight: bold; }
+
+.button-error {
+  background: #ca3c3c;
+  /* this is a maroon */ }
+
+.button-warning {
+  background: #df7514;
+  /* this is an orange */ }
+
+.button-secondary {
+  background: #42b8dd;
+  /* this is a light blue */ }
+
+.button-cancel {
+  background: #c8c8c8;
+  /* this is a green */ }
+
+.messages li {
+  list-style: none;
+  padding: 1em;
+  border-radius: 10px;
+  color: #fff;
+  font-weight: bold; }
+  .messages li.message {
+    background: rgba(255, 255, 255, 0.2); }
+  .messages li.error {
+    background: rgba(255, 1, 1, 0.5); }
+  .messages li.notice {
+    background: rgba(255, 255, 255, 0.5); }
+
+.messages.with-share-link > *:hover {
+  cursor: pointer; }
+
+#notification-customisation {
+  border: 1px solid #ccc;
+  padding: 0.5rem;
+  border-radius: 5px; }
+
+#notification-error-log {
+  border: 1px solid #ccc;
+  padding: 1rem;
+  border-radius: 5px;
+  overflow-wrap: break-word; }
+
+#token-table.pure-table td, #token-table.pure-table th {
+  font-size: 80%; }
+
+#new-watch-form {
+  background: rgba(0, 0, 0, 0.05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em; }
+  #new-watch-form input {
+    width: auto !important;
+    display: inline-block; }
+  #new-watch-form .label {
+    display: none; }
+  #new-watch-form legend {
+    color: #fff;
+    font-weight: bold; }
+
+#diff-col {
+  padding-left: 40px; }
+
+#diff-jump {
+  position: fixed;
+  left: 0px;
+  top: 120px;
+  background: #fff;
+  padding: 10px;
+  border-top-right-radius: 5px;
+  border-bottom-right-radius: 5px;
+  box-shadow: 5px 0 5px -2px #888; }
+  #diff-jump a {
+    color: #1b98f8;
+    cursor: grabbing;
+    -moz-user-select: none;
+    -webkit-user-select: none;
+    -ms-user-select: none;
+    user-select: none;
+    -o-user-select: none; }
+
+footer {
+  padding: 10px;
+  background: #fff;
+  color: #444;
+  text-align: center; }
+
+#feed-icon {
+  vertical-align: middle; }
+
+#top-right-menu {
+  /*
+    position: absolute;
+    right: 0px;
+    background: linear-gradient(to right, #fff0, #fff 10%);
+    padding-left: 20px;
+    padding-right: 10px;
+    */ }
+
+.sticky-tab {
+  position: absolute;
+  top: 60px;
+  font-size: 65%;
+  background: #fff;
+  padding: 10px; }
+  .sticky-tab#left-sticky {
+    left: 0px; }
+  .sticky-tab#right-sticky {
+    right: 0px; }
+  .sticky-tab#hosted-sticky {
+    right: 0px;
+    top: 100px;
+    font-weight: bold; }
+
+#new-version-text a {
+  color: #e07171; }
+
+.paused-state.state-False img {
+  opacity: 0.2; }
+
+.paused-state.state-False:hover img {
+  opacity: 0.8; }
+
+.monospaced-textarea textarea {
+  width: 100%;
+  font-family: monospace;
+  white-space: pre;
+  overflow-wrap: normal;
+  overflow-x: scroll; }
+
+.pure-form {
+  /* The input fields with errors */
+  /* The list of errors */ }
+  .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
+    padding-bottom: 1em; }
+    .pure-form .pure-control-group div, .pure-form .pure-group div, .pure-form .pure-controls div {
+      margin: 0px; }
+    .pure-form .pure-control-group .checkbox > *, .pure-form .pure-group .checkbox > *, .pure-form .pure-controls .checkbox > * {
+      display: inline;
+      vertical-align: middle; }
+    .pure-form .pure-control-group .checkbox > label, .pure-form .pure-group .checkbox > label, .pure-form .pure-controls .checkbox > label {
+      padding-left: 5px; }
+  .pure-form .error input {
+    background-color: #ffebeb; }
+  .pure-form ul.errors {
+    padding: .5em .6em;
+    border: 1px solid #dd0000;
+    border-radius: 4px;
+    vertical-align: middle;
+    -webkit-box-sizing: border-box;
+    box-sizing: border-box; }
+    .pure-form ul.errors li {
+      margin-left: 1em;
+      color: #dd0000; }
+  .pure-form label {
+    font-weight: bold; }
+  .pure-form textarea {
+    width: 100%; }
+  .pure-form ul.fetch-backend {
+    margin: 0px;
+    list-style: none; }
+    .pure-form ul.fetch-backend li > * {
+      display: inline-block; }
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+  .box {
+    max-width: 95%; }
+  .edit-form {
+    padding: 0.5em;
+    margin: 0; }
+  #nav-menu {
+    overflow-x: scroll; } }
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 800px) {
+  div.sticky-tab#hosted-sticky {
+    top: 60px;
+    left: 0px;
+    right: auto; }
+  section.content {
+    padding-top: 110px; }
+  div.tabs.collapsable ul li {
+    display: block;
+    border-radius: 0px; }
+  input[type='text'] {
+    width: 100%; }
+  /*
+Max width before this PARTICULAR table gets nasty
+This query will take effect for any screen smaller than 760px
+and also iPads specifically.
+*/
+  .watch-table {
+    /* Force table to not be like tables anymore */
+    /* Force table to not be like tables anymore */
+    /* Hide table headers (but not display: none;, for accessibility) */ }
+    .watch-table thead, .watch-table tbody, .watch-table th, .watch-table td, .watch-table tr {
+      display: block; }
+    .watch-table .last-checked::before {
+      color: #555;
+      content: "Last Checked "; }
+    .watch-table .last-changed::before {
+      color: #555;
+      content: "Last Changed "; }
+    .watch-table td.inline {
+      display: inline-block; }
+    .watch-table thead tr {
+      position: absolute;
+      top: -9999px;
+      left: -9999px; }
+    .watch-table .pure-table td, .watch-table .pure-table th {
+      border: none; }
+    .watch-table td {
+      /* Behave  like a "row" */
+      border: none;
+      border-bottom: 1px solid #eee; }
+      .watch-table td:before {
+        /* Top/left values mimic padding */
+        top: 6px;
+        left: 6px;
+        width: 45%;
+        padding-right: 10px;
+        white-space: nowrap; }
+    .watch-table.pure-table-striped tr {
+      background-color: #fff; }
+    .watch-table.pure-table-striped tr:nth-child(2n-1) {
+      background-color: #eee; }
+    .watch-table.pure-table-striped tr:nth-child(2n-1) td {
+      background-color: inherit; } }
+
+/** Desktop vs mobile input field strategy
+- We dont use 'size' with <input> because `size` is too unreliable to override, and will often push-out
+- Rely always on width in CSS
+*/
+@media only screen and (min-width: 761px) {
+  /* m-d is medium-desktop */
+  .m-d {
+    min-width: 80%; } }
+
+.tabs ul {
+  margin: 0px;
+  padding: 0px;
+  display: block; }
+  .tabs ul li {
+    margin-right: 3px;
+    display: inline-block;
+    color: #fff;
+    border-top-left-radius: 5px;
+    border-top-right-radius: 5px;
+    background-color: rgba(255, 255, 255, 0.2); }
+    .tabs ul li.active, .tabs ul li :target {
+      background-color: #fff; }
+      .tabs ul li.active a, .tabs ul li :target a {
+        color: #222;
+        font-weight: bold; }
+    .tabs ul li a {
+      display: block;
+      padding: 0.8em;
+      color: #fff; }
+
+.pure-form-stacked > div:first-child {
+  display: block; }
+
+.login-form .inner {
+  background: #fff;
+  padding: 20px;
+  border-radius: 5px; }
+
+.tab-pane-inner {
+  padding: 0px; }
+  .tab-pane-inner:not(:target) {
+    display: none; }
+  .tab-pane-inner:target {
+    display: block; }
+
+.edit-form {
+  min-width: 70%;
+  /* so it cant overflow */
+  max-width: 95%; }
+  .edit-form .box-wrap {
+    position: relative; }
+  .edit-form .inner {
+    background: #fff;
+    padding: 20px; }
+  .edit-form #actions {
+    display: block;
+    background: #fff; }
+  .edit-form .pure-form-message-inline {
+    padding-left: 0; }
+
+ul {
+  padding-left: 1em;
+  padding-top: 0px;
+  margin-top: 4px; }
+
+.time-check-widget tr {
+  display: inline; }
+  .time-check-widget tr input[type="number"] {
+    width: 4em; }

changedetectionio/static/styles/styles.scss

@@ -0,0 +1,641 @@
+/*
+ * -- BASE STYLES --
+ * Most of these are inherited from Base, but I want to change a few.
+ * nvm use v14.18.1
+ * npm install
+ * npm run build
+ * or npm run watch
+ */
+body {
+  color: #333;
+  background: #262626;
+}
+.pure-table-even {
+  background: #fff;
+}
+
+/* Some styles from https://css-tricks.com/ */
+a {
+  text-decoration: none;
+  color: #1b98f8;
+}
+
+a.github-link {
+  color: #fff;
+}
+
+.pure-menu-horizontal {
+  background: #fff;
+  padding: 5px;
+  display: flex;
+  justify-content: space-between;
+  border-bottom: 2px solid #ed5900;
+  align-items: center;
+}
+
+section.content {
+  padding-top: 5em;
+  padding-bottom: 1em;
+  flex-direction: column;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+code {
+  background: #eee;
+}
+
+/* table related */
+.watch-table {
+  width: 100%;
+  font-size: 80%;
+
+  tr.unviewed {
+    font-weight: bold;
+  }
+
+  .error {
+    color: #a00;
+  }
+
+  td {
+    white-space: nowrap;
+  }
+
+  td.title-col {
+    word-break: break-all;
+    white-space: normal;
+  }
+
+  th {
+    white-space: nowrap;
+  }
+
+  .title-col a[target="_blank"]::after, .current-diff-url::after {
+    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+    margin: 0 3px 0 5px;
+  }
+}
+
+.watch-tag-list {
+  color: #e70069;
+  white-space: nowrap;
+}
+
+.box {
+  max-width: 80%;
+  flex-direction: column;
+  display: flex;
+  justify-content: center;
+}
+
+
+#post-list-buttons {
+  text-align: right;
+  padding: 0px;
+  margin: 0px;
+
+  li {
+    display: inline-block;
+  }
+
+  a {
+    border-top-left-radius: initial;
+    border-top-right-radius: initial;
+    border-bottom-left-radius: 5px;
+    border-bottom-right-radius: 5px;
+  }
+}
+
+
+body:after {
+  content: "";
+  background: linear-gradient(130deg, #5ad8f7, #2f50af 41.07%, #9150bf 84.05%);
+}
+
+body:after, body:before {
+  display: block;
+  height: 650px;
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  z-index: -1;
+}
+
+body::after {
+  opacity: 0.91;
+}
+
+body::before {
+  // background-image set in base.html so it works with reverse proxies etc
+  content: "";
+  background-size: cover
+}
+
+body:after, body:before {
+  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
+  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
+}
+
+.arrow {
+  border: solid black;
+  border-width: 0 3px 3px 0;
+  display: inline-block;
+  padding: 3px;
+    &.right {
+      transform: rotate(-45deg);
+      -webkit-transform: rotate(-45deg);
+    }
+    &.left {
+      transform: rotate(135deg);
+      -webkit-transform: rotate(135deg);
+    }
+    &.up {
+      transform: rotate(-135deg);
+      -webkit-transform: rotate(-135deg);
+    }
+    &.down {
+      transform: rotate(45deg);
+      -webkit-transform: rotate(45deg);
+    }
+}
+
+.button-small {
+  font-size: 85%;
+}
+
+.fetch-error {
+  padding-top: 1em;
+  font-size: 60%;
+  max-width: 400px;
+  display: block;
+}
+
+
+.button-secondary {
+  color: white;
+  border-radius: 4px;
+  text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
+}
+
+.button-success {
+  background: rgb(28, 184, 65);
+  /* this is a green */
+}
+
+.button-tag {
+  background: rgb(99, 99, 99);
+  color: #fff;
+  font-size: 65%;
+  border-bottom-left-radius: initial;
+  border-bottom-right-radius: initial;
+
+  &.active {
+    background: #9c9c9c;
+    font-weight: bold;
+  }
+
+}
+
+.button-error {
+  background: rgb(202, 60, 60);
+  /* this is a maroon */
+}
+
+.button-warning {
+  background: rgb(223, 117, 20);
+  /* this is an orange */
+}
+
+.button-secondary {
+  background: rgb(66, 184, 221);
+  /* this is a light blue */
+}
+
+
+.button-cancel {
+  background: rgb(200, 200, 200);
+  /* this is a green */
+}
+
+.messages {
+    li {
+        list-style: none;
+        padding: 1em;
+        border-radius: 10px;
+        color: #fff;
+        font-weight: bold;
+        &.message {
+            background: rgba(255, 255, 255, .2);
+        }
+        &.error {
+            background: rgba(255, 1, 1, .5);
+        }
+        &.notice {
+            background: rgba(255, 255, 255, .5);
+        }
+    }
+    &.with-share-link {
+     > *:hover {
+       cursor:pointer;
+     }
+    }
+}
+
+#notification-customisation {
+    border: 1px solid #ccc;
+    padding: 0.5rem;
+    border-radius: 5px;
+}
+
+#notification-error-log {
+    border: 1px solid #ccc;
+    padding: 1rem;
+    border-radius: 5px;
+    overflow-wrap: break-word;
+}
+
+#token-table {
+    &.pure-table td, &.pure-table th {
+        font-size: 80%;
+    }
+}
+
+#new-watch-form {
+  background: rgba(0, 0, 0, .05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+  input {
+    width: auto !important;
+    display: inline-block;
+  }
+  .label {
+    display: none;
+  }
+  legend {
+    color: #fff;
+    font-weight: bold;
+  }
+}
+
+
+
+
+#diff-col {
+  padding-left: 40px;
+}
+
+#diff-jump {
+  position: fixed;
+  left: 0px;
+  top: 120px;
+  background: #fff;
+  padding: 10px;
+  border-top-right-radius: 5px;
+  border-bottom-right-radius: 5px;
+  box-shadow: 5px 0 5px -2px #888;
+     a {
+      color: #1b98f8;
+      cursor: grabbing;
+      -moz-user-select: none;
+      -webkit-user-select: none;
+      -ms-user-select: none;
+      user-select: none;
+      -o-user-select: none;
+    }
+}
+
+footer {
+  padding: 10px;
+  background: #fff;
+  color: #444;
+  text-align: center;
+}
+
+#feed-icon {
+  vertical-align: middle;
+}
+
+#top-right-menu {
+// Just let flex overflow the x axis for now
+/*
+    position: absolute;
+    right: 0px;
+    background: linear-gradient(to right, #fff0, #fff 10%);
+    padding-left: 20px;
+    padding-right: 10px;
+    */
+}
+
+.sticky-tab {
+  position: absolute;
+  top: 60px;
+  font-size: 65%;
+  background: #fff;
+  padding: 10px;
+  &#left-sticky {
+    left: 0px;
+  }
+  &#right-sticky {
+    right: 0px;
+  }
+  &#hosted-sticky {
+    right: 0px;
+    top: 100px;
+    font-weight: bold;
+  }
+}
+
+#new-version-text a {
+  color: #e07171;
+}
+
+.paused-state {
+  &.state-False img {
+    opacity: 0.2;
+  }
+
+  &.state-False:hover img {
+    opacity: 0.8;
+  }
+}
+
+.monospaced-textarea {
+    textarea {
+        width: 100%;
+        font-family: monospace;
+        white-space: pre;
+        overflow-wrap: normal;
+        overflow-x: scroll;
+    }
+}
+
+
+.pure-form {
+    .pure-control-group, .pure-group, .pure-controls {
+        padding-bottom: 1em;
+        div {
+            margin: 0px;
+        }
+        .checkbox {
+            > * {
+              display: inline;
+              vertical-align: middle;
+            }
+            > label {
+               padding-left: 5px;
+            }
+        }
+    }
+  /* The input fields with errors */
+  .error {
+    input {
+        background-color: #ffebeb;
+    }
+  }
+
+  /* The list of errors */
+  ul.errors {
+    padding: .5em .6em;
+    border: 1px solid #dd0000;
+    border-radius: 4px;
+    vertical-align: middle;
+    -webkit-box-sizing: border-box;
+    box-sizing: border-box;
+    li {
+        margin-left: 1em;
+        color: #dd0000;
+    }
+  }
+
+  label {
+    font-weight: bold;
+  }
+
+  textarea {
+    width: 100%;
+  }
+  ul.fetch-backend {
+    margin: 0px;
+    list-style: none;
+    li {
+        > * {
+            display: inline-block;
+        }
+    }
+  }
+}
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+  .box {
+    max-width: 95%
+  }
+  .edit-form {
+    padding: 0.5em;
+    margin: 0;
+  }
+  #nav-menu {
+    overflow-x: scroll;
+  }
+}
+
+
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 800px) {
+
+  div.sticky-tab#hosted-sticky {
+    top: 60px;
+    left: 0px;
+    right: auto;
+  }
+
+  section.content {
+    padding-top: 110px;
+  }
+
+  // Make the tabs easier to hit, they will be all nice and horizontal
+  div.tabs.collapsable ul li {
+    display: block;
+    border-radius: 0px;
+  }
+
+  input[type='text'] {
+    width: 100%;
+  }
+
+/*
+Max width before this PARTICULAR table gets nasty
+This query will take effect for any screen smaller than 760px
+and also iPads specifically.
+*/
+  .watch-table {
+    /* Force table to not be like tables anymore */
+    thead, tbody, th, td, tr {
+      display: block;
+    }
+
+    .last-checked::before {
+      color: #555;
+      content: "Last Checked ";
+    }
+
+    .last-changed::before {
+      color: #555;
+      content: "Last Changed ";
+    }
+
+    /* Force table to not be like tables anymore */
+    td.inline {
+      display: inline-block;
+    }
+
+    /* Hide table headers (but not display: none;, for accessibility) */
+    thead tr {
+      position: absolute;
+      top: -9999px;
+      left: -9999px;
+    }
+
+    .pure-table td, .pure-table th {
+      border: none;
+    }
+
+    td {
+      /* Behave  like a "row" */
+      border: none;
+      border-bottom: 1px solid #eee;
+
+      &:before {
+        /* Top/left values mimic padding */
+        top: 6px;
+        left: 6px;
+        width: 45%;
+        padding-right: 10px;
+        white-space: nowrap;
+      }
+    }
+
+    &.pure-table-striped {
+      tr {
+        background-color: #fff;
+      }
+
+      tr:nth-child(2n-1) {
+        background-color: #eee;
+      }
+
+      tr:nth-child(2n-1) td {
+        background-color: inherit;
+      }
+    }
+
+  }
+}
+
+
+/** Desktop vs mobile input field strategy
+- We dont use 'size' with <input> because `size` is too unreliable to override, and will often push-out
+- Rely always on width in CSS
+*/
+@media only screen and (min-width: 761px) {
+/* m-d is medium-desktop */
+    .m-d {
+        min-width: 80%;
+    }
+}
+
+
+
+.tabs {
+  ul {
+    margin: 0px;
+    padding: 0px;
+    display:block;
+    li {
+      margin-right: 3px;
+      display: inline-block;
+      color: #fff;
+      border-top-left-radius: 5px;
+      border-top-right-radius: 5px;
+      background-color: rgba(255, 255, 255, 0.2);
+
+      &.active,:target {
+        background-color: #fff;
+        a {
+          color: #222;
+          font-weight: bold;
+        }
+      }
+      a {
+        display: block;
+        padding: 0.8em;
+        color: #fff;
+      }
+    }
+  }
+}
+
+$form-edge-padding: 20px;
+.pure-form-stacked {
+  >div:first-child {
+    display: block;
+  }
+}
+
+.login-form {
+  .inner {
+    background: #fff;;
+    padding: $form-edge-padding;
+    border-radius: 5px;
+  }
+}
+
+.tab-pane-inner {
+    &:not(:target) {
+        display: none;
+    }
+    &:target {
+      display: block;
+    }
+    // doesnt need padding because theres another row of buttons/activity
+    padding: 0px;
+}
+
+.edit-form {
+  min-width: 70%;
+  /* so it cant overflow */
+  max-width: 95%;
+  .box-wrap {
+    position: relative;
+  }
+  .inner {
+    background: #fff;;
+    padding: $form-edge-padding;
+  }
+  #actions {
+    display: block;
+    background: #fff;
+  }
+
+  .pure-form-message-inline {
+    padding-left: 0;
+  }
+}
+
+ul {
+    padding-left: 1em;
+    padding-top: 0px;
+    margin-top: 4px;
+}
+
+.time-check-widget {
+    tr {
+        display: inline;
+        input[type="number"] {
+            width: 4em;
+        }
+    }
+}

changedetectionio/store.py

@@ -0,0 +1,470 @@
+from flask import (
+    flash
+)
+import json
+import logging
+import os
+import threading
+import time
+import uuid as uuid_builder
+from copy import deepcopy
+from os import mkdir, path, unlink
+from threading import Lock
+import re
+import requests
+
+from changedetectionio.model import Watch, App
+
+
+# Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
+# Open a github issue if you know something :)
+# https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
+class ChangeDetectionStore:
+    lock = Lock()
+    # For general updates/writes that can wait a few seconds
+    needs_write = False
+
+    # For when we edit, we should write to disk
+    needs_write_urgent = False
+
+    def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
+        # Should only be active for docker
+        # logging.basicConfig(filename='/dev/stdout', level=logging.INFO)
+        self.needs_write = False
+        self.datastore_path = datastore_path
+        self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
+        self.stop_thread = False
+
+        self.__data = App.model()
+
+        # Base definition for all watchers
+        # deepcopy part of #569 - not sure why its needed exactly
+        self.generic_definition = deepcopy(Watch.model())
+
+        if path.isfile('changedetectionio/source.txt'):
+            with open('changedetectionio/source.txt') as f:
+                # Should be set in Dockerfile to look for /source.txt , this will give us the git commit #
+                # So when someone gives us a backup file to examine, we know exactly what code they were running.
+                self.__data['build_sha'] = f.read()
+
+        try:
+            # @todo retest with ", encoding='utf-8'"
+            with open(self.json_store_path) as json_file:
+                from_disk = json.load(json_file)
+
+                # @todo isnt there a way todo this dict.update recursively?
+                # Problem here is if the one on the disk is missing a sub-struct, it wont be present anymore.
+                if 'watching' in from_disk:
+                    self.__data['watching'].update(from_disk['watching'])
+
+                if 'app_guid' in from_disk:
+                    self.__data['app_guid'] = from_disk['app_guid']
+
+                if 'settings' in from_disk:
+                    if 'headers' in from_disk['settings']:
+                        self.__data['settings']['headers'].update(from_disk['settings']['headers'])
+
+                    if 'requests' in from_disk['settings']:
+                        self.__data['settings']['requests'].update(from_disk['settings']['requests'])
+
+                    if 'application' in from_disk['settings']:
+                        self.__data['settings']['application'].update(from_disk['settings']['application'])
+
+                # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
+                # @todo pretty sure theres a python we todo this with an abstracted(?) object!
+                for uuid, watch in self.__data['watching'].items():
+                    _blank = deepcopy(self.generic_definition)
+                    _blank.update(watch)
+                    self.__data['watching'].update({uuid: _blank})
+                    self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
+                    print("Watching:", uuid, self.__data['watching'][uuid]['url'])
+
+        # First time ran, doesnt exist.
+        except (FileNotFoundError, json.decoder.JSONDecodeError):
+            if include_default_watches:
+                print("Creating JSON store at", self.datastore_path)
+
+                self.add_watch(url='http://www.quotationspage.com/random.php', tag='test')
+                self.add_watch(url='https://news.ycombinator.com/', tag='Tech news')
+                self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
+                self.add_watch(url='https://changedetection.io/CHANGELOG.txt')
+
+        self.__data['version_tag'] = version_tag
+
+        # Helper to remove password protection
+        password_reset_lockfile = "{}/removepassword.lock".format(self.datastore_path)
+        if path.isfile(password_reset_lockfile):
+            self.__data['settings']['application']['password'] = False
+            unlink(password_reset_lockfile)
+
+        if not 'app_guid' in self.__data:
+            import os
+            import sys
+            if "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ:
+                self.__data['app_guid'] = "test-" + str(uuid_builder.uuid4())
+            else:
+                self.__data['app_guid'] = str(uuid_builder.uuid4())
+
+        # Generate the URL access token for RSS feeds
+        if not 'rss_access_token' in self.__data['settings']['application']:
+            import secrets
+            secret = secrets.token_hex(16)
+            self.__data['settings']['application']['rss_access_token'] = secret
+
+        # Bump the update version by running updates
+        self.run_updates()
+
+        self.needs_write = True
+
+        # Finally start the thread that will manage periodic data saves to JSON
+        save_data_thread = threading.Thread(target=self.save_datastore).start()
+
+    # Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0.
+    def get_newest_history_key(self, uuid):
+        if len(self.__data['watching'][uuid]['history']) == 1:
+            return 0
+
+        dates = list(self.__data['watching'][uuid]['history'].keys())
+        # Convert to int, sort and back to str again
+        # @todo replace datastore getter that does this automatically
+        dates = [int(i) for i in dates]
+        dates.sort(reverse=True)
+        if len(dates):
+            # always keyed as str
+            return str(dates[0])
+
+        return 0
+
+    def set_last_viewed(self, uuid, timestamp):
+        self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
+        self.needs_write = True
+
+    def remove_password(self):
+        self.__data['settings']['application']['password'] = False
+        self.needs_write = True
+
+    def update_watch(self, uuid, update_obj):
+
+        with self.lock:
+
+            # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
+            for dict_key, d in self.generic_definition.items():
+                if isinstance(d, dict):
+                    if update_obj is not None and dict_key in update_obj:
+                        self.__data['watching'][uuid][dict_key].update(update_obj[dict_key])
+                        del (update_obj[dict_key])
+
+            self.__data['watching'][uuid].update(update_obj)
+            self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
+
+        self.needs_write = True
+
+    @property
+    def threshold_seconds(self):
+        seconds = 0
+        mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
+        minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60))
+        for m, n in mtable.items():
+            x = self.__data['settings']['requests']['time_between_check'].get(m)
+            if x:
+                seconds += x * n
+        return max(seconds, minimum_seconds_recheck_time)
+
+    @property
+    def data(self):
+        has_unviewed = False
+        for uuid, v in self.__data['watching'].items():
+            self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
+            if int(v['newest_history_key']) <= int(v['last_viewed']):
+                self.__data['watching'][uuid]['viewed'] = True
+
+            else:
+                self.__data['watching'][uuid]['viewed'] = False
+                has_unviewed = True
+
+            # #106 - Be sure this is None on empty string, False, None, etc
+            # Default var for fetch_backend
+            if not self.__data['watching'][uuid]['fetch_backend']:
+                self.__data['watching'][uuid]['fetch_backend'] = self.__data['settings']['application']['fetch_backend']
+
+        # Re #152, Return env base_url if not overriden, @todo also prefer the proxy pass url
+        env_base_url = os.getenv('BASE_URL','')
+        if not self.__data['settings']['application']['base_url']:
+          self.__data['settings']['application']['base_url'] = env_base_url.strip('" ')
+
+        self.__data['has_unviewed'] = has_unviewed
+
+        return self.__data
+
+    def get_all_tags(self):
+        tags = []
+        for uuid, watch in self.data['watching'].items():
+
+            # Support for comma separated list of tags.
+            for tag in watch['tag'].split(','):
+                tag = tag.strip()
+                if tag not in tags:
+                    tags.append(tag)
+
+        tags.sort()
+        return tags
+
+    def unlink_history_file(self, path):
+        try:
+            unlink(path)
+        except (FileNotFoundError, IOError):
+            pass
+
+    # Delete a single watch by UUID
+    def delete(self, uuid):
+        with self.lock:
+            if uuid == 'all':
+                self.__data['watching'] = {}
+
+                # GitHub #30 also delete history records
+                for uuid in self.data['watching']:
+                    for path in self.data['watching'][uuid]['history'].values():
+                        self.unlink_history_file(path)
+
+            else:
+                for path in self.data['watching'][uuid]['history'].values():
+                    self.unlink_history_file(path)
+
+                del self.data['watching'][uuid]
+
+            self.needs_write_urgent = True
+
+    # Clone a watch by UUID
+    def clone(self, uuid):
+        url = self.data['watching'][uuid]['url']
+        tag = self.data['watching'][uuid]['tag']
+        extras = self.data['watching'][uuid]
+        new_uuid = self.add_watch(url=url, tag=tag, extras=extras)
+        return new_uuid
+
+    def url_exists(self, url):
+
+        # Probably their should be dict...
+        for watch in self.data['watching'].values():
+            if watch['url'] == url:
+                return True
+
+        return False
+
+    def get_val(self, uuid, val):
+        # Probably their should be dict...
+        return self.data['watching'][uuid].get(val)
+
+    # Remove a watchs data but keep the entry (URL etc)
+    def scrub_watch(self, uuid):
+        import pathlib
+
+        self.__data['watching'][uuid].update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'newest_history_key': 0, 'previous_md5': False})
+        self.needs_write_urgent = True
+
+        for item in pathlib.Path(self.datastore_path).rglob(uuid+"/*.txt"):
+            unlink(item)
+
+    def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
+        if extras is None:
+            extras = {}
+        # Incase these are copied across, assume it's a reference and deepcopy()
+        apply_extras = deepcopy(extras)
+
+        # Was it a share link? try to fetch the data
+        if (url.startswith("https://changedetection.io/share/")):
+            try:
+                r = requests.request(method="GET",
+                                     url=url,
+                                     # So we know to return the JSON instead of the human-friendly "help" page
+                                     headers={'App-Guid': self.__data['app_guid']})
+                res = r.json()
+
+                # List of permisable stuff we accept from the wild internet
+                for k in ['url', 'tag',
+                                   'paused', 'title',
+                                   'previous_md5', 'headers',
+                                   'body', 'method',
+                                   'ignore_text', 'css_filter',
+                                   'subtractive_selectors', 'trigger_text',
+                                   'extract_title_as_title']:
+                    if res.get(k):
+                        apply_extras[k] = res[k]
+
+            except Exception as e:
+                logging.error("Error fetching metadata for shared watch link", url, str(e))
+                flash("Error fetching metadata for {}".format(url), 'error')
+                return False
+
+        with self.lock:
+            # @todo use a common generic version of this
+            new_uuid = str(uuid_builder.uuid4())
+            # #Re 569
+            # Not sure why deepcopy was needed here, sometimes new watches would appear to already have 'history' set
+            # I assumed this would instantiate a new object but somehow an existing dict was getting used
+            new_watch = deepcopy(Watch.model({
+                'url': url,
+                'tag': tag
+            }))
+
+
+            for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
+                if k in apply_extras:
+                    del apply_extras[k]
+
+            new_watch.update(apply_extras)
+            self.__data['watching'][new_uuid]=new_watch
+
+        # Get the directory ready
+        output_path = "{}/{}".format(self.datastore_path, new_uuid)
+        try:
+            mkdir(output_path)
+        except FileExistsError:
+            print(output_path, "already exists.")
+
+        if write_to_disk_now:
+            self.sync_to_json()
+        return new_uuid
+
+    # Save some text file to the appropriate path and bump the history
+    # result_obj from fetch_site_status.run()
+    def save_history_text(self, watch_uuid, contents):
+        import uuid
+
+        output_path = "{}/{}".format(self.datastore_path, watch_uuid)
+        # Incase the operator deleted it, check and create.
+        if not os.path.isdir(output_path):
+            mkdir(output_path)
+
+        fname = "{}/{}.stripped.txt".format(output_path, uuid.uuid4())
+        with open(fname, 'wb') as f:
+            f.write(contents)
+            f.close()
+
+        return fname
+
+    def get_screenshot(self, watch_uuid):
+        output_path = "{}/{}".format(self.datastore_path, watch_uuid)
+        fname = "{}/last-screenshot.png".format(output_path)
+        if path.isfile(fname):
+            return fname
+
+        return False
+
+    # Save as PNG, PNG is larger but better for doing visual diff in the future
+    def save_screenshot(self, watch_uuid, screenshot: bytes):
+        output_path = "{}/{}".format(self.datastore_path, watch_uuid)
+        fname = "{}/last-screenshot.png".format(output_path)
+        with open(fname, 'wb') as f:
+            f.write(screenshot)
+            f.close()
+
+    def sync_to_json(self):
+        logging.info("Saving JSON..")
+        print("Saving JSON..")
+        try:
+            data = deepcopy(self.__data)
+        except RuntimeError as e:
+            # Try again in 15 seconds
+            time.sleep(15)
+            logging.error ("! Data changed when writing to JSON, trying again.. %s", str(e))
+            self.sync_to_json()
+            return
+        else:
+
+            try:
+                # Re #286  - First write to a temp file, then confirm it looks OK and rename it
+                # This is a fairly basic strategy to deal with the case that the file is corrupted,
+                # system was out of memory, out of RAM etc
+                with open(self.json_store_path+".tmp", 'w') as json_file:
+                    json.dump(data, json_file, indent=4)
+                os.replace(self.json_store_path+".tmp", self.json_store_path)
+            except Exception as e:
+                logging.error("Error writing JSON!! (Main JSON file save was skipped) : %s", str(e))
+
+            self.needs_write = False
+            self.needs_write_urgent = False
+
+    # Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON
+    # by just running periodically in one thread, according to python, dict updates are threadsafe.
+    def save_datastore(self):
+
+        while True:
+            if self.stop_thread:
+                print("Shutting down datastore thread")
+                return
+
+            if self.needs_write or self.needs_write_urgent:
+                self.sync_to_json()
+
+            # Once per minute is enough, more and it can cause high CPU usage
+            # better here is to use something like self.app.config.exit.wait(1), but we cant get to 'app' from here
+            for i in range(120):
+                time.sleep(0.5)
+                if self.stop_thread or self.needs_write_urgent:
+                    break
+
+    # Go through the datastore path and remove any snapshots that are not mentioned in the index
+    # This usually is not used, but can be handy.
+    def remove_unused_snapshots(self):
+        print ("Removing snapshots from datastore that are not in the index..")
+
+        index=[]
+        for uuid in self.data['watching']:
+            for id in self.data['watching'][uuid]['history']:
+                index.append(self.data['watching'][uuid]['history'][str(id)])
+
+        import pathlib
+
+        # Only in the sub-directories
+        for uuid in self.data['watching']:
+            for item in pathlib.Path(self.datastore_path).rglob(uuid+"/*.txt"):
+                if not str(item) in index:
+                    print ("Removing",item)
+                    unlink(item)
+
+    # Run all updates
+    # IMPORTANT - Each update could be run even when they have a new install and the schema is correct
+    #             So therefor - each `update_n` should be very careful about checking if it needs to actually run
+    #             Probably we should bump the current update schema version with each tag release version?
+    def run_updates(self):
+        import inspect
+        import shutil
+
+        updates_available = []
+        for i, o in inspect.getmembers(self, predicate=inspect.ismethod):
+            m = re.search(r'update_(\d+)$', i)
+            if m:
+                updates_available.append(int(m.group(1)))
+        updates_available.sort()
+
+        for update_n in updates_available:
+            if update_n > self.__data['settings']['application']['schema_version']:
+                print ("Applying update_{}".format((update_n)))
+                # Wont exist on fresh installs
+                if os.path.exists(self.json_store_path):
+                    shutil.copyfile(self.json_store_path, self.datastore_path+"/url-watches-before-{}.json".format(update_n))
+
+                try:
+                    update_method = getattr(self, "update_{}".format(update_n))()
+                except Exception as e:
+                    print("Error while trying update_{}".format((update_n)))
+                    print(e)
+                    # Don't run any more updates
+                    return
+                else:
+                    # Bump the version, important
+                    self.__data['settings']['application']['schema_version'] = update_n
+
+    # Convert minutes to seconds on settings and each watch
+    def update_1(self):
+        if self.data['settings']['requests'].get('minutes_between_check'):
+            self.data['settings']['requests']['time_between_check']['minutes'] = self.data['settings']['requests']['minutes_between_check']
+            # Remove the default 'hours' that is set from the model
+            self.data['settings']['requests']['time_between_check']['hours'] = None
+
+        for uuid, watch in self.data['watching'].items():
+            if 'minutes_between_check' in watch:
+                # Only upgrade individual watch time if it was set
+                if watch.get('minutes_between_check', False):
+                    self.data['watching'][uuid]['time_between_check']['minutes'] = watch['minutes_between_check']

changedetectionio/templates/_common_fields.jinja

@@ -0,0 +1,101 @@
+
+{% from '_helpers.jinja' import render_field %}
+
+{% macro render_common_settings_form(form, current_base_url, emailprefix) %}
+
+                        <div class="pure-control-group">
+                            {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
+    Gitter - gitter://token/room
+    Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+    AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
+    SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com", class="notification-urls")
+                            }}
+                            <div class="pure-form-message-inline">
+                              <ul>
+                                <li>Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.</li>
+                                <li><code>discord://</code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
+                                <li>Go here for <a href="{{url_for('notification_logs')}}">notification debug logs</a></li>
+                              </ul>
+                            </div>
+                            <br/>
+                            <a id="send-test-notification" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Send test notification</a>
+{% if emailprefix %}
+                            <a id="add-email-helper" class="pure-button button-secondary button-xsmall" style="font-size: 70%">Add email</a>
+{% endif %}
+                        </div>
+                        <div id="notification-customisation" class="pure-control-group">
+                            <div class="pure-control-group">
+                                {{ render_field(form.notification_title, class="m-d notification-title") }}
+                                <span class="pure-form-message-inline">Title for all notifications</span>
+                            </div>
+                            <div class="pure-control-group">
+                                {{ render_field(form.notification_body , rows=5, class="notification-body") }}
+                                <span class="pure-form-message-inline">Body for all notifications</span>
+                            </div>
+                            <div class="pure-control-group">
+                                {{ render_field(form.notification_format , rows=5, class="notification-format") }}
+                                <span class="pure-form-message-inline">Format for all notifications</span>
+                            </div>
+                            <div class="pure-controls">
+                            <span class="pure-form-message-inline">
+                                These tokens can be used in the notification body and title to customise the notification text.
+
+                                <table class="pure-table" id="token-table">
+                                    <thead>
+                                    <tr>
+                                        <th>Token</th>
+                                        <th>Description</th>
+                                    </tr>
+                                    </thead>
+                                    <tbody>
+                                    <tr>
+                                        <td><code>{base_url}</code></td>
+                                        <td>The URL of the changedetection.io instance you are running.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{watch_url}</code></td>
+                                        <td>The URL being watched.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{watch_uuid}</code></td>
+                                        <td>The UUID of the watch.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{watch_title}</code></td>
+                                        <td>The title of the watch.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{watch_tag}</code></td>
+                                        <td>The tag of the watch.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{preview_url}</code></td>
+                                        <td>The URL of the preview page generated by changedetection.io.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{diff}</code></td>
+                                        <td>The diff output - differences only</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{diff_full}</code></td>
+                                        <td>The diff output - full difference output</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{diff_url}</code></td>
+                                        <td>The URL of the diff page generated by changedetection.io.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{current_snapshot}</code></td>
+                                        <td>The current snapshot value, useful when combined with JSON or CSS filters
+                                        </td>
+                                    </tr>
+                                    </tbody>
+                                </table>
+                                <br/>
+                                URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
+                                Your <code>BASE_URL</code> var is currently "{{current_base_url}}"
+                            </span>
+                            </div>
+                        </div>
+{% endmacro %}

changedetectionio/templates/_helpers.jinja

@@ -0,0 +1,57 @@
+{% macro render_field(field) %}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
+
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </div>
+{% endmacro %}
+
+{% macro render_checkbox_field(field) %}
+  <div class="checkbox {% if field.errors %} error {% endif %}">
+  {{ field(**kwargs)|safe }} {{ field.label }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </div>
+{% endmacro %}
+
+{% macro render_field(field) %}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
+  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </div>
+{% endmacro %}
+
+{% macro render_simple_field(field) %}
+  <span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>
+  <span {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </span>
+{% endmacro %}
+
+
+{% macro render_button(field) %}
+  {{ field(**kwargs)|safe }}
+{% endmacro %}

changedetectionio/templates/base.html

@@ -0,0 +1,110 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="description" content="Self hosted website change detection.">
+    <title>Change Detection{{extra_title}}</title>
+    <link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag %}- {{active_tag}}{% endif %}" href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}" />
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}">
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}">
+    {% if extra_stylesheets %}
+        {% for m in extra_stylesheets %}
+        <link rel="stylesheet" href="{{ m }}?ver=1000">
+        {% endfor %}
+    {% endif %}
+    <style>
+    body::before {
+        background-image: url({{url_for('static_content', group='images', filename='gradient-border.png')}});
+    }
+    </style>
+    <script type="text/javascript" src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
+
+</head>
+
+<body>
+
+<div class="header">
+
+    <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
+        {% if has_password and not current_user.is_authenticated %}
+            <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener"><strong>Change</strong>Detection.io</a>
+        {% else %}
+            <a class="pure-menu-heading" href="{{url_for('index')}}"><strong>Change</strong>Detection.io</a>
+        {% endif %}
+        {% if current_diff_url %}
+        <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</span></a>
+        {% else %}
+        {% if new_version_available and not (has_password and not current_user.is_authenticated) %}
+        <span id="new-version-text" class="pure-menu-heading"><a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a></span>
+        {% endif %}
+        {% endif %}
+
+        <ul class="pure-menu-list"  id="top-right-menu">
+        {% if current_user.is_authenticated or not has_password %}
+            {% if not current_diff_url %}
+            <li class="pure-menu-item">
+                <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
+            </li>
+            <li class="pure-menu-item">
+                <a href="{{ url_for('import_page')}}" class="pure-menu-link">IMPORT</a>
+            </li>
+            <li class="pure-menu-item">
+                <a href="{{ url_for('get_backup')}}" class="pure-menu-link">BACKUP</a>
+            </li>
+            {% else %}
+            <li class="pure-menu-item">
+                <a href="{{ url_for('edit_page', uuid=uuid, next='diff') }}" class="pure-menu-link">EDIT</a>
+            </li>
+            {% endif %}
+        {% else %}
+            <li class="pure-menu-item">
+                <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
+            </li>
+        {% endif %}
+
+        {% if current_user.is_authenticated %}
+            <li class="pure-menu-item"><a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a></li>
+        {% endif %}
+            <li class="pure-menu-item"><a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
+                <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16"
+                     version="1.1"
+                     width="32" aria-hidden="true">
+                    <path fill-rule="evenodd"
+                          d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
+                </svg>
+            </a></li>
+        </ul>
+    </div>
+</div>
+{% if hosted_sticky %}<div class="sticky-tab" id="hosted-sticky"><a href="https://lemonade.changedetection.io/start?ref={{guid}}">Let us host your instance!</a></div>{% endif %}
+{% if left_sticky %}<div class="sticky-tab" id="left-sticky"><a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a></div> {% endif %}
+{% if right_sticky %}<div class="sticky-tab" id="right-sticky">{{ right_sticky }}</div> {% endif %}
+<section class="content">
+    <header>
+        {% block header %}{% endblock %}
+    </header>
+
+    {% with messages = get_flashed_messages(with_categories=true) %}
+      {% if messages %}
+        <ul class=messages>
+        {% for category, message in messages %}
+          <li class="{{ category }}">{{ message }}</li>
+        {% endfor %}
+        </ul>
+      {% endif %}
+    {% endwith %}
+
+    {% if session['share-link'] %}
+        <ul class="messages with-share-link">
+          <li class="message">Share this link: <span id="share-link">{{ session['share-link'] }}</span> <img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='copy.svg')}}" /></li>
+        </ul>
+    {% endif %}
+
+    {% block content %}
+
+    {% endblock %}
+</section>
+
+</body>
+</html>

changedetectionio/templates/diff.html

@@ -0,0 +1,198 @@
+{% extends 'base.html' %}
+
+{% block content %}
+<div id="settings">
+    <h1>Differences</h1>
+    <form class="pure-form " action="" method="GET">
+        <fieldset>
+
+            <label for="diffWords" class="pure-checkbox">
+                <input type="radio" name="diff_type" id="diffWords" value="diffWords"/> Words</label>
+            <label for="diffLines" class="pure-checkbox">
+                <input type="radio" name="diff_type" id="diffLines" value="diffLines" checked=""/> Lines</label>
+
+            <label for="diffChars" class="pure-checkbox">
+                <input type="radio" name="diff_type" id="diffChars" value="diffChars"/> Chars</label>
+
+            {% if versions|length >= 1 %}
+            <label for="diff-version">Compare newest (<span id="current-v-date"></span>) with</label>
+            <select id="diff-version" name="previous_version">
+                {% for version in versions %}
+                <option value="{{version}}" {% if version== current_previous_version %} selected="" {% endif %}>
+                    {{version}}
+                </option>
+                {% endfor %}
+            </select>
+            <button type="submit" class="pure-button pure-button-primary">Go</button>
+            {% endif %}
+        </fieldset>
+    </form>
+    <del>Removed text</del>
+    <ins>Inserted Text</ins>
+</div>
+
+<div id="diff-jump">
+    <a onclick="next_diff();">Jump</a>
+</div>
+
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<div class="tabs">
+    <ul>
+        <li class="tab" id="default-tab"><a href="#text">Text</a></li>
+{% if screenshot %}
+        <li class="tab"><a href="#screenshot">Current screenshot</a></li>
+{% endif %}
+    </ul>
+</div>
+
+<div id="diff-ui">
+     <div class="tab-pane-inner" id="text">
+         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.
+         </div>
+         <table>
+             <tbody>
+             <tr>
+                 <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
+                 <td id="a" style="display: none;">{{previous}}</td>
+                 <td id="b" style="display: none;">{{newest}}</td>
+                 <td id="diff-col">
+                     <span id="result"></span>
+                 </td>
+             </tr>
+             </tbody>
+         </table>
+         Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff">github.com/kpdecker/jsdiff</a>
+     </div>
+
+{% if screenshot %}
+     <div class="tab-pane-inner" id="screenshot">
+         <p>
+         <i>For now, only the most recent screenshot is saved and displayed.</i>
+             </p>
+
+        <img src="{{url_for('static_content', group='screenshot', filename=uuid)}}">
+     </div>
+{% endif %}
+
+</div>
+
+
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff.js')}}"></script>
+
+<script defer="">
+
+
+var a = document.getElementById('a');
+var b = document.getElementById('b');
+var result = document.getElementById('result');
+
+function changed() {
+	var diff = JsDiff[window.diffType](a.textContent, b.textContent);
+	var fragment = document.createDocumentFragment();
+	for (var i=0; i < diff.length; i++) {
+
+		if (diff[i].added && diff[i + 1] && diff[i + 1].removed) {
+			var swap = diff[i];
+			diff[i] = diff[i + 1];
+			diff[i + 1] = swap;
+		}
+
+		var node;
+		if (diff[i].removed) {
+			node = document.createElement('del');
+			node.classList.add("change");
+			node.appendChild(document.createTextNode(diff[i].value));
+
+		} else if (diff[i].added) {
+			node = document.createElement('ins');
+			node.classList.add("change");
+			node.appendChild(document.createTextNode(diff[i].value));
+		} else {
+			node = document.createTextNode(diff[i].value);
+		}
+		fragment.appendChild(node);
+	}
+
+	result.textContent = '';
+	result.appendChild(fragment);
+
+	// Jump at start
+	inputs.current=0;
+    next_diff();
+}
+
+window.onload = function() {
+
+
+    /* Convert what is options from UTC time.time() to local browser time */
+    var diffList=document.getElementById("diff-version");
+    if (typeof(diffList) != 'undefined' && diffList != null) {
+        for (var option of diffList.options) {
+          var dateObject = new Date(option.value*1000);
+          option.label=dateObject.toLocaleString();
+        }
+    }
+
+    /* Set current version date as local time in the browser also */
+    var current_v = document.getElementById("current-v-date");
+    var dateObject = new Date({{ newest_version_timestamp }}*1000);
+    current_v.innerHTML=dateObject.toLocaleString();
+
+
+	onDiffTypeChange(document.querySelector('#settings [name="diff_type"]:checked'));
+	changed();
+
+};
+
+a.onpaste = a.onchange =
+b.onpaste = b.onchange = changed;
+
+if ('oninput' in a) {
+	a.oninput = b.oninput = changed;
+} else {
+	a.onkeyup = b.onkeyup = changed;
+}
+
+function onDiffTypeChange(radio) {
+	window.diffType = radio.value;
+// Not necessary 
+//	document.title = "Diff " + radio.value.slice(4);
+}
+
+var radio = document.getElementsByName('diff_type');
+for (var i = 0; i < radio.length; i++) {
+	radio[i].onchange = function(e) {
+		onDiffTypeChange(e.target);
+		changed();
+	}
+}
+
+
+var inputs = document.getElementsByClassName('change');
+inputs.current=0;
+
+
+function next_diff() {
+
+    var element = inputs[inputs.current];
+    var headerOffset = 80;
+    var elementPosition = element.getBoundingClientRect().top;
+    var offsetPosition = elementPosition - headerOffset +  window.scrollY;
+
+    window.scrollTo({
+         top: offsetPosition,
+         behavior: "smooth"
+    });
+
+    inputs.current++;
+    if(inputs.current >= inputs.length) {
+      inputs.current=0;
+    }
+}
+
+
+
+</script>
+
+
+{% endblock %}

changedetectionio/templates/edit.html

@@ -0,0 +1,194 @@
+{% extends 'base.html' %}
+{% block content %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script>
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+{% if emailprefix %}
+    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
+{% endif %}
+</script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
+
+<div class="edit-form monospaced-textarea">
+
+    <div class="tabs collapsable">
+        <ul>
+            <li class="tab" id="default-tab"><a href="#general">General</a></li>
+            <li class="tab"><a href="#request">Request</a></li>
+            <li class="tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
+            <li class="tab"><a href="#notifications">Notifications</a></li>
+        </ul>
+    </div>
+
+    <div class="box-wrap inner">
+        <form class="pure-form pure-form-stacked"
+              action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next') ) }}" method="POST">
+             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+
+            <div class="tab-pane-inner" id="general">
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
+                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span>
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.title, class="m-d") }}
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.tag) }}
+                        <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.time_between_check, class="time-check-widget") }}
+                        {% if has_empty_checktime %}
+                        <span class="pure-form-message-inline">Currently using the <a
+                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
+                        {% else %}
+                        <span class="pure-form-message-inline">Set to blank to use the <a
+                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
+                        {% endif %}
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.extract_title_as_title) }}
+                    </div>
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="request">
+                    <div class="pure-control-group">
+                        {{ render_field(form.fetch_backend, class="fetch-backend") }}
+                        <span class="pure-form-message-inline">
+                            <p>Use the <strong>Basic</strong> method (default) where your watched site doesn't need Javascript to render.</p>
+                            <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
+                        </span>
+                    </div>
+
+                <hr/>
+                <fieldset class="pure-group">
+
+                    <span class="pure-form-message-inline">
+                        <strong>Request override is currently only used by the <i>Basic fast Plaintext/HTTP Client</i> method.</strong>
+                    </span>
+                    <div class="pure-control-group">
+                        {{ render_field(form.method) }}
+                    </div>
+                    <div class="pure-control-group">
+{{ render_field(form.headers, rows=5, placeholder="Example
+Cookie: foobar
+User-Agent: wonderbra 1.0") }}
+                    </div>
+                    <div class="pure-control-group">
+                                        {{ render_field(form.body, rows=5, placeholder="Example
+{
+   \"name\":\"John\",
+   \"age\":30,
+   \"car\":null
+}") }}
+                    </div>
+                    <div>
+                        {{ render_checkbox_field(form.ignore_status_codes) }}
+                    </div>
+                </fieldset>
+                <br/>
+            </div>
+
+            <div class="tab-pane-inner" id="notifications">
+                <strong>Note: <i>These settings override the global settings for this watch.</i></strong>
+                <fieldset>
+                    <div class="field-group">
+                        {{ render_common_settings_form(form, current_base_url, emailprefix) }}
+                    </div>
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="filters-and-triggers">
+                <fieldset>
+                        <div class="pure-control-group">
+                            <strong>Pro-tips:</strong><br/>
+                            <ul>
+                                <li>
+                                    Use the preview page to see your filters and triggers highlighted.
+                                </li>
+                                <li>
+                                    Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a>
+                                </li>
+                            </ul>
+                    </div>
+
+                    <div class="pure-control-group">
+                        {{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.",
+                        class="m-d") }}
+                        <span class="pure-form-message-inline">
+                    <ul>
+                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
+                        <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <code>"json:"</code>, use <code>json:$</code> to force re-formatting if required,  <a
+                                href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
+                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash, example  <code>//*[contains(@class, 'sametext')]</code>, <a
+                                href="http://xpather.com/" target="new">test your XPath here</a></li>
+                    </ul>
+                    Please be sure that you thoroughly understand how to write CSS or JSONPath, XPath selector rules before filing an issue on GitHub! <a
+                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
+                </span>
+                    </div>
+                    <fieldset class="pure-group">
+                      {{ render_field(form.subtractive_selectors, rows=5, placeholder="header
+footer
+nav
+.stockticker") }}
+                      <span class="pure-form-message-inline">
+                        <ul>
+                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
+                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
+                        </ul>
+                      </span>
+                    </fieldset>
+                </fieldset>
+                <fieldset class="pure-group">
+                    {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
+/some.regex\d{2}/ for case-INsensitive regex
+                    ") }}
+                    <span class="pure-form-message-inline">
+                        <ul>
+                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
+                            <li>Regular Expression support, wrap the line in forward slash <code>/regex/</code></li>
+                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
+                            <li>Use the preview/show current tab to see ignores</li>
+                        </ul>
+                </span>
+
+            </fieldset>
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line
+/some.regex\d{2}/ for case-INsensitive regex
+                    ") }}
+                        <span class="pure-form-message-inline">
+                    <ul>
+                        <li>Text to wait for before triggering a change/notification, all text and regex are tested <i>case-insensitive</i>.</li>
+                        <li>Trigger text is processed from the result-text that comes out of any CSS/JSON Filters for this watch</li>
+                        <li>Each line is processed separately (think of each line as "OR")</li>
+                        <li>Note: Wrap in forward slash / to use regex  example: <code>/foo\d/</code></li>
+                    </ul>
+                        </span>
+                    </div>
+                </fieldset>
+            </div>
+
+            <div id="actions">
+                <div class="pure-control-group">
+
+                      {{ render_button(form.save_button) }} {{ render_button(form.save_and_preview_button) }}
+
+                    <a href="{{url_for('api_delete', uuid=uuid)}}"
+                       class="pure-button button-small button-error ">Delete</a>
+                    <a href="{{url_for('api_clone', uuid=uuid)}}"
+                       class="pure-button button-small ">Create Copy</a>
+                </div>
+            </div>
+        </form>
+    </div>
+</div>
+
+{% endblock %}

changedetectionio/templates/import.html

@@ -0,0 +1,30 @@
+{% extends 'base.html' %}
+
+{% block content %}
+<div class="edit-form">
+     <div class="inner">
+        <form class="pure-form pure-form-aligned" action="{{url_for('import_page')}}" method="POST">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+            <fieldset class="pure-group">
+              <legend>
+                Enter one URL per line, and optionally add tags for each URL after a space, delineated by comma (,):
+                <br>
+                <code>https://example.com tag1, tag2, last tag</code>
+                <br>
+                URLs which do not pass validation will stay in the textarea.
+              </legend>
+              
+
+                <textarea name="urls" class="pure-input-1-2" placeholder="https://"
+                          style="width: 100%;
+                                font-family:monospace;
+                                white-space: pre;
+                                overflow-wrap: normal;
+                                overflow-x: scroll;" rows="25">{{ remaining }}</textarea>
+            </fieldset>
+            <button type="submit" class="pure-button pure-input-1-2 pure-button-primary">Import</button>
+        </form>
+     </div>
+</div>
+
+{% endblock %}

changedetectionio/templates/login.html

@@ -0,0 +1,23 @@
+{% extends 'base.html' %}
+
+{% block content %}
+<div class="login-form">
+ <div class="inner">
+    <form class="pure-form pure-form-stacked" action="{{url_for('login')}}" method="POST">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+        <fieldset>
+            <div class="pure-control-group">
+                <label for="password">Password</label>
+                <input type="password" id="password" required="" name="password" value=""
+                       size="15" autofocus />
+                <input type="hidden" id="email" name="email" value="defaultuser@changedetection.io" />
+            </div>
+            <div class="pure-control-group">
+                <button type="submit" class="pure-button pure-button-primary">Login</button>
+            </div>
+        </fieldset>
+    </form>
+  </div>
+ </div>
+
+{% endblock %}

changedetectionio/templates/notification-log.html

@@ -0,0 +1,19 @@
+{% extends 'base.html' %}
+
+{% block content %}
+<div class="edit-form">
+     <div class="inner">
+
+         <h4 style="margin-top: 0px;">The following issues were detected when sending notifications</h4>
+                <div id="notification-error-log">
+                <ul style="font-size: 80%; margin:0px; padding: 0 0 0 7px">
+                {% for log in logs|reverse %}
+                    <li>{{log}}</li>
+                {% endfor %}
+                </ul>
+                </div>
+
+     </div>
+</div>
+
+{% endblock %}

changedetectionio/templates/preview.html

@@ -0,0 +1,45 @@
+{% extends 'base.html' %}
+
+{% block content %}
+
+<div id="settings">
+    <h1>Current - {{watch.last_checked|format_timestamp_timeago}}</h1>
+</div>
+
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<div class="tabs">
+    <ul>
+        <li class="tab" id="default-tab"><a href="#text">Text</a></li>
+{% if screenshot %}
+        <li class="tab"><a href="#screenshot">Current screenshot</a></li>
+{% endif %}
+    </ul>
+</div>
+
+<div id="diff-ui">
+    <div class="tab-pane-inner" id="text">
+        <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
+        <table>
+            <tbody>
+            <tr>
+                <td id="diff-col">
+                    {% for row in content %}
+                    <div class="{{row.classes}}">{{row.line}}</div>
+                    {% endfor %}
+                </td>
+            </tr>
+            </tbody>
+        </table>
+    </div>
+
+{% if screenshot %}
+     <div class="tab-pane-inner" id="screenshot">
+         <p>
+         <i>For now, only the most recent screenshot is saved and displayed.</i>
+             </p>
+
+        <img src="{{url_for('static_content', group='screenshot', filename=uuid)}}">
+     </div>
+{% endif %}
+</div>
+{% endblock %}

changedetectionio/templates/scrub.html

@@ -0,0 +1,32 @@
+{% extends 'base.html' %}
+
+{% block content %}
+<div class="edit-form">
+    <div class="box-wrap inner">
+    <form class="pure-form pure-form-stacked" action="{{url_for('scrub_page')}}" method="POST">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+        <fieldset>
+            <div class="pure-control-group">
+                This will remove ALL version snapshots/data, but keep your list of URLs. <br/>
+                You may like to use the <strong>BACKUP</strong> link first.<br/>
+            </div>
+            <br/>
+            <div class="pure-control-group">
+                <label for="confirmtext">Confirmation text</label>
+                <input type="text" id="confirmtext" required="" name="confirmtext" value="" size="10"/>
+                <span class="pure-form-message-inline">Type in the word <strong>scrub</strong> to confirm that you understand!</span>
+            </div>
+            <br/>
+            <div class="pure-control-group">
+                <button type="submit" class="pure-button pure-button-primary">Scrub!</button>
+            </div>
+            <br/>
+            <div class="pure-control-group">
+                <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Cancel</a>
+            </div>
+        </fieldset>
+    </form>
+    </div>
+</div>
+
+{% endblock %}

changedetectionio/templates/settings.html

@@ -0,0 +1,142 @@
+{% extends 'base.html' %}
+
+{% block content %}
+{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
+{% from '_common_fields.jinja' import render_common_settings_form %}
+<script>
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+{% if emailprefix %}
+    const email_notification_prefix=JSON.parse('{{emailprefix|tojson}}');
+{% endif %}
+</script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='settings.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
+
+<div class="edit-form">
+    <div class="tabs collapsable">
+        <ul>
+            <li class="tab" id="default-tab"><a href="#general">General</a></li>
+            <li class="tab"><a href="#notifications">Notifications</a></li>
+            <li class="tab"><a href="#fetching">Fetching</a></li>
+            <li class="tab"><a href="#filters">Global Filters</a></li>
+        </ul>
+    </div>
+    <div class="box-wrap inner">
+        <form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+            <div class="tab-pane-inner" id="general">
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }}
+                        <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
+                    </div>
+                    <div class="pure-control-group">
+                        {% if not hide_remove_pass %}
+                            {% if current_user.is_authenticated %}
+                                {{ render_button(form.application.form.removepassword_button) }}
+                            {% else %}
+                            {{ render_field(form.application.form.password) }}
+                            <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
+                            {% endif %}
+                        {% else %}
+                            <span class="pure-form-message-inline">Password is locked.</span>
+                        {% endif %}
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
+                        class="m-d") }}
+                        <span class="pure-form-message-inline">
+                            Base URL used for the {base_url} token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{current_base_url}}"),
+                            <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
+                        </span>
+                    </div>
+
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.application.form.extract_title_as_title) }}
+                        <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
+                    </div>
+
+                    <div class="pure-control-group">
+                        {{ render_checkbox_field(form.application.form.real_browser_save_screenshot) }}
+                        <span class="pure-form-message-inline">When using a Chrome browser, a screenshot from the last check will be available on the Diff page</span>
+                    </div>
+
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="notifications">
+                <fieldset>
+                    <div class="field-group">
+                        {{ render_common_settings_form(form.application.form, current_base_url, emailprefix) }}
+                    </div>
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="fetching">
+                <div class="pure-control-group">
+                    {{ render_field(form.application.form.fetch_backend, class="fetch-backend") }}
+                    <span class="pure-form-message-inline">
+                        <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
+                        <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p>
+                    </span>
+                </div>
+            </div>
+
+
+            <div class="tab-pane-inner" id="filters">
+
+                    <fieldset class="pure-group">
+                    {{ render_checkbox_field(form.application.form.ignore_whitespace) }}
+                    <span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br/>
+                    <i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc.
+                    </span>
+                    </fieldset>
+                <fieldset class="pure-group">
+                    {{ render_checkbox_field(form.application.form.render_anchor_tag_content) }}
+                    <span class="pure-form-message-inline">Render anchor tag content, default disabled, when enabled renders links as <code>(link text)[https://somesite.com]</code>
+                        <br/>
+                    <i>Note:</i> Changing this could affect the content of your existing watches, possibly trigger alerts etc.
+                    </span>
+                    </fieldset>
+                    <fieldset class="pure-group">
+                      {{ render_field(form.application.form.global_subtractive_selectors, rows=5, placeholder="header
+footer
+nav
+.stockticker") }}
+                      <span class="pure-form-message-inline">
+                        <ul>
+                          <li> Remove HTML element(s) by CSS selector before text conversion. </li>
+                          <li> Add multiple elements or CSS selectors per line to ignore multiple parts of the HTML. </li>
+                        </ul>
+                      </span>
+                    </fieldset>
+                    <fieldset class="pure-group">
+                    {{ render_field(form.application.form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line
+/some.regex\d{2}/ for case-INsensitive regex
+                    ") }}
+                    <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br/>
+                    <span class="pure-form-message-inline">
+                        <ul>
+                            <li>Note: This is applied globally in addition to the per-watch rules.</li>
+                            <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
+                            <li>Regular Expression support, wrap the line in forward slash <code>/regex/</code></li>
+                            <li>Changing this will affect the comparison checksum which may trigger an alert</li>
+                            <li>Use the preview/show current tab to see ignores</li>
+                        </ul>
+                     </span>
+                    </fieldset>
+           </div>
+
+            <div id="actions">
+                <div class="pure-control-group">
+                    {{ render_button(form.save_button) }}
+                    <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
+                    <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
+                </div>
+            </div>
+        </form>
+    </div>
+</div>
+
+{% endblock %}

changedetectionio/templates/watch-overview.html

@@ -0,0 +1,108 @@
+{% extends 'base.html' %}
+{% block content %}
+{% from '_helpers.jinja' import render_simple_field %}
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='jquery-3.6.0.min.js')}}"></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='watch-overview.js')}}" defer></script>
+<div class="box">
+
+    <form class="pure-form" action="{{ url_for('api_watch_add') }}" method="POST" id="new-watch-form">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+        <fieldset>
+            <legend>Add a new change detection watch</legend>
+                {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
+                {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch group") }}
+            <button type="submit" class="pure-button pure-button-primary">Watch</button>
+        </fieldset>
+        <span style="color:#eee; font-size: 80%;"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /> Tip: You can also add 'shared' watches. <a href="#">More info</a></a></span>
+    </form>
+    <div>
+        <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
+        {% for tag in tags %}
+            {% if tag != "" %}
+                <a href="{{url_for('index', tag=tag) }}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
+            {% endif %}
+        {% endfor %}
+    </div>
+
+    <div id="watch-table-wrapper">
+        <table class="pure-table pure-table-striped watch-table">
+            <thead>
+            <tr>
+                <th>#</th>
+                <th></th>
+                <th></th>
+                <th>Last Checked</th>
+                <th>Last Changed</th>
+                <th></th>
+            </tr>
+            </thead>
+            <tbody>
+
+
+            {% for watch in watches %}
+            <tr id="{{ watch.uuid }}"
+                class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
+                {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
+                {% if watch.last_notification_error is defined and watch.last_notification_error != False %}error{% endif %}
+                {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
+                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}
+                {% if watch.uuid in queued_uuids %}queued{% endif %}">
+                <td class="inline">{{ loop.index }}</td>
+                <td class="inline paused-state state-{{watch.paused}}"><a href="{{url_for('index', pause=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause" title="Pause"/></a></td>
+
+                <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
+                    <a class="external" target="_blank" rel="noopener" href="{{ watch.url.replace('source:','') }}"></a>
+                    <a href="{{url_for('api_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
+
+                    {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
+
+                    {% if watch.last_error is defined and watch.last_error != False %}
+                    <div class="fetch-error">{{ watch.last_error }}</div>
+                    {% endif %}
+                    {% if watch.last_notification_error is defined and watch.last_notification_error != False %}
+                    <div class="fetch-error notification-error">{{ watch.last_notification_error }}</div>
+                    {% endif %}
+                    {% if not active_tag %}
+                    <span class="watch-tag-list">{{ watch.tag}}</span>
+                    {% endif %}
+                </td>
+                <td class="last-checked">{{watch|format_last_checked_time}}</td>
+                <td class="last-changed">{% if watch.history|length >= 2 and watch.last_changed %}
+                    {{watch.last_changed|format_timestamp_timeago}}
+                    {% else %}
+                    Not yet
+                    {% endif %}
+                </td>
+                <td>
+                    <a {% if watch.uuid in queued_uuids %}disabled="true"{% endif %} href="{{ url_for('api_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
+                       class="recheck pure-button button-small pure-button-primary">{% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %}</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
+                    {% if watch.history|length >= 2 %}
+                    <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary diff-link">Diff</a>
+                    {% else %}
+                        {% if watch.history|length == 1 %}
+                            <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
+                        {% endif %}
+                    {% endif %}
+                </td>
+            </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+        <ul id="post-list-buttons">
+            {% if has_unviewed %}
+            <li>
+                <a href="{{url_for('mark_all_viewed', tag=request.args.get('tag')) }}" class="pure-button button-tag ">Mark all viewed</a>
+            </li>
+            {% endif %}
+            <li>
+               <a href="{{ url_for('api_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
+                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
+            </li>
+            <li>
+                <a href="{{ url_for('rss', tag=active_tag , token=app_rss_token)}}"><img alt="RSS Feed" id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15"></a>
+            </li>
+        </ul>
+    </div>
+</div>
+{% endblock %}

changedetectionio/tests/__init__.py

@@ -0,0 +1,2 @@
+"""Tests for the app."""
+

changedetectionio/tests/conftest.py

@@ -0,0 +1,58 @@
+#!/usr/bin/python3
+
+import pytest
+from changedetectionio import changedetection_app
+from changedetectionio import store
+import os
+
+# https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
+# Much better boilerplate than the docs
+# https://www.python-boilerplate.com/py3+flask+pytest/
+
+global app
+
+
+def cleanup(datastore_path):
+    # Unlink test output files
+    files = ['output.txt',
+             'url-watches.json',
+             'secret.txt',
+             'notification.txt',
+             'count.txt',
+             'endpoint-content.txt'
+                 ]
+    for file in files:
+        try:
+            os.unlink("{}/{}".format(datastore_path, file))
+        except FileNotFoundError:
+            pass
+
+@pytest.fixture(scope='session')
+def app(request):
+    """Create application for the tests."""
+    datastore_path = "./test-datastore"
+
+    try:
+        os.mkdir(datastore_path)
+    except FileExistsError:
+        pass
+
+    cleanup(datastore_path)
+
+    app_config = {'datastore_path': datastore_path}
+    cleanup(app_config['datastore_path'])
+    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
+    app = changedetection_app(app_config, datastore)
+
+    # Disable CSRF while running tests
+    app.config['WTF_CSRF_ENABLED'] = False
+    app.config['STOP_THREADS'] = True
+
+    def teardown():
+        datastore.stop_thread = True
+        app.config.exit.set()
+        cleanup(app_config['datastore_path'])
+
+       
+    request.addfinalizer(teardown)
+    yield app

changedetectionio/tests/test_access_control.py

@@ -0,0 +1,79 @@
+from flask import url_for
+from . util import live_server_setup
+
+def test_check_access_control(app, client):
+    # Still doesnt work, but this is closer.
+
+    with app.test_client(use_cookies=True) as c:
+        # Check we don't have any password protection enabled yet.
+        res = c.get(url_for("settings_page"))
+        assert b"Remove password" not in res.data
+
+        # Enable password check.
+        res = c.post(
+            url_for("settings_page"),
+            data={"application-password": "foobar",
+                  "requests-time_between_check-minutes": 180,
+                  'application-fetch_backend': "html_requests"},
+            follow_redirects=True
+        )
+
+        assert b"Password protection enabled." in res.data
+        assert b"LOG OUT" not in res.data
+
+        # Check we hit the login
+        res = c.get(url_for("index"), follow_redirects=True)
+
+        assert b"Login" in res.data
+
+        # Menu should not be available yet
+        #        assert b"SETTINGS" not in res.data
+        #        assert b"BACKUP" not in res.data
+        #        assert b"IMPORT" not in res.data
+
+        # defaultuser@changedetection.io is actually hardcoded for now, we only use a single password
+        res = c.post(
+            url_for("login"),
+            data={"password": "foobar"},
+            follow_redirects=True
+        )
+
+        assert b"LOG OUT" in res.data
+        res = c.get(url_for("settings_page"))
+
+        # Menu should be available now
+        assert b"SETTINGS" in res.data
+        assert b"BACKUP" in res.data
+        assert b"IMPORT" in res.data
+        assert b"LOG OUT" in res.data
+        assert b"time_between_check-minutes" in res.data
+        assert b"fetch_backend" in res.data
+
+        ##################################################
+        # Remove password button, and check that it worked
+        ##################################################
+        res = c.post(
+            url_for("settings_page"),
+            data={
+                "requests-time_between_check-minutes": 180,
+                "application-fetch_backend": "html_webdriver",
+                "application-removepassword_button": "Remove password"
+            },
+            follow_redirects=True,
+        )
+        assert b"Password protection removed." in res.data
+        assert b"LOG OUT" not in res.data
+
+        ############################################################
+        # Be sure a blank password doesnt setup password protection
+        ############################################################
+        res = c.post(
+            url_for("settings_page"),
+            data={"application-password": "",
+                  "requests-time_between_check-minutes": 180,
+                  'application-fetch_backend': "html_requests"},
+            follow_redirects=True
+        )
+
+        assert b"Password protection enabled" not in res.data
+

changedetectionio/tests/test_api.py

@@ -0,0 +1,74 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def set_response_data(test_return_data):
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_snapshot_api_detects_change(client, live_server):
+    test_return_data = "Some initial text"
+
+    test_return_data_modified = "Some NEW nice initial text"
+
+    sleep_time_for_fetch_thread = 3
+
+    set_response_data(test_return_data)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="text/plain",
+                       _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(
+        url_for("api_snapshot", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert test_return_data.encode() == res.data
+
+    #  Make a change
+    set_response_data(test_return_data_modified)
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(
+        url_for("api_snapshot", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert test_return_data_modified.encode() == res.data
+
+def test_snapshot_api_invalid_uuid(client, live_server):
+
+    res = client.get(
+        url_for("api_snapshot", uuid="invalid"),
+        follow_redirects=True
+    )
+
+    assert res.status_code == 400
+

changedetectionio/tests/test_auth.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_basic_auth(client, live_server):
+
+    live_server_setup(live_server)
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@")
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Check form validation
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(1)
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b'myuser mypass basic' in res.data

changedetectionio/tests/test_backend.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from urllib.request import urlopen
+from . util import set_original_response, set_modified_response, live_server_setup
+
+sleep_time_for_fetch_thread = 3
+
+# Basic test to check inscriptus is not adding return line chars, basically works etc
+def test_inscriptus():
+    from inscriptis import get_text
+    html_content="<html><body>test!<br/>ok man</body></html>"
+    stripped_text_from_html = get_text(html_content)
+    assert stripped_text_from_html == 'test!\nok man'
+
+
+def test_check_basic_change_detection_functionality(client, live_server):
+    set_original_response()
+    live_server_setup(live_server)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url_for('test_endpoint', _external=True)},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Do this a few times.. ensures we dont accidently set the status
+    for n in range(3):
+        client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+        # Give the thread time to pick it up
+        time.sleep(sleep_time_for_fetch_thread)
+
+        # It should report nothing found (no new 'unviewed' class)
+        res = client.get(url_for("index"))
+        assert b'unviewed' not in res.data
+        assert b'test-endpoint' in res.data
+
+        # Default no password set, this stuff should be always available.
+
+        assert b"SETTINGS" in res.data
+        assert b"BACKUP" in res.data
+        assert b"IMPORT" in res.data
+
+    #####################
+
+    # Check HTML conversion detected and workd
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    # Check this class does not appear (that we didnt see the actual source)
+    assert b'foobar-detection' not in res.data
+
+    # Make a change
+    set_modified_response()
+
+    res = urlopen(url_for('test_endpoint', _external=True))
+    assert b'which has this one new line' in res.read()
+
+    # Force recheck
+    res = client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    assert b'1 watches are queued for rechecking.' in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Now something should be ready, indicated by having a 'unviewed' class
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    # #75, and it should be in the RSS feed
+    res = client.get(url_for("rss"))
+    expected_url = url_for('test_endpoint', _external=True)
+    assert b'<rss' in res.data
+
+    # re #16 should have the diff in here too
+    assert b'(into   ) which has this one new line' in res.data
+    assert b'CDATA' in res.data
+    
+    assert expected_url.encode('utf-8') in res.data
+
+    # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
+    res = client.get(url_for("diff_history_page", uuid="first"))
+    assert b'Compare newest' in res.data
+
+    time.sleep(2)
+
+    # Do this a few times.. ensures we dont accidently set the status
+    for n in range(2):
+        client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+        # Give the thread time to pick it up
+        time.sleep(sleep_time_for_fetch_thread)
+
+        # It should report nothing found (no new 'unviewed' class)
+        res = client.get(url_for("index"))
+        assert b'unviewed' not in res.data
+        assert b'head title' not in res.data # Should not be present because this is off by default
+        assert b'test-endpoint' in res.data
+
+    set_original_response()
+
+    # Enable auto pickup of <title> in settings
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-extract_title_as_title": "1", "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+    # It should have picked up the <title>
+    assert b'head title' in res.data
+
+    #
+    # Cleanup everything
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+

changedetectionio/tests/test_backup.py

@@ -0,0 +1,25 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from urllib.request import urlopen
+from . util import set_original_response, set_modified_response, live_server_setup
+
+
+def test_backup(client, live_server):
+
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    res = client.get(
+        url_for("get_backup"),
+        follow_redirects=True
+    )
+
+    # Should get the right zip content type
+    assert res.content_type == "application/zip"
+    # Should be PK/ZIP stream
+    assert res.data.count(b'PK') >= 2
+

changedetectionio/tests/test_clone.py

@@ -0,0 +1,30 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+
+
+def test_trigger_functionality(client, live_server):
+
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": "https://changedetection.io"},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+
+    res = client.get(
+        url_for("api_clone", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"Cloned." in res.data

changedetectionio/tests/test_css_selector.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+from ..html_tools import *
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that will change</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that changes</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+# Test that the CSS extraction works how we expect, important here is the right placing of new lines \n's
+def test_css_filter_output():
+    from changedetectionio import fetch_site_status
+    from inscriptis import get_text
+
+    # Check text with sub-parts renders correctly
+    content = """<html> <body><div id="thingthing" >  Some really <b>bold</b> text  </div> </body> </html>"""
+    html_blob = css_filter(css_filter="#thingthing", html_content=content)
+    text = get_text(html_blob)
+    assert text == "  Some really bold text"
+
+    content = """<html> <body>
+    <p>foo bar blah</p>
+    <div class="parts">Block A</div> <div class="parts">Block B</div></body> 
+    </html>
+"""
+    html_blob = css_filter(css_filter=".parts", html_content=content)
+    text = get_text(html_blob)
+
+    # Divs are converted to 4 whitespaces by inscriptis
+    assert text == "    Block A\n    Block B"
+
+
+# Tests the whole stack works with the CSS Filter
+def test_check_markup_css_filter_restriction(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    css_filter = "#sametext"
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(css_filter.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+    #  Make a change
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should have 'unviewed' still
+    # Because it should be looking at only that 'sametext' id
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data

changedetectionio/tests/test_element_removal.py

@@ -0,0 +1,168 @@
+#!/usr/bin/python3
+
+import time
+
+from flask import url_for
+
+from ..html_tools import *
+from .util import live_server_setup
+
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def set_original_response():
+    test_return_data = """<html>
+    <header>
+    <h2>Header</h2>
+    </header>
+    <nav>
+    <ul>
+      <li><a href="#">A</a></li>
+      <li><a href="#">B</a></li>
+      <li><a href="#">C</a></li>
+    </ul>
+    </nav>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+    <div id="changetext">Some text that will change</div>
+     </body>
+    <footer>
+    <p>Footer</p>
+    </footer>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_modified_response():
+    test_return_data = """<html>
+    <header>
+    <h2>Header changed</h2>
+    </header>
+    <nav>
+    <ul>
+      <li><a href="#">A changed</a></li>
+      <li><a href="#">B</a></li>
+      <li><a href="#">C</a></li>
+    </ul>
+    </nav>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+    <div id="changetext">Some text that changes</div>
+     </body>
+    <footer>
+    <p>Footer changed</p>
+    </footer>
+     </html>
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_element_removal_output():
+    from changedetectionio import fetch_site_status
+    from inscriptis import get_text
+
+    # Check text with sub-parts renders correctly
+    content = """<html>
+    <header>
+    <h2>Header</h2>
+    </header>
+    <nav>
+    <ul>
+      <li><a href="#">A</a></li>
+    </ul>
+    </nav>
+       <body>
+     Some initial text</br>
+     <p>across multiple lines</p>
+     <div id="changetext">Some text that changes</div>
+     </body>
+    <footer>
+    <p>Footer</p>
+    </footer>
+     </html>
+    """
+    html_blob = element_removal(
+        ["header", "footer", "nav", "#changetext"], html_content=content
+    )
+    text = get_text(html_blob)
+    assert (
+        text
+        == """Some initial text
+
+across multiple lines
+"""
+    )
+
+
+def test_element_removal_full(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for("test_endpoint", _external=True)
+    res = client.post(
+        url_for("import_page"), data={"urls": test_url}, follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Goto the edit page, add the filter data
+    # Not sure why \r needs to be added - absent of the #changetext this is not necessary
+    subtractive_selectors_data = "header\r\nfooter\r\nnav\r\n#changetext"
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "subtractive_selectors": subtractive_selectors_data,
+            "url": test_url,
+            "tag": "",
+            "headers": "",
+            "fetch_backend": "html_requests",
+        },
+        follow_redirects=True,
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(subtractive_selectors_data.encode("utf-8")) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # No change yet - first check
+    res = client.get(url_for("index"))
+    assert b"unviewed" not in res.data
+
+    #  Make a change to header/footer/nav
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # There should not be an unviewed change, as changes should be removed
+    res = client.get(url_for("index"))
+    assert b"unviewed" not in res.data

changedetectionio/tests/test_encoding.py

@@ -0,0 +1,87 @@
+#!/usr/bin/python3
+# coding=utf-8
+
+import time
+from flask import url_for
+from .util import live_server_setup
+import pytest
+
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def set_html_response():
+    test_return_data = """
+<html><body><span class="nav_second_img_text">
+                  &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;铸大国重器，挺制造脊梁，致力能源未来，赋能美好生活。
+                                  </span>
+</body></html>
+    """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+
+# In the case the server does not issue a charset= or doesnt have content_type header set
+def test_check_encoding_detection(client, live_server):
+    set_html_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', content_type="text/html", _external=True)
+    client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(2)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should see the proper string
+    assert "铸大国重".encode('utf-8') in res.data
+    # Should not see the failed encoding
+    assert b'\xc2\xa7' not in res.data
+
+
+# In the case the server does not issue a charset= or doesnt have content_type header set
+def test_check_encoding_detection_missing_content_type_header(client, live_server):
+    set_html_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(2)
+
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should see the proper string
+    assert "铸大国重".encode('utf-8') in res.data
+    # Should not see the failed encoding
+    assert b'\xc2\xa7' not in res.data

changedetectionio/tests/test_errorhandling.py

@@ -0,0 +1,65 @@
+#!/usr/bin/python3
+
+import time
+
+from flask import url_for
+from . util import live_server_setup
+
+from ..html_tools import *
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+
+def test_error_handler(client, live_server):
+
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint',
+                       status_code=403,
+                       _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'Status Code 403' in res.data
+    assert bytes("just now".encode('utf-8')) in res.data
+
+# Just to be sure error text is properly handled
+def test_error_text_handler(client, live_server):
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": "https://errorfuldomainthatnevereallyexists12356.com"},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    res = client.get(url_for("index"))
+    assert b'Name or service not known' in res.data
+    assert bytes("just now".encode('utf-8')) in res.data
+

changedetectionio/tests/test_html_to_text.py

@@ -0,0 +1,38 @@
+#!/usr/bin/python3
+"""Test suite for the method to extract text from an html string"""
+from ..html_tools import html_to_text
+
+
+def test_html_to_text_func():
+    test_html = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     <a href="/first_link"> More Text </a>
+     </br>
+     So let's see what happens.  </br>
+     <a href="second_link.com"> Even More Text </a>
+     </body>
+     </html>
+    """
+
+    # extract text, with 'render_anchor_tag_content' set to False
+    text_content = html_to_text(test_html, render_anchor_tag_content=False)
+
+    no_links_text = \
+        "Some initial text\n\nWhich is across multiple " \
+        "lines\n\nMore Text So let's see what happens. Even More Text"
+
+    # check that no links are in the extracted text
+    assert text_content == no_links_text
+
+    # extract text, with 'render_anchor_tag_content' set to True
+    text_content = html_to_text(test_html, render_anchor_tag_content=True)
+
+    links_text = \
+        "Some initial text\n\nWhich is across multiple lines\n\n[ More Text " \
+        "](/first_link) So let's see what happens. [ Even More Text ]" \
+        "(second_link.com)"
+
+    # check that links are present in the extracted text
+    assert text_content == links_text

changedetectionio/tests/test_ignore_regex_text.py

@@ -0,0 +1,32 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+from changedetectionio import html_tools
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+# Unit test of the stripper
+# Always we are dealing in utf-8
+def test_strip_regex_text_func():
+    from changedetectionio import fetch_site_status
+
+    test_content = """
+    but sometimes we want to remove the lines.
+    
+    but 1 lines
+    but including 1234 lines
+    igNORe-cAse text we dont want to keep    
+    but not always."""
+
+    ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"]
+
+    fetcher = fetch_site_status.perform_site_check(datastore=False)
+    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
+
+    assert b"but 1 lines" in stripped_content
+    assert b"igNORe-cAse text" not in stripped_content
+    assert b"but 1234 lines" not in stripped_content
+

changedetectionio/tests/test_ignore_text.py

@@ -0,0 +1,260 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+from changedetectionio import html_tools
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+# Unit test of the stripper
+# Always we are dealing in utf-8
+def test_strip_text_func():
+    from changedetectionio import fetch_site_status
+
+    test_content = """
+    Some content
+    is listed here
+
+    but sometimes we want to remove the lines.
+
+    but not always."""
+
+    ignore_lines = ["sometimes"]
+
+    fetcher = fetch_site_status.perform_site_check(datastore=False)
+    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
+
+    assert b"sometimes" not in stripped_content
+    assert b"Some content" in stripped_content
+
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_modified_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some NEW nice initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <p>new ignore stuff</p>
+     <p>blah</p>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+# Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
+def set_modified_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     <P>ZZZZz</P>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_check_ignore_text_functionality(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    # Use a mix of case in ZzZ to prove it works case-insensitive.
+    ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
+    set_original_ignore_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"ignore_text": ignore_text, "url": test_url, 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(ignore_text.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+    #  Make a change
+    set_modified_ignore_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+
+
+
+
+    # Just to be sure.. set a regular modified change..
+    set_modified_original_ignore_response()
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    # Check the preview/highlighter, we should be able to see what we ignored, but it should be highlighted
+    # We only introduce the "modified" content that includes what we ignore so we can prove the newest version also displays
+    # at /preview
+    res = client.get(url_for("preview_page", uuid="first"))
+    # We should be able to see what we ignored
+    assert b'<div class="ignored">new ignore stuff' in res.data
+
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+def test_check_global_ignore_text_functionality(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
+    set_original_ignore_response()
+
+    # Goto the settings page, add our ignore text
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-global_ignore_text": ignore_text,
+            'application-fetch_backend': "html_requests"
+        },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+
+    # Goto the edit page of the item, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"ignore_text": "something irrelevent but just to check", "url": test_url, 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("settings_page"),
+    )
+    assert bytes(ignore_text.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # so that we are sure everything is viewed and in a known 'nothing changed' state
+    res = client.get(url_for("diff_history_page", uuid="first"))
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+
+    #  Make a change which includes the ignore text
+    set_modified_ignore_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+    # Just to be sure.. set a regular modified change that will trigger it
+    set_modified_original_ignore_response()
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data