Merge branch 'master' of github.com:dgtlmoon/changedetection.io

* Re #106 - handling empty title with gettr cleanup

* Re #110 - Global recheck time improvements, add tests, add form feedback, follow default minutes

* Adding comments

.dockerignore

@@ -0,0 +1,2 @@
+.git
+.github

.github/FUNDING.yml

@@ -1,12 +1,3 @@
 # These are supported funding model platforms
 
 github: dgtlmoon
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,62 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  schedule:
+    - cron: '27 9 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/image.yml

@@ -0,0 +1,88 @@
+name: Test, build and push to Docker Hub
+
+on:
+  push:
+    branches: [ master, arm-build ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Create release metadata
+        run: |
+          # COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > backend/source.txt
+          echo ${{ github.ref }} > backend/tag.txt
+
+      - name: Test with pytest
+        run: |
+          # Each test is totally isolated and performs its own cleanup/reset
+          cd backend; ./run_all_tests.sh
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: all
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+          version: latest
+          driver-opts: image=moby/buildkit:master
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
+          #                ${{ secrets.DOCKER_HUB_USERNAME }}:/changedetection.io:${{ env.RELEASE_VERSION }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+#          platforms: linux/amd64
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+
+      - name: Image digest
+        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
+
+# failed: Cache service responded with 503
+#      - name: Cache Docker layers
+#        uses: actions/cache@v2
+#        with:
+#          path: /tmp/.buildx-cache
+#          key: ${{ runner.os }}-buildx-${{ github.sha }}
+#          restore-keys: |
+#            ${{ runner.os }}-buildx-
+
+

.github/workflows/test-only.yml

@@ -0,0 +1,33 @@
+name: ChangeDetection.io Test
+
+# Triggers the workflow on push or pull request events
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Test with pytest
+        run: |
+          # Each test is totally isolated and performs its own cleanup/reset
+          cd backend; ./run_all_tests.sh
+

.gitignore

@@ -2,4 +2,6 @@ __pycache__
 .idea
 *.pyc
 datastore/url-watches.json
-datastore/*
+datastore/*
+__pycache__
+.pytest_cache

Dockerfile

@@ -1,17 +1,56 @@
-FROM python:3.8-slim
-COPY requirements.txt /tmp/requirements.txt
-RUN pip3 install -r /tmp/requirements.txt
+# pip dependencies install stage
+FROM python:3.8-slim as builder
 
-COPY backend /app
-WORKDIR /app
+# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
+ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl-dev \
+    libffi-dev \
+    gcc \
+    libc-dev \
+    libxslt-dev \
+    zlib1g-dev \
+    g++
+    
+RUN mkdir /install
+WORKDIR /install
+
+COPY requirements.txt /requirements.txt
+
+RUN pip install --target=/dependencies -r /requirements.txt
+
+# Final image stage
+FROM python:3.8-slim
+
+# Actual packages needed at runtime, usually due to the notification (apprise) backend
+# rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
+ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
+
+# Re #93, #73, excluding rustc (adds another 430Mb~)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl-dev \
+    libffi-dev \
+    gcc \
+    libc-dev \
+    libxslt-dev \
+    zlib1g-dev \
+    g++
 
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
-
 ENV PYTHONUNBUFFERED=1
 
 RUN [ ! -d "/datastore" ] && mkdir /datastore
 
-CMD [ "python", "./backend.py" ]
+# Copy modules over to the final image and add their dir to PYTHONPATH
+COPY --from=builder /dependencies /usr/local
+ENV PYTHONPATH=/usr/local
 
+# The actual flask app
+COPY backend /app/backend
+# The eventlet server wrapper
+COPY changedetection.py /app/changedetection.py
 
+WORKDIR /app
 
+CMD [ "python", "./changedetection.py" , "-d", "/datastore"]

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,9 +1,20 @@
 #  changedetection.io
+![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
+<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
+  <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
+</a>
+<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
+  <img src="https://img.shields.io/github/v/release/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
+</a>
 
 ## Self-hosted change monitoring of web pages.
 
-_Know when web pages change! Stay ontop of new information!_
+_Know when web pages change! Stay ontop of new information!_ 
 
+Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
+
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
 
 #### Example use cases
 
@@ -11,32 +22,108 @@ Know when ...
 
 - Government department updates (changes are often only on their websites)
 - Local government news (changes are often only on their websites)
-- New software releases 
+- New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
 - Realestate listing changes
+- COVID related news from government websites
 
 
-Get monitoring now! super simple, one command!
+**Get monitoring now! super simple, one command!**
+Run the python code on your own machine by cloning this repository, or with <a href="https://docs.docker.com/get-docker/">docker</a> and/or <a href="https://www.digitalocean.com/community/tutorial_collections/how-to-install-docker-compose">docker-compose</a>
 
+With one docker-compose command
+
+```bash
+docker-compose up -d
 ```
-$ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore dgtlmoon/changedetection.io
+
+or
+
+
+```bash
+docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```  
 
 Now visit http://127.0.0.1:5000 , You should now be able to access the UI.
 
+#### Updating to latest version
+
+Highly recommended :)
+
+```bash
+docker pull dgtlmoon/changedetection.io
+docker kill $(docker ps -a|grep changedetection.io|awk '{print $1}')
+docker rm $(docker ps -a|grep changedetection.io|awk '{print $1}')
+docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
+```
   
+### Screenshots
 
-![Self-hoster web page change monitoring application screenshot](screenshot.png?raw=true "Self-hosted web page change monitoring screenshot")
+Examining differences in content.
 
 
-### Future plans
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
 
-- Greater configuration of check interval times, page request headers.
-- ~~General options for timeout, default headers~~
-- On change detection, callout to another API (handy for notices/issue trackers)
-- ~~Explore the differences that were detected~~ 
-- Add more options to explore versions of differences
-- Use a graphic/rendered page difference instead of text (see the experimental `selenium-screenshot-diff` branch)
-
- 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
+
+### Notifications
+
+ChangeDetection.io supports a massive amount of notifications (including email, office365, custom APIs, etc) when a web-page has a change detected thanks to the <a href="https://github.com/caronc/apprise">apprise</a> library.
+Simply set one or more notification URL's in the _[edit]_ tab of that watch.
+
+Just some examples
+
+    discord://webhook_id/webhook_token
+    flock://app_token/g:channel_id
+    gitter://token/room
+    gchat://workspace/key/token
+    msteams://TokenA/TokenB/TokenC/
+    o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+    rocket://user:password@hostname/#Channel
+    mailto://user:pass@example.com?to=receivingAddress@example.com
+    json://someserver.com/custom-api
+    syslog://
+ 
+<a href="https://github.com/caronc/apprise">And everything else in this list!</a>
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
+
+### Proxy
+
+A proxy for ChangeDectection.io can be configured by setting environment the 
+`HTTP_PROXY`, `HTTPS_PROXY` variables, examples are also in the `docker-compose.yml`
+
+`NO_PROXY` exclude list can be specified by following `"localhost,192.168.0.0/24"`
+
+as `docker run` with `-e`
+
+```
+docker run -d --restart always -e HTTPS_PROXY="socks5h://10.10.1.10:1080" -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
+```
+
+With `docker-compose`, see the `Proxy support example` in <a href="https://github.com/dgtlmoon/changedetection.io/blob/master/docker-compose.yml">docker-compose.yml</a>.
+
+For more information see https://docs.python-requests.org/en/master/user/advanced/#proxies
+
+This proxy support also extends to the notifications https://github.com/caronc/apprise/issues/387#issuecomment-841718867
+
+### Notes
+
+- Does not yet support Javascript
+- Wont work with Cloudfare type "Please turn on javascript" protected pages
+- You can use the 'headers' section to monitor password protected web page changes
+
+### RaspberriPi support?
+
+RaspberriPi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! 
+
+
+### Support us
+
+Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
+
+Please support us, even small amounts help a LOT.
+
+BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />

backend/__init__.py

@@ -0,0 +1,894 @@
+#!/usr/bin/python3
+
+
+# @todo logging
+# @todo extra options for url like , verify=False etc.
+# @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option?
+# @todo option for interval day/6 hour/etc
+# @todo on change detected, config for calling some API
+# @todo fetch title into json
+# https://distill.io/features
+# proxy per check
+#  - flask_cors, itsdangerous,MarkupSafe
+
+import time
+import os
+import timeago
+import flask_login
+from flask_login import login_required
+
+import threading
+from threading import Event
+
+import queue
+
+from flask import Flask, render_template, request, send_from_directory, abort, redirect, url_for, flash
+
+from feedgen.feed import FeedGenerator
+from flask import make_response
+import datetime
+import pytz
+
+datastore = None
+
+# Local
+running_update_threads = []
+ticker_thread = None
+
+extra_stylesheets = []
+
+update_q = queue.Queue()
+
+notification_q = queue.Queue()
+
+app = Flask(__name__, static_url_path="/var/www/change-detection/backend/static")
+
+# Stop browser caching of assets
+app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
+
+app.config.exit = Event()
+
+app.config['NEW_VERSION_AVAILABLE'] = False
+
+app.config['LOGIN_DISABLED'] = False
+
+#app.config["EXPLAIN_TEMPLATE_LOADING"] = True
+
+# Disables caching of the templates
+app.config['TEMPLATES_AUTO_RELOAD'] = True
+
+
+def init_app_secret(datastore_path):
+    secret = ""
+
+    path = "{}/secret.txt".format(datastore_path)
+
+    try:
+        with open(path, "r") as f:
+            secret = f.read()
+
+    except FileNotFoundError:
+        import secrets
+        with open(path, "w") as f:
+            secret = secrets.token_hex(32)
+            f.write(secret)
+
+    return secret
+
+# Remember python is by reference
+# populate_form in wtfors didnt work for me. (try using a setattr() obj type on datastore.watch?)
+def populate_form_from_watch(form, watch):
+    for i in form.__dict__.keys():
+        if i[0] != '_':
+            p = getattr(form, i)
+            if hasattr(p, 'data') and i in watch:
+                if not p.data:
+                    setattr(p, "data", watch[i])
+
+
+# We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
+# running or something similar.
+@app.template_filter('format_last_checked_time')
+def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
+    # Worker thread tells us which UUID it is currently processing.
+    for t in running_update_threads:
+        if t.current_uuid == watch_obj['uuid']:
+            return "Checking now.."
+
+    if watch_obj['last_checked'] == 0:
+        return 'Not yet'
+
+    return timeago.format(int(watch_obj['last_checked']), time.time())
+
+
+# @app.context_processor
+# def timeago():
+#    def _timeago(lower_time, now):
+#        return timeago.format(lower_time, now)
+#    return dict(timeago=_timeago)
+
+@app.template_filter('format_timestamp_timeago')
+def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
+    return timeago.format(timestamp, time.time())
+    # return timeago.format(timestamp, time.time())
+    # return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
+
+
+class User(flask_login.UserMixin):
+    id=None
+
+    def set_password(self, password):
+        return True
+    def get_user(self, email="defaultuser@changedetection.io"):
+        return self
+    def is_authenticated(self):
+
+        return True
+    def is_active(self):
+        return True
+    def is_anonymous(self):
+        return False
+    def get_id(self):
+        return str(self.id)
+
+    def check_password(self, password):
+
+        import hashlib
+        import base64
+
+        # Getting the values back out
+        raw_salt_pass = base64.b64decode(datastore.data['settings']['application']['password'])
+        salt_from_storage = raw_salt_pass[:32]  # 32 is the length of the salt
+
+        # Use the exact same setup you used to generate the key, but this time put in the password to check
+        new_key = hashlib.pbkdf2_hmac(
+            'sha256',
+            password.encode('utf-8'),  # Convert the password to bytes
+            salt_from_storage,
+            100000
+        )
+        new_key =  salt_from_storage + new_key
+
+        return new_key == raw_salt_pass
+
+    pass
+
+def changedetection_app(config=None, datastore_o=None):
+    global datastore
+    datastore = datastore_o
+
+    app.config.update(dict(DEBUG=True))
+    #app.config.update(config or {})
+
+    login_manager = flask_login.LoginManager(app)
+    login_manager.login_view = 'login'
+    app.secret_key = init_app_secret(config['datastore_path'])
+
+    # Setup cors headers to allow all domains
+    # https://flask-cors.readthedocs.io/en/latest/
+    #    CORS(app)
+
+    @login_manager.user_loader
+    def user_loader(email):
+        user = User()
+        user.get_user(email)
+        return user
+
+    @login_manager.unauthorized_handler
+    def unauthorized_handler():
+        # @todo validate its a URL of this host and use that
+        return redirect(url_for('login', next=url_for('index')))
+
+    @app.route('/logout')
+    def logout():
+        flask_login.logout_user()
+        return redirect(url_for('index'))
+
+    # https://github.com/pallets/flask/blob/93dd1709d05a1cf0e886df6223377bdab3b077fb/examples/tutorial/flaskr/__init__.py#L39
+    # You can divide up the stuff like this
+    @app.route('/login', methods=['GET', 'POST'])
+    def login():
+
+        if not datastore.data['settings']['application']['password']:
+            flash("Login not required, no password enabled.", "notice")
+            return redirect(url_for('index'))
+
+        if request.method == 'GET':
+            output = render_template("login.html")
+            return output
+
+        user = User()
+        user.id = "defaultuser@changedetection.io"
+
+        password = request.form.get('password')
+
+        if (user.check_password(password)):
+            flask_login.login_user(user, remember=True)
+            next = request.args.get('next')
+            #            if not is_safe_url(next):
+            #                return flask.abort(400)
+            return redirect(next or url_for('index'))
+
+        else:
+            flash('Incorrect password', 'error')
+
+        return redirect(url_for('login'))
+
+    @app.before_request
+    def do_something_whenever_a_request_comes_in():
+        # Disable password  loginif there is not one set
+        app.config['LOGIN_DISABLED'] = datastore.data['settings']['application']['password'] == False
+
+    @app.route("/", methods=['GET'])
+    @login_required
+    def index():
+        limit_tag = request.args.get('tag')
+
+        pause_uuid = request.args.get('pause')
+
+        if pause_uuid:
+            try:
+                datastore.data['watching'][pause_uuid]['paused'] ^= True
+                datastore.needs_write = True
+
+                return redirect(url_for('index', tag = limit_tag))
+            except KeyError:
+                pass
+
+
+        # Sort by last_changed and add the uuid which is usually the key..
+        sorted_watches = []
+        for uuid, watch in datastore.data['watching'].items():
+
+            if limit_tag != None:
+                # Support for comma separated list of tags.
+                for tag_in_watch in watch['tag'].split(','):
+                    tag_in_watch = tag_in_watch.strip()
+                    if tag_in_watch == limit_tag:
+                        watch['uuid'] = uuid
+                        sorted_watches.append(watch)
+
+            else:
+                watch['uuid'] = uuid
+                sorted_watches.append(watch)
+
+        sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)
+
+        existing_tags = datastore.get_all_tags()
+        rss = request.args.get('rss')
+
+        if rss:
+            fg = FeedGenerator()
+            fg.title('changedetection.io')
+            fg.description('Feed description')
+            fg.link(href='https://changedetection.io')
+
+            for watch in sorted_watches:
+                if not watch['viewed']:
+                    fe = fg.add_entry()
+                    fe.title(watch['url'])
+                    fe.link(href=watch['url'])
+                    fe.description(watch['url'])
+                    fe.guid(watch['uuid'], permalink=False)
+                    dt = datetime.datetime.fromtimestamp(int(watch['newest_history_key']))
+                    dt = dt.replace(tzinfo=pytz.UTC)
+                    fe.pubDate(dt)
+
+            response = make_response(fg.rss_str())
+            response.headers.set('Content-Type', 'application/rss+xml')
+            return response
+
+        else:
+            output = render_template("watch-overview.html",
+                                     watches=sorted_watches,
+                                     tags=existing_tags,
+                                     active_tag=limit_tag,
+                                     has_unviewed=datastore.data['has_unviewed'])
+
+        return output
+
+    @app.route("/scrub", methods=['GET', 'POST'])
+    @login_required
+    def scrub_page():
+
+        import re
+
+        if request.method == 'POST':
+            confirmtext = request.form.get('confirmtext')
+            limit_date = request.form.get('limit_date')
+
+            try:
+                limit_date = limit_date.replace('T', ' ')
+                # I noticed chrome will show '/' but actually submit '-'
+                limit_date = limit_date.replace('-', '/')
+                # In the case that :ss seconds are supplied
+                limit_date = re.sub('(\d\d:\d\d)(:\d\d)', '\\1', limit_date)
+
+                str_to_dt = datetime.datetime.strptime(limit_date, '%Y/%m/%d %H:%M')
+                limit_timestamp = int(str_to_dt.timestamp())
+
+                if limit_timestamp > time.time():
+                    flash("Timestamp is in the future, cannot continue.", 'error')
+                    return redirect(url_for('scrub_page'))
+
+            except ValueError:
+                flash('Incorrect date format, cannot continue.', 'error')
+                return redirect(url_for('scrub_page'))
+
+            if confirmtext == 'scrub':
+                changes_removed = 0
+                for uuid, watch in datastore.data['watching'].items():
+                    if limit_timestamp:
+                        changes_removed += datastore.scrub_watch(uuid, limit_timestamp=limit_timestamp)
+                    else:
+                        changes_removed += datastore.scrub_watch(uuid)
+
+                flash("Cleared snapshot history ({} snapshots removed)".format(changes_removed))
+            else:
+                flash('Incorrect confirmation text.', 'error')
+
+            return redirect(url_for('index'))
+
+        output =  render_template("scrub.html")
+        return output
+
+
+    # If they edited an existing watch, we need to know to reset the current/previous md5 to include
+    # the excluded text.
+    def get_current_checksum_include_ignore_text(uuid):
+
+        import hashlib
+        from backend import fetch_site_status
+
+        # Get the most recent one
+        newest_history_key = datastore.get_val(uuid, 'newest_history_key')
+
+        # 0 means that theres only one, so that there should be no 'unviewed' history availabe
+        if newest_history_key == 0:
+            newest_history_key = list(datastore.data['watching'][uuid]['history'].keys())[0]
+
+        if newest_history_key:
+            with open(datastore.data['watching'][uuid]['history'][newest_history_key],
+                      encoding='utf-8') as file:
+                raw_content = file.read()
+
+                handler = fetch_site_status.perform_site_check(datastore=datastore)
+                stripped_content = handler.strip_ignore_text(raw_content,
+                                                             datastore.data['watching'][uuid]['ignore_text'])
+
+                checksum = hashlib.md5(stripped_content).hexdigest()
+                return checksum
+
+        return datastore.data['watching'][uuid]['previous_md5']
+
+
+    @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
+    @login_required
+    def edit_page(uuid):
+        from backend import forms
+        form = forms.watchForm(request.form)
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        if request.method == 'GET':
+            if not uuid in datastore.data['watching']:
+                flash("No watch with the UUID %s found." % (uuid), "error")
+                return redirect(url_for('index'))
+
+            populate_form_from_watch(form, datastore.data['watching'][uuid])
+
+        if request.method == 'POST' and form.validate():
+
+            # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
+            if form.minutes_between_check.data == datastore.data['settings']['requests']['minutes_between_check']:
+                form.minutes_between_check.data = None
+
+            update_obj = {'url': form.url.data.strip(),
+                          'minutes_between_check': form.minutes_between_check.data,
+                          'tag': form.tag.data.strip(),
+                          'title': form.title.data.strip(),
+                          'headers': form.headers.data
+                          }
+
+            # Notification URLs
+            datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data
+
+            # Ignore text
+            form_ignore_text = form.ignore_text.data
+            datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
+
+            # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
+            if form_ignore_text:
+                if len(datastore.data['watching'][uuid]['history']):
+                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
+
+
+            datastore.data['watching'][uuid]['css_filter'] = form.css_filter.data.strip()
+
+            # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
+            if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']:
+                if len(datastore.data['watching'][uuid]['history']):
+                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
+
+
+            datastore.data['watching'][uuid].update(update_obj)
+            datastore.needs_write = True
+            flash("Updated watch.")
+
+            # Queue the watch for immediate recheck
+            update_q.put(uuid)
+
+            if form.trigger_check.data:
+                n_object = {'watch_url': form.url.data.strip(),
+                            'notification_urls': form.notification_urls.data}
+                notification_q.put(n_object)
+
+                flash('Notifications queued.')
+
+            return redirect(url_for('index'))
+
+        else:
+            if request.method == 'POST' and not form.validate():
+                flash("An error occurred, please see below.", "error")
+
+            # Re #110 offer the default minutes
+            using_default_minutes = False
+            if form.minutes_between_check.data == None:
+                form.minutes_between_check.data = datastore.data['settings']['requests']['minutes_between_check']
+                using_default_minutes = True
+
+            output = render_template("edit.html",
+                                     uuid=uuid,
+                                     watch=datastore.data['watching'][uuid],
+                                     form=form,
+                                     using_default_minutes=using_default_minutes
+                                     )
+
+        return output
+
+    @app.route("/settings", methods=['GET', "POST"])
+    @login_required
+    def settings_page():
+
+        from backend import forms
+        form = forms.globalSettingsForm(request.form)
+
+        if request.method == 'GET':
+            form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check'])
+            form.notification_urls.data = datastore.data['settings']['application']['notification_urls']
+            form.extract_title_as_title.data = datastore.data['settings']['application']['extract_title_as_title']
+
+            # Password unset is a GET
+            if request.values.get('removepassword') == 'true':
+                from pathlib import Path
+                datastore.data['settings']['application']['password'] = False
+                flash("Password protection removed.", 'notice')
+                flask_login.logout_user()
+
+        if request.method == 'POST' and form.validate():
+
+            datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
+            datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data
+            datastore.data['settings']['application']['extract_title_as_title'] = form.extract_title_as_title.data
+
+            if len(form.notification_urls.data):
+                import apprise
+                apobj = apprise.Apprise()
+                apobj.debug = True
+
+                # Add each notification
+                for n in datastore.data['settings']['application']['notification_urls']:
+                    apobj.add(n)
+                outcome = apobj.notify(
+                    body='Hello from the worlds best and simplest web page change detection and monitoring service!',
+                    title='Changedetection.io Notification Test',
+                )
+
+                if outcome:
+                    flash("{} Notification URLs reached.".format(len(form.notification_urls.data)), "notice")
+                else:
+                    flash("One or more Notification URLs failed", 'error')
+
+
+            datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
+            datastore.needs_write = True
+
+            if form.trigger_check.data:
+                n_object = {'watch_url': "Test from changedetection.io!",
+                            'notification_urls': form.notification_urls.data}
+                notification_q.put(n_object)
+                flash('Notifications queued.')
+
+            if form.password.encrypted_password:
+                datastore.data['settings']['application']['password'] = form.password.encrypted_password
+                flash("Password protection enabled.", 'notice')
+                flask_login.logout_user()
+                return redirect(url_for('index'))
+
+            flash("Settings updated.")
+
+        if request.method == 'POST' and not form.validate():
+            flash("An error occurred, please see below.", "error")
+
+        output = render_template("settings.html", form=form)
+        return output
+
+    @app.route("/import", methods=['GET', "POST"])
+    @login_required
+    def import_page():
+        import validators
+        remaining_urls = []
+
+        good = 0
+
+        if request.method == 'POST':
+            urls = request.values.get('urls').split("\n")
+            for url in urls:
+                url = url.strip()
+                if len(url) and validators.url(url):
+                    new_uuid = datastore.add_watch(url=url.strip(), tag="")
+                    # Straight into the queue.
+                    update_q.put(new_uuid)
+                    good += 1
+                else:
+                    if len(url):
+                        remaining_urls.append(url)
+
+            flash("{} Imported, {} Skipped.".format(good, len(remaining_urls)))
+
+            if len(remaining_urls) == 0:
+                # Looking good, redirect to index.
+                return redirect(url_for('index'))
+
+        # Could be some remaining, or we could be on GET
+        output = render_template("import.html",
+                                 remaining="\n".join(remaining_urls)
+                                 )
+        return output
+
+    # Clear all statuses, so we do not see the 'unviewed' class
+    @app.route("/api/mark-all-viewed", methods=['GET'])
+    @login_required
+    def mark_all_viewed():
+
+        # Save the current newest history as the most recently viewed
+        for watch_uuid, watch in datastore.data['watching'].items():
+            datastore.set_last_viewed(watch_uuid, watch['newest_history_key'])
+
+        flash("Cleared all statuses.")
+        return redirect(url_for('index'))
+
+    @app.route("/diff/<string:uuid>", methods=['GET'])
+    @login_required
+    def diff_history_page(uuid):
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        extra_stylesheets = ['/static/styles/diff.css']
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            flash("No history found for the specified link, bad link?", "error")
+            return redirect(url_for('index'))
+
+        dates = list(watch['history'].keys())
+        # Convert to int, sort and back to str again
+        dates = [int(i) for i in dates]
+        dates.sort(reverse=True)
+        dates = [str(i) for i in dates]
+
+        if len(dates) < 2:
+            flash("Not enough saved change detection snapshots to produce a report.", "error")
+            return redirect(url_for('index'))
+
+        # Save the current newest history as the most recently viewed
+        datastore.set_last_viewed(uuid, dates[0])
+
+        newest_file = watch['history'][dates[0]]
+        with open(newest_file, 'r') as f:
+            newest_version_file_contents = f.read()
+
+        previous_version = request.args.get('previous_version')
+
+        try:
+            previous_file = watch['history'][previous_version]
+        except KeyError:
+            # Not present, use a default value, the second one in the sorted list.
+            previous_file = watch['history'][dates[1]]
+
+        with open(previous_file, 'r') as f:
+            previous_version_file_contents = f.read()
+
+        output = render_template("diff.html", watch_a=watch,
+                                 newest=newest_version_file_contents,
+                                 previous=previous_version_file_contents,
+                                 extra_stylesheets=extra_stylesheets,
+                                 versions=dates[1:],
+                                 uuid=uuid,
+                                 newest_version_timestamp=dates[0],
+                                 current_previous_version=str(previous_version),
+                                 current_diff_url=watch['url'])
+
+        return output
+
+    @app.route("/preview/<string:uuid>", methods=['GET'])
+    @login_required
+    def preview_page(uuid):
+
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        extra_stylesheets = ['/static/styles/diff.css']
+
+        try:
+            watch = datastore.data['watching'][uuid]
+        except KeyError:
+            flash("No history found for the specified link, bad link?", "error")
+            return redirect(url_for('index'))
+
+        newest = list(watch['history'].keys())[-1]
+        with open(watch['history'][newest], 'r') as f:
+            content = f.readlines()
+
+        output = render_template("preview.html",
+                                 content=content,
+                                 extra_stylesheets=extra_stylesheets,
+                                 current_diff_url=watch['url'],
+                                 uuid=uuid)
+        return output
+
+
+    @app.route("/favicon.ico", methods=['GET'])
+    def favicon():
+        return send_from_directory("/app/static/images", filename="favicon.ico")
+
+    # We're good but backups are even better!
+    @app.route("/backup", methods=['GET'])
+    @login_required
+    def get_backup():
+
+        import zipfile
+        from pathlib import Path
+
+        # Remove any existing backup file, for now we just keep one file
+        for previous_backup_filename in Path(app.config['datastore_path']).rglob('changedetection-backup-*.zip'):
+            os.unlink(previous_backup_filename)
+
+        # create a ZipFile object
+        backupname = "changedetection-backup-{}.zip".format(int(time.time()))
+
+        # We only care about UUIDS from the current index file
+        uuids = list(datastore.data['watching'].keys())
+        backup_filepath = os.path.join(app.config['datastore_path'], backupname)
+
+        with zipfile.ZipFile(backup_filepath, "w",
+                             compression=zipfile.ZIP_DEFLATED,
+                             compresslevel=8) as zipObj:
+
+            # Be sure we're written fresh
+            datastore.sync_to_json()
+
+            # Add the index
+            zipObj.write(os.path.join(app.config['datastore_path'], "url-watches.json"), arcname="url-watches.json")
+
+            # Add the flask app secret
+            zipObj.write(os.path.join(app.config['datastore_path'], "secret.txt"), arcname="secret.txt")
+
+            # Add any snapshot data we find, use the full path to access the file, but make the file 'relative' in the Zip.
+            for txt_file_path in Path(app.config['datastore_path']).rglob('*.txt'):
+                parent_p = txt_file_path.parent
+                if parent_p.name in uuids:
+                    zipObj.write(txt_file_path,
+                                 arcname=str(txt_file_path).replace(app.config['datastore_path'], ''),
+                                 compress_type=zipfile.ZIP_DEFLATED,
+                                 compresslevel=8)
+
+            # Create a list file with just the URLs, so it's easier to port somewhere else in the future
+            list_file = os.path.join(app.config['datastore_path'], "url-list.txt")
+            with open(list_file, "w") as f:
+                for uuid in datastore.data['watching']:
+                    url = datastore.data['watching'][uuid]['url']
+                    f.write("{}\r\n".format(url))
+
+            # Add it to the Zip
+            zipObj.write(list_file,
+                         arcname="url-list.txt",
+                         compress_type=zipfile.ZIP_DEFLATED,
+                         compresslevel=8)
+
+        return send_from_directory(app.config['datastore_path'], backupname, as_attachment=True)
+
+    @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
+    def static_content(group, filename):
+        # These files should be in our subdirectory
+        full_path = os.path.realpath(__file__)
+        p = os.path.dirname(full_path)
+
+        try:
+            return send_from_directory("{}/static/{}".format(p, group), filename=filename)
+        except FileNotFoundError:
+            abort(404)
+
+    @app.route("/api/add", methods=['POST'])
+    @login_required
+    def api_watch_add():
+
+        url = request.form.get('url').strip()
+        if datastore.url_exists(url):
+            flash('The URL {} already exists'.format(url), "error")
+            return redirect(url_for('index'))
+
+        # @todo add_watch should throw a custom Exception for validation etc
+        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
+        # Straight into the queue.
+        update_q.put(new_uuid)
+
+        flash("Watch added.")
+        return redirect(url_for('index'))
+
+    @app.route("/api/delete", methods=['GET'])
+    @login_required
+    def api_delete():
+
+        uuid = request.args.get('uuid')
+        datastore.delete(uuid)
+        flash('Deleted.')
+
+        return redirect(url_for('index'))
+
+    @app.route("/api/checknow", methods=['GET'])
+    @login_required
+    def api_watch_checknow():
+
+        tag = request.args.get('tag')
+        uuid = request.args.get('uuid')
+        i = 0
+
+        running_uuids = []
+        for t in running_update_threads:
+            running_uuids.append(t.current_uuid)
+
+        # @todo check thread is running and skip
+
+        if uuid:
+            if uuid not in running_uuids:
+                update_q.put(uuid)
+            i = 1
+
+        elif tag != None:
+            # Items that have this current tag
+            for watch_uuid, watch in datastore.data['watching'].items():
+                if (tag != None and tag in watch['tag']):
+                    if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
+                        update_q.put(watch_uuid)
+                        i += 1
+
+        else:
+            # No tag, no uuid, add everything.
+            for watch_uuid, watch in datastore.data['watching'].items():
+
+                if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
+                    update_q.put(watch_uuid)
+                    i += 1
+        flash("{} watches are rechecking.".format(i))
+        return redirect(url_for('index', tag=tag))
+
+    # @todo handle ctrl break
+    ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
+
+    threading.Thread(target=notification_runner).start()
+
+    # Check for new release version
+    threading.Thread(target=check_for_new_version).start()
+    return app
+
+
+# Check for new version and anonymous stats
+def check_for_new_version():
+    import requests
+
+    import urllib3
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+    while not app.config.exit.is_set():
+        try:
+            r = requests.post("https://changedetection.io/check-ver.php",
+                              data={'version': datastore.data['version_tag'],
+                                    'app_guid': datastore.data['app_guid'],
+                                    'watch_count': len(datastore.data['watching'])
+                                    },
+
+                              verify=False)
+        except:
+            pass
+
+        try:
+            if "new_version" in r.text:
+                app.config['NEW_VERSION_AVAILABLE'] = True
+        except:
+            pass
+
+        # Check daily
+        app.config.exit.wait(86400)
+
+def notification_runner():
+
+    while not app.config.exit.is_set():
+        try:
+            # At the moment only one thread runs (single runner)
+            n_object = notification_q.get(block=False)
+        except queue.Empty:
+            time.sleep(1)
+            pass
+
+        else:
+            import apprise
+
+            # Create an Apprise instance
+            try:
+                apobj = apprise.Apprise()
+                for url in n_object['notification_urls']:
+                    apobj.add(url.strip())
+
+                n_body = n_object['watch_url']
+
+                # 65 - Append URL of instance to the notification if it is set.
+                base_url = os.getenv('BASE_URL')
+                if base_url != None:
+                    n_body += "\n" + base_url
+
+                apobj.notify(
+                    body=n_body,
+                    # @todo This should be configurable.
+                    title="ChangeDetection.io Notification - {}".format(n_object['watch_url'])
+                )
+
+            except Exception as e:
+                print("Watch URL: {}  Error {}".format(n_object['watch_url'],e))
+
+
+# Thread runner to check every minute, look for new watches to feed into the Queue.
+def ticker_thread_check_time_launch_checks():
+    from backend import update_worker
+
+    # Spin up Workers.
+    for _ in range(datastore.data['settings']['requests']['workers']):
+        new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
+        running_update_threads.append(new_worker)
+        new_worker.start()
+
+    while not app.config.exit.is_set():
+
+        # Get a list of watches by UUID that are currently fetching data
+        running_uuids = []
+        for t in running_update_threads:
+            if t.current_uuid:
+                running_uuids.append(t.current_uuid)
+
+        # Check for watches outside of the time threshold to put in the thread queue.
+        for uuid, watch in datastore.data['watching'].items():
+
+            # If they supplied an individual entry minutes to threshold.
+            if 'minutes_between_check' in watch and watch['minutes_between_check'] is not None:
+                max_time = watch['minutes_between_check'] * 60
+            else:
+                # Default system wide.
+                max_time = datastore.data['settings']['requests']['minutes_between_check'] * 60
+
+            threshold = time.time() - max_time
+
+            # Yeah, put it in the queue, it's more than time.
+            if not watch['paused'] and watch['last_checked'] <= threshold:
+                if not uuid in running_uuids and uuid not in update_q.queue:
+                    update_q.put(uuid)
+
+        # Wait a few seconds before checking the list again
+        time.sleep(3)
+
+        # Should be low so we can break this out in testing
+        app.config.exit.wait(1)

backend/backend.py

@@ -1,365 +0,0 @@
-#!/usr/bin/python3
-
-
-# @todo logging
-# @todo sort by last_changed
-# @todo extra options for url like , verify=False etc.
-# @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option?
-# @todo maybe a button to reset all 'last-changed'.. so you can see it clearly when something happens since your last visit
-# @todo option for interval day/6 hour/etc
-# @todo on change detected, config for calling some API
-# @todo make tables responsive!
-# @todo fetch title into json
-# https://distill.io/features
-# proxy per check
-#i
-import json
-import eventlet
-import eventlet.wsgi
-
-import time
-import os
-import getopt
-import sys
-import datetime
-import timeago
-
-import threading
-
-from flask import Flask, render_template, request, send_file, send_from_directory, safe_join, abort, redirect, url_for
-
-# Local
-import store
-import fetch_site_status
-
-ticker_thread = None
-
-datastore = store.ChangeDetectionStore()
-messages = []
-extra_stylesheets = []
-running_update_threads = {}
-
-app = Flask(__name__, static_url_path='/static')
-app.config['STATIC_RESOURCES'] = "/app/static"
-
-# app.config['SECRET_KEY'] = 'secret!'
-
-# Disables caching of the templates
-app.config['TEMPLATES_AUTO_RELOAD'] = True
-
-
-# We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
-# running or something similar.
-@app.template_filter('format_last_checked_time')
-def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
-    global running_update_threads
-    if watch_obj['uuid'] in running_update_threads:
-        if running_update_threads[watch_obj['uuid']].is_alive():
-            return "Checking now.."
-
-    if watch_obj['last_checked'] == 0:
-        return 'Not yet'
-
-    return timeago.format(int(watch_obj['last_checked']), time.time())
-
-
-# @app.context_processor
-# def timeago():
-#    def _timeago(lower_time, now):
-#        return timeago.format(lower_time, now)
-#    return dict(timeago=_timeago)
-
-@app.template_filter('format_timestamp_timeago')
-def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
-    if timestamp == 0:
-        return 'Not yet'
-    return timeago.format(timestamp, time.time())
-    # return timeago.format(timestamp, time.time())
-    # return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
-
-
-@app.route("/", methods=['GET'])
-def main_page():
-    global messages
-
-    limit_tag = request.args.get('tag')
-
-    # Sort by last_changed and add the uuid which is usually the key..
-    sorted_watches = []
-    for uuid, watch in datastore.data['watching'].items():
-        if limit_tag != None:
-            # Support for comma separated list of tags.
-            for tag_in_watch in watch['tag'].split(','):
-                tag_in_watch = tag_in_watch.strip()
-                if tag_in_watch == limit_tag:
-                    watch['uuid'] = uuid
-                    sorted_watches.append(watch)
-
-        else:
-            watch['uuid'] = uuid
-            sorted_watches.append(watch)
-
-    sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)
-
-    existing_tags = datastore.get_all_tags()
-    output = render_template("watch-overview.html",
-                             watches=sorted_watches,
-                             messages=messages,
-                             tags=existing_tags,
-                             active_tag=limit_tag)
-
-    # Show messages but once.
-    messages = []
-    return output
-
-
-@app.route("/edit", methods=['GET'])
-def edit_page():
-    global messages
-
-    uuid = request.args.get('uuid')
-    output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], messages=messages)
-    return output
-
-
-@app.route("/settings", methods=['GET', "POST"])
-def settings_page():
-    global messages
-    if request.method == 'POST':
-        try:
-            minutes = int(request.values.get('minutes').strip())
-        except ValueError:
-            messages.append({'class': 'error', 'message': "Invalid value given, use an integer."})
-
-        else:
-            if minutes >= 5 and minutes <= 600:
-                datastore.data['settings']['requests']['minutes_between_check'] = minutes
-                datastore.needs_write = True
-
-                messages.append({'class': 'ok', 'message': "Updated"})
-            else:
-                messages.append({'class': 'error', 'message': "Must be equal to or greater than 5 and less than 600 minutes"})
-
-    output = render_template("settings.html", messages=messages, minutes=datastore.data['settings']['requests']['minutes_between_check'])
-    messages =[]
-
-    return output
-
-@app.route("/import", methods=['GET', "POST"])
-def import_page():
-    import validators
-    global messages
-    remaining_urls=[]
-
-    good = 0
-
-    if request.method == 'POST':
-        urls = request.values.get('urls').split("\n")
-        for url in urls:
-            url = url.strip()
-            if len(url) and validators.url(url):
-                datastore.add_watch(url=url.strip(), tag="")
-                good += 1
-            else:
-                if len(url):
-                    remaining_urls.append(url)
-
-        messages.append({'class': 'ok', 'message': "{} Imported, {} Skipped.".format(good, len(remaining_urls))})
-
-        launch_checks()
-
-    output = render_template("import.html",
-                             messages=messages,
-                             remaining="\n".join(remaining_urls)
-                             )
-    messages = []
-    return output
-
-
-@app.route("/diff/<string:uuid>", methods=['GET'])
-def diff_history_page(uuid):
-    global messages
-    global extra_stylesheets
-    extra_stylesheets.append('/static/css/diff.css')
-
-    watch = datastore.data['watching'][uuid]
-
-    dates = list(watch['history'].keys())
-    dates = [int(i) for i in dates]
-    dates.sort(reverse=True)
-
-    left_file_contents = right_file_contents = ""
-    l_file = watch['history'][str(dates[-1])]
-    with open(l_file, 'r') as f:
-        left_file_contents = f.read()
-
-    r_file  = watch['history'][str(dates[-2])]
-    with open(r_file, 'r') as f:
-        right_file_contents = f.read()
-
-    output = render_template("diff.html", watch_a=watch, messages=messages, left=left_file_contents,
-                             right=right_file_contents, extra_stylesheets=extra_stylesheets)
-    return output
-
-@app.route("/favicon.ico", methods=['GET'])
-def favicon():
-    return send_from_directory("/app/static/images", filename="favicon.ico")
-
-
-@app.route("/static/<string:group>/<string:filename>", methods=['GET'])
-def static_content(group, filename):
-    try:
-        return send_from_directory("/app/static/{}".format(group), filename=filename)
-    except FileNotFoundError:
-        abort(404)
-
-
-@app.route("/api/add", methods=['POST'])
-def api_watch_add():
-    global messages
-
-    # @todo add_watch should throw a custom Exception for validation etc
-    datastore.add_watch(url=request.form.get('url').strip(), tag=request.form.get('tag').strip())
-    messages.append({'class': 'ok', 'message': 'Watch added.'})
-    launch_checks()
-    return redirect(url_for('main_page'))
-
-
-@app.route("/api/delete", methods=['GET'])
-def api_delete():
-    global messages
-    uuid = request.args.get('uuid')
-    datastore.delete(uuid)
-    messages.append({'class': 'ok', 'message': 'Deleted.'})
-
-    return redirect(url_for('main_page'))
-
-
-@app.route("/api/update", methods=['POST'])
-def api_update():
-    global messages
-    import validators
-
-    uuid = request.args.get('uuid')
-
-    url = request.form.get('url').strip()
-    tag = request.form.get('tag').strip()
-
-    form_headers = request.form.get('headers').strip().split("\n")
-    extra_headers = {}
-    if form_headers:
-        for header in form_headers:
-            if len(header):
-                parts = header.split(':', 1)
-                extra_headers.update({parts[0].strip(): parts[1].strip()})
-
-
-
-    validators.url(url) #@todo switch to prop/attr/observer
-    datastore.data['watching'][uuid].update({'url': url,
-                                             'tag': tag,
-                                             'headers':extra_headers})
-    datastore.needs_write = True
-
-    messages.append({'class': 'ok', 'message': 'Updated watch.'})
-
-    return redirect(url_for('main_page'))
-
-@app.route("/api/checknow", methods=['GET'])
-def api_watch_checknow():
-    global messages
-
-    uuid = request.args.get('uuid')
-
-    running_update_threads[uuid] = fetch_site_status.perform_site_check(uuid=uuid,
-                                                                        datastore=datastore)
-    running_update_threads[uuid].start()
-
-    return redirect(url_for('main_page'))
-
-
-@app.route("/api/recheckall", methods=['GET'])
-def api_watch_recheckall():
-    import fetch_site_status
-
-    global running_update_threads
-    i = 0
-    for uuid, watch in datastore.data['watching'].items():
-        i = i + 1
-
-        running_update_threads[watch['uuid']] = fetch_site_status.perform_site_check(uuid=uuid,
-                                                                                     datastore=datastore)
-        running_update_threads[watch['uuid']].start()
-
-    return "{} triggered recheck of {} watches.".format(i, len(datastore.data['watching']))
-
-
-# Can be used whenever, launch threads that need launching to update the stored information
-def launch_checks():
-    import fetch_site_status
-    global running_update_threads
-
-
-    minutes = datastore.data['settings']['requests']['minutes_between_check']
-    for uuid, watch in datastore.data['watching'].items():
-
-
-        if watch['last_checked'] <= time.time() - (minutes * 60):
-            running_update_threads[watch['uuid']] = fetch_site_status.perform_site_check(uuid=uuid,
-                                                                                         datastore=datastore)
-            running_update_threads[watch['uuid']].start()
-
-
-# Thread runner to check every minute
-def ticker_thread_check_time_launch_checks():
-    while True:
-        launch_checks()
-        time.sleep(60)
-
-# Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON
-# by just running periodically in one thread.
-def save_datastore():
-    while True:
-        if datastore.needs_write:
-            datastore.sync_to_json()
-        time.sleep(5)
-
-def main(argv):
-    ssl_mode = False
-    port = 5000
-
-    try:
-        opts, args = getopt.getopt(argv, "sp:", "purge")
-    except getopt.GetoptError:
-        print('backend.py -s SSL enable -p [port]')
-        sys.exit(2)
-
-    for opt, arg in opts:
-        if opt == '--purge':
-            # Remove history, the actual files you need to delete manually.
-            for uuid, watch in datastore.data['watching'].items():
-                watch.update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': None})
-
-        if opt == '-s':
-            ssl_mode = True
-
-        if opt == '-p':
-            port = arg
-
-    # @todo handle ctrl break
-    ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
-    save_data_thread = threading.Thread(target=save_datastore).start()
-
-    # @todo finalise SSL config, but this should get you in the right direction if you need it.
-    if ssl_mode:
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen(('', port)),
-                                               certfile='cert.pem',
-                                               keyfile='privkey.pem',
-                                               server_side=True), app)
-
-    else:
-        eventlet.wsgi.server(eventlet.listen(('', port)), app)
-
-
-if __name__ == '__main__':
-    main(sys.argv[1:])

backend/dev-docker/Dockerfile

@@ -3,9 +3,7 @@ FROM python:3.8-slim
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1
 
-COPY requirements.txt /tmp/requirements.txt
-RUN pip3 install -r /tmp/requirements.txt
-
+WORKDIR /app
 
 RUN [ ! -d "/datastore" ] && mkdir /datastore
 

backend/dev-docker/requirements.txt

@@ -1,20 +0,0 @@
-aiohttp
-async-timeout
-chardet==2.3.0
-multidict
-python-engineio
-six==1.10.0
-yarl
-flask
-
-eventlet
-requests
-validators
-
-bleach==3.2.1
-html5lib==0.9999999       # via bleach
-timeago
-html2text
-
-# @notes
-# - Dont install socketio, it interferes with flask_socketio

backend/dev-docker/sleep.py

@@ -1,9 +1,7 @@
 import time
-import sys
 
 print ("Sleep loop, you should run your script from the console")
 
 while True: 
     # Wait for 5 seconds
-
-    time.sleep(2)
+    time.sleep(2)

backend/fetch_site_status.py

@@ -1,62 +1,76 @@
-from threading import Thread
 import time
 import requests
 import hashlib
-import os
-import re
-import html2text
-# Not needed due to inscriptis being way better.
-#from urlextract import URLExtract
 from inscriptis import get_text
+import urllib3
+from . import html_tools
 
-# Hmm Polymorphism datastore, thread, etc
-class perform_site_check(Thread):
-    def __init__(self, *args, uuid=False, datastore, **kwargs):
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+# Some common stuff here that can be moved to a base class
+class perform_site_check():
+
+    def __init__(self, *args, datastore, **kwargs):
         super().__init__(*args, **kwargs)
-        self.timestamp = int(time.time())  # used for storage etc too
-        self.uuid = uuid
         self.datastore = datastore
-        self.url = datastore.get_val(uuid, 'url')
-        self.current_md5 = datastore.get_val(uuid, 'previous_md5')
-        self.output_path = "/datastore/{}".format(self.uuid)
 
-    def save_firefox_screenshot(self, uuid, output):
-        # @todo call selenium or whatever
-        return
+    def strip_ignore_text(self, content, list_ignore_text):
+        import re
+        ignore = []
+        ignore_regex = []
+        for k in list_ignore_text:
 
-    def ensure_output_path(self):
+            # Is it a regex?
+            if k[0] == '/':
+                ignore_regex.append(k.strip(" /"))
+            else:
+                ignore.append(k)
 
-        try:
-            os.stat(self.output_path)
-        except:
-            os.mkdir(self.output_path)
+        output = []
+        for line in content.splitlines():
 
-    def save_response_html_output(self, output):
-        # @todo maybe record a history.json, [timestamp, md5, filename]
-        with open("{}/{}.txt".format(self.output_path, self.timestamp), 'w') as f:
-            f.write(output)
-            f.close()
+            # Always ignore blank lines in this mode. (when this function gets called)
+            if len(line.strip()):
+                regex_matches = False
 
-    def save_response_stripped_output(self, output):
-        fname = "{}/{}.stripped.txt".format(self.output_path, self.timestamp)
-        with open(fname, 'w') as f:
-            f.write(output)
-            f.close()
+                # if any of these match, skip
+                for regex in ignore_regex:
+                    try:
+                        if re.search(regex, line, re.IGNORECASE):
+                            regex_matches = True
+                    except Exception as e:
+                        continue
 
-        return fname
+                if not regex_matches and not any(skip_text in line for skip_text in ignore):
+                    output.append(line.encode('utf8'))
 
-    def run(self):
+        return "\n".encode('utf8').join(output)
 
-        extra_headers = self.datastore.get_val(self.uuid, 'headers')
+
+
+    def run(self, uuid):
+        timestamp = int(time.time())  # used for storage etc too
+
+        stripped_text_from_html = False
+        changed_detected = False
+
+        update_obj = {'previous_md5': self.datastore.data['watching'][uuid]['previous_md5'],
+                      'history': {},
+                      "last_checked": timestamp
+                      }
+
+        extra_headers = self.datastore.get_val(uuid, 'headers')
 
         # Tweak the base config with the per-watch ones
-        request_headers = self.datastore.data['settings']['headers'].copy()
+        request_headers = self.datastore.data['settings']['headers']
         request_headers.update(extra_headers)
 
-        print("Checking", self.url)
-        #print(request_headers)
-
-        self.ensure_output_path()
+        # https://github.com/psf/requests/issues/4525
+        # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
+        # do this by accident.
+        if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
+            request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
 
         try:
             timeout = self.datastore.data['settings']['requests']['timeout']
@@ -65,70 +79,73 @@ class perform_site_check(Thread):
             timeout = 15
 
         try:
-            r = requests.get(self.url,
+            url = self.datastore.get_val(uuid, 'url')
+
+            r = requests.get(url,
                              headers=request_headers,
                              timeout=timeout,
                              verify=False)
 
-            stripped_text_from_html = get_text(r.text)
-
-
-            # @todo This should be a config option.
-            # Many websites include junk in the links, trackers, etc.. Since we are really a service all about text changes..
-
-# inscriptis handles this much cleaner, probably not needed..
-#            extractor = URLExtract()
-#            urls = extractor.find_urls(stripped_text_from_html)
-            # Remove the urls, longest first so that we dont end up chewing up bigger links with parts of smaller ones.
-#            if urls:
-#                urls.sort(key=len, reverse=True)
-#                for url in urls:
-#                    # Sometimes URLExtract will consider something like 'foobar.com' as a link when that was just text.
-#                    if "://" in url:
-#                        # print ("Stripping link", url)
-#                        stripped_text_from_html = stripped_text_from_html.replace(url, '')
+            html = r.text
 
+            # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+            css_filter_rule = self.datastore.data['watching'][uuid]['css_filter']
+            if css_filter_rule and len(css_filter_rule.strip()):
+                html = html_tools.css_filter(css_filter=css_filter_rule, html_content=r.content)
 
+            stripped_text_from_html = get_text(html)
 
         # Usually from networkIO/requests level
         except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
-            self.datastore.update_watch(self.uuid, 'last_error', str(e))
+            update_obj["last_error"] = str(e)
             print(str(e))
 
         except requests.exceptions.MissingSchema:
-            print("Skipping {} due to missing schema/bad url".format(self.uuid))
+            print("Skipping {} due to missing schema/bad url".format(uuid))
 
         # Usually from html2text level
-        except UnicodeDecodeError as e:
-            self.datastore.update_watch(self.uuid, 'last_error', str(e))
+        except Exception as e:
+            #        except UnicodeDecodeError as e:
+            update_obj["last_error"] = str(e)
             print(str(e))
             # figure out how to deal with this cleaner..
             # 'utf-8' codec can't decode byte 0xe9 in position 480: invalid continuation byte
 
+
         else:
+            # We rely on the actual text in the html output.. many sites have random script vars etc,
+            # in the future we'll implement other mechanisms.
 
-            # We rely on the actual text in the html output.. many sites have random script vars etc
-            self.datastore.update_watch(self.uuid, 'last_error', False)
-            self.datastore.update_watch(self.uuid, 'last_check_status', r.status_code)
+            update_obj["last_check_status"] = r.status_code
+            update_obj["last_error"] = False
 
-            fetched_md5 = hashlib.md5(stripped_text_from_html.encode('utf-8')).hexdigest()
+            if not len(r.text):
+                update_obj["last_error"] = "Empty reply"
 
-            if self.current_md5 != fetched_md5:
+            # If there's text to skip
+            # @todo we could abstract out the get_text() to handle this cleaner
+            if len(self.datastore.data['watching'][uuid]['ignore_text']):
+                content = self.strip_ignore_text(stripped_text_from_html,
+                                                 self.datastore.data['watching'][uuid]['ignore_text'])
+            else:
+                content = stripped_text_from_html.encode('utf8')
 
-                # Dont confuse people by putting last-changed, when it actually just changed from nothing..
-                if self.datastore.get_val(self.uuid, 'previous_md5') is not None:
-                    self.datastore.update_watch(self.uuid, 'last_changed', self.timestamp)
+            fetched_md5 = hashlib.md5(content).hexdigest()
 
-                self.datastore.update_watch(self.uuid, 'previous_md5', fetched_md5)
-                self.save_response_html_output(r.text)
-                output_filepath = self.save_response_stripped_output(stripped_text_from_html)
+            # could be None or False depending on JSON type
+            if self.datastore.data['watching'][uuid]['previous_md5'] != fetched_md5:
+                changed_detected = True
 
-                # Update history with the stripped text for future reference, this will also mean we save the first
-                # attempt because 'self.current_md5 != fetched_md5'  (current_md5 will be None when not run)
-                # need to learn more about attr/setters/getters
-                history = self.datastore.get_val(self.uuid, 'history')
-                history.update(dict([(str(self.timestamp), output_filepath)]))
-                self.datastore.update_watch(self.uuid, 'history', history)
+                # Don't confuse people by updating as last-changed, when it actually just changed from None..
+                if self.datastore.get_val(uuid, 'previous_md5'):
+                    update_obj["last_changed"] = timestamp
 
-        self.datastore.update_watch(self.uuid, 'last_checked', int(time.time()))
-        pass
+                update_obj["previous_md5"] = fetched_md5
+
+            # Extract title as title
+            if self.datastore.data['settings']['application']['extract_title_as_title']:
+                if not self.datastore.data['watching'][uuid]['title'] or not len(self.datastore.data['watching'][uuid]['title']):
+                    update_obj['title'] = html_tools.extract_element(find='title', html_content=html)
+
+
+        return changed_detected, update_obj, stripped_text_from_html

backend/forms.py

@@ -0,0 +1,132 @@
+from wtforms import Form, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \
+    Field
+from wtforms import widgets
+from wtforms.validators import ValidationError
+from wtforms.fields import html5
+
+
+class StringListField(StringField):
+    widget = widgets.TextArea()
+
+    def _value(self):
+        if self.data:
+            return "\r\n".join(self.data)
+        else:
+            return u''
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            # Remove empty strings
+            cleaned = list(filter(None, valuelist[0].split("\n")))
+            self.data = [x.strip() for x in cleaned]
+            p = 1
+        else:
+            self.data = []
+
+
+
+class SaltyPasswordField(StringField):
+    widget = widgets.PasswordInput()
+    encrypted_password = ""
+
+    def build_password(self, password):
+        import hashlib
+        import base64
+        import secrets
+
+        # Make a new salt on every new password and store it with the password
+        salt = secrets.token_bytes(32)
+
+        key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
+        store = base64.b64encode(salt + key).decode('ascii')
+
+        return store
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            # Be really sure it's non-zero in length
+            if len(valuelist[0].strip()) > 0:
+                self.encrypted_password = self.build_password(valuelist[0])
+                self.data = ""
+        else:
+            self.data = False
+
+
+# Separated by  key:value
+class StringDictKeyValue(StringField):
+    widget = widgets.TextArea()
+
+    def _value(self):
+        if self.data:
+            output = u''
+            for k in self.data.keys():
+                output += "{}: {}\r\n".format(k, self.data[k])
+
+            return output
+        else:
+            return u''
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            self.data = {}
+            # Remove empty strings
+            cleaned = list(filter(None, valuelist[0].split("\n")))
+            for s in cleaned:
+                parts = s.strip().split(':')
+                if len(parts) == 2:
+                    self.data.update({parts[0].strip(): parts[1].strip()})
+
+        else:
+            self.data = {}
+
+class ListRegex(object):
+    """
+    Validates that anything that looks like a regex passes as a regex
+    """
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        import re
+
+        for line in field.data:
+            if line[0] == '/' and line[-1] == '/':
+                # Because internally we dont wrap in /
+                line = line.strip('/')
+                try:
+                    re.compile(line)
+                except re.error:
+                    message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
+                    raise ValidationError(message % (line))
+
+
+class watchForm(Form):
+    # https://wtforms.readthedocs.io/en/2.3.x/fields/#module-wtforms.fields.html5
+    # `require_tld` = False is needed even for the test harness "http://localhost:5005.." to run
+
+    url = html5.URLField('URL', [validators.URL(require_tld=False)])
+    tag = StringField('Tag', [validators.Optional(), validators.Length(max=35)])
+    minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
+                                               [validators.Optional(), validators.NumberRange(min=1)])
+    css_filter = StringField('CSS Filter')
+    title = StringField('Title')
+
+    ignore_text = StringListField('Ignore Text', [ListRegex()])
+    notification_urls = StringListField('Notification URL List')
+    headers = StringDictKeyValue('Request Headers')
+    trigger_check = BooleanField('Send test notification on save')
+
+
+class globalSettingsForm(Form):
+
+    password = SaltyPasswordField()
+
+    minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
+                                               [validators.NumberRange(min=1)])
+
+    notification_urls = StringListField('Notification URL List')
+    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title')
+    trigger_check = BooleanField('Send test notification on save')

backend/html_tools.py

@@ -0,0 +1,26 @@
+from bs4 import BeautifulSoup
+
+
+# Given a CSS Rule, and a blob of HTML, return the blob of HTML that matches
+def css_filter(css_filter, html_content):
+    soup = BeautifulSoup(html_content, "html.parser")
+    html_block = ""
+    for item in soup.select(css_filter, separator=""):
+        html_block += str(item)
+
+    return html_block + "\n"
+
+
+# Extract/find element
+def extract_element(find='title', html_content=''):
+
+    #Re #106, be sure to handle when its not found
+    element_text = None
+
+    soup = BeautifulSoup(html_content, 'html.parser')
+    result = soup.find(find)
+    if result and result.string:
+        element_text = result.string.strip()
+
+    return element_text
+

backend/nocache.py

@@ -1,14 +0,0 @@
-
-from flask import make_response
-from functools import wraps, update_wrapper
-from datetime import datetime
-
-def nocache(view):
-    @wraps(view)
-    def no_cache(*args, **kwargs):
-        response = make_response(view(*args, **kwargs))
-        response.headers['hmm'] = datetime.now()
-
-        return response
-
-    return update_wrapper(no_cache, view)

backend/pytest.ini

@@ -0,0 +1,12 @@
+[pytest]
+addopts = --no-start-live-server --live-server-port=5005
+#testpaths = tests pytest_invenio
+#live_server_scope = function
+
+filterwarnings =
+    ignore::DeprecationWarning:urllib3.*:
+
+; logging options
+log_cli = 1
+log_cli_level = DEBUG
+log_cli_format = %(asctime)s %(name)s: %(levelname)s %(message)s

backend/requirements.txt

@@ -1,20 +0,0 @@
-aiohttp
-async-timeout
-chardet==2.3.0
-multidict
-python-engineio
-six==1.10.0
-yarl
-flask
-
-eventlet
-requests
-validators
-
-bleach==3.2.1
-html5lib==0.9999999       # via bleach
-timeago
-html2text
-
-# @notes
-# - Dont install socketio, it interferes with flask_socketio

backend/run_all_tests.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+
+# live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions
+# and I like to restart the server for each test (and have the test cleanup after each test)
+# merge request welcome :)
+
+
+# exit when any command fails
+set -e
+
+# Re #65 - Ability to include a link back to the installation, in the notification.
+export BASE_URL="https://foobar.com"
+
+find tests/test_*py -type f|while read test_name
+do
+  echo "TEST RUNNING $test_name"
+  pytest $test_name
+done

backend/static/css/diff.css

@@ -1,61 +0,0 @@
-table {
-	table-layout: fixed;
-	width: 100%;
-}
-td {
-	width: 33%;
-	padding: 3px 4px;
-	border: 1px solid transparent;
-	vertical-align: top;
-	font: 1em monospace;
-	text-align: left;
-	white-space: pre-wrap;
-}
-h1 {
-	display: inline;
-	font-size: 100%;
-}
-del {
-	text-decoration: none;
-	color: #b30000;
-	background: #fadad7;
-}
-
-ins {
-	background: #eaf2c2;
-	color: #406619;
-	text-decoration: none;
-}
-
-#result {
-	white-space: pre-wrap;
-}
-
-#settings {
-
-
-	line-height: 2em;
-}
-#settings label {
-	margin-left: 1em;
-}
-
-.source {
-	position: absolute;
-	right: 1%;
-	top: .2em;
-}
-
-@-moz-document url-prefix() {
-	body {
-		height: 99%; /* Hide scroll bar in Firefox */
-	}
-}
-
-#diff-ui {
-background: #fff;
-    padding: 2em;
-    margin: 1em;
-    border-radius: 5px;
-    font-size: 9px;
-}

backend/static/css/styles.css

@@ -1,201 +0,0 @@
-/*
- * -- BASE STYLES --
- * Most of these are inherited from Base, but I want to change a few.
- */
-body {
-  color: #333;
-  background: #262626;
-}
-
-.pure-table-even {
-  background: #fff;
-}
-/* Some styles from https://css-tricks.com/ */
-
-a {
-  text-decoration: none;
-  color: #1b98f8;
-}
-a.github-link {
-  color: #fff;
-}
-
-
-.pure-menu-horizontal {
-  background: #fff;
-  padding: 5px;
-  display: flex;
-  justify-content: space-between;
-  border-bottom: 2px solid #ed5900;
-
-  align-items: center;
-}
-
-section.content {
-
-  padding-top: 5em;
-  padding-bottom: 5em;
-  flex-direction: column;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-}
-
-.pure-table.watch-table td {
-  font-size: 80%;
-}
-
-/* table related */
-.watch-table {
-  width: 100%;
-}
-.watch-tag-list {
-  color: #e70069;
-  white-space: nowrap;
-}
-
-.box {
-  max-width: 80%;
-  flex-direction: column;
-  display: flex;
-  justify-content: center;
-}
-
-.watch-table .error {
-  color: #a00;
-}
-
-.watch-table td {
-  white-space: nowrap;
-}
-
-.watch-table td.title-col {
-  word-break: break-all;
-  white-space: normal;
-}
-.watch-table th {
-  white-space: nowrap;
-}
-
-.watch-table .title-col a[target="_blank"]::after {
-  content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
-  margin: 0 3px 0 5px;
-}
-
-/* hotovo */
-
-
-body:after {
-  content: "";
-  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)
-}
-
-body:after, body:before {
-  display: block;
-  height: 600px;
-  position: absolute;
-  top: 0;
-  left: 0;
-  width: 100%;
-  z-index: -1;
-}
-body::after {
-  opacity: 0.91;
-}
-body::before {
-  content: "";
-  background-image: url(/static/images/gradient-border.png);
-}
-body:before {
-  background-size: cover
-}
-
-body:after, body:before {
-  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
-  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
-}
-
-
-.button-small {
-  font-size: 85%;
-}
-
-
-.fetch-error {
-  padding-top: 1em;
-  font-size: 60%;
-  max-width: 400px;
-  display: block;
-}
-
-.edit-form {
-   background: #fff;
-   padding: 2em;
-   margin: 1em;
-   border-radius: 5px;
-}
-.button-secondary {
-    color: white;
-    border-radius: 4px;
-    text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
-}
-
-.button-success {
-    background: rgb(28, 184, 65);
-    /* this is a green */
-}
-
-.button-tag {
-    background: rgb(99, 99, 99);
-    color: #fff;
-    font-size: 65%;
-    border-bottom-left-radius: initial;
-    border-bottom-right-radius: initial;
-}
-.button-tag.active {
-    background: #9c9c9c;
-    font-weight: bold;
-}
-.button-error {
-    background: rgb(202, 60, 60);
-    /* this is a maroon */
-}
-
-.button-warning {
-    background: rgb(223, 117, 20);
-    /* this is an orange */
-}
-
-.button-secondary {
-    background: rgb(66, 184, 221);
-    /* this is a light blue */
-}
-
-
-.button-cancel {
-    background: rgb(200, 200, 200);
-    /* this is a green */
-}
-
-.messages {
-    padding: 1em;
-    background: rgba(255,255,255,.2);
-    border-radius: 10px;
-    color: #fff;
-    font-weight: bold;
-}
-
-.pure-form label  {
-  font-weight: bold;
-}
-
-#new-watch-form {
-    background: rgba(0,0,0,.05);
-    padding: 1em;
-    border-radius: 10px;
-    margin-bottom: 1em;
-}
-#new-watch-form legend {
-    color: #fff;
-}
-

backend/static/images/Generic_Feed-icon.svg

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg"
+     id="RSSicon"
+     viewBox="0 0 8 8" width="256" height="256">
+
+  <title>RSS feed icon</title>
+
+  <style type="text/css">
+    .button {stroke: none; fill: orange;}
+    .symbol {stroke: none; fill: white;}
+  </style>
+
+  <rect   class="button" width="8" height="8" rx="1.5" />
+  <circle class="symbol" cx="2" cy="6" r="1" />
+  <path   class="symbol" d="m 1,4 a 3,3 0 0 1 3,3 h 1 a 4,4 0 0 0 -4,-4 z" />
+  <path   class="symbol" d="m 1,2 a 5,5 0 0 1 5,5 h 1 a 6,6 0 0 0 -6,-6 z" />
+
+</svg>

backend/static/images/pause.svg

@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   version="1.1"
+   id="Capa_1"
+   x="0px"
+   y="0px"
+   viewBox="0 0 15 14.998326"
+   xml:space="preserve"
+   width="15"
+   height="14.998326"><metadata
+   id="metadata39"><rdf:RDF><cc:Work
+       rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
+         rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
+   id="defs37" />
+<path
+   id="path2"
+   style="fill:#1b98f8;fill-opacity:1;stroke-width:0.0292893"
+   d="M 7.4975161,6.5052867e-4 C 4.549072,-0.04028702 1.7055675,1.8548221 0.58868606,4.5801341 -0.57739762,7.2574642 0.02596981,10.583326 2.069916,12.671949 4.0364753,14.788409 7.2763651,15.56067 9.989207,14.57284 12.801145,13.617602 14.87442,10.855325 14.985833,7.8845744 15.172496,4.9966544 13.49856,2.1100704 10.911002,0.8209349 9.8598067,0.28073592 8.6791261,-0.00114855 7.4975161,6.5052867e-4 Z M 6.5602569,10.251923 c -0.00509,0.507593 -0.5693885,0.488472 -0.9352002,0.468629 -0.3399386,0.0018 -0.8402048,0.07132 -0.9297965,-0.374189 -0.015842,-1.8973128 -0.015872,-3.7979649 0,-5.6952784 0.1334405,-0.5224315 0.7416869,-0.3424086 1.1377562,-0.374189 0.3969969,-0.084515 0.8245634,0.1963256 0.7272405,0.6382917 0,1.7789118 0,3.5578239 0,5.3367357 z m 3.7490371,0 c -0.0051,0.507593 -0.5693888,0.488472 -0.9352005,0.468629 -0.3399386,0.0018 -0.8402048,0.07132 -0.9297965,-0.374189 -0.015842,-1.8973128 -0.015872,-3.7979649 0,-5.6952784 0.1334405,-0.5224315 0.7416869,-0.3424086 1.1377562,-0.374189 0.3969969,-0.084515 0.8245638,0.1963256 0.7272408,0.6382917 0,1.7789118 0,3.5578239 0,5.3367357 z" />
+<g
+   id="g4"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g6"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g8"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g10"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g12"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g14"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g16"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g18"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g20"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g22"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g24"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g26"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g28"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g30"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+<g
+   id="g32"
+   transform="translate(-0.01903604,0.02221043)">
+</g>
+</svg>

backend/static/styles/.gitignore

@@ -0,0 +1 @@
+node_modules

backend/static/styles/diff.css

@@ -0,0 +1,56 @@
+#diff-ui {
+  background: #fff;
+  padding: 2em;
+  margin: 1em;
+  border-radius: 5px;
+  font-size: 9px; }
+  #diff-ui table {
+    table-layout: fixed;
+    width: 100%; }
+  #diff-ui td {
+    padding: 3px 4px;
+    border: 1px solid transparent;
+    vertical-align: top;
+    font: 1em monospace;
+    text-align: left; }
+  #diff-ui pre {
+    white-space: pre-wrap; }
+
+h1 {
+  display: inline;
+  font-size: 100%; }
+
+del {
+  text-decoration: none;
+  color: #b30000;
+  background: #fadad7; }
+
+ins {
+  background: #eaf2c2;
+  color: #406619;
+  text-decoration: none; }
+
+#result {
+  white-space: pre-wrap; }
+
+#settings {
+  background: rgba(0, 0, 0, 0.05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+  color: #fff;
+  font-size: 80%; }
+  #settings label {
+    margin-left: 1em;
+    display: inline-block;
+    font-weight: normal; }
+
+.source {
+  position: absolute;
+  right: 1%;
+  top: .2em; }
+
+@-moz-document url-prefix() {
+  body {
+    height: 99%;
+    /* Hide scroll bar in Firefox */ } }

backend/static/styles/diff.scss

@@ -0,0 +1,68 @@
+#diff-ui {
+
+    background: #fff;
+    padding: 2em;
+    margin: 1em;
+    border-radius: 5px;
+    font-size: 9px;
+
+    table {
+        table-layout: fixed;
+        width: 100%;
+    }
+    td {
+        padding: 3px 4px;
+        border: 1px solid transparent;
+        vertical-align: top;
+        font: 1em monospace;
+        text-align: left;
+    }
+    pre {
+            white-space: pre-wrap;
+    }
+}
+h1 {
+	display: inline;
+	font-size: 100%;
+}
+del {
+	text-decoration: none;
+	color: #b30000;
+	background: #fadad7;
+}
+
+ins {
+	background: #eaf2c2;
+	color: #406619;
+	text-decoration: none;
+}
+
+#result {
+	white-space: pre-wrap;
+}
+
+#settings {
+    background: rgba(0,0,0,.05);
+    padding: 1em;
+    border-radius: 10px;
+    margin-bottom: 1em;
+    color: #fff;
+    font-size: 80%;
+    label {
+	    margin-left: 1em;
+	    display: inline-block;
+	    font-weight: normal;
+    }
+}
+
+.source {
+	position: absolute;
+	right: 1%;
+	top: .2em;
+}
+
+@-moz-document url-prefix() {
+	body {
+		height: 99%; /* Hide scroll bar in Firefox */
+	}
+}

backend/static/styles/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "changedetection.io-theme",
+  "version": "0.0.3",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "scss": "node-sass --watch styles.scss diff.scss -o ."
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "node-sass": "^6.0.0"
+  }
+}

backend/static/styles/styles.css

@@ -0,0 +1,314 @@
+/*
+ * -- BASE STYLES --
+ * Most of these are inherited from Base, but I want to change a few.
+ * npm run scss
+ */
+body {
+  color: #333;
+  background: #262626; }
+
+.pure-table-even {
+  background: #fff; }
+
+/* Some styles from https://css-tricks.com/ */
+a {
+  text-decoration: none;
+  color: #1b98f8; }
+
+a.github-link {
+  color: #fff; }
+
+.pure-menu-horizontal {
+  background: #fff;
+  padding: 5px;
+  display: flex;
+  justify-content: space-between;
+  border-bottom: 2px solid #ed5900;
+  align-items: center; }
+
+section.content {
+  padding-top: 5em;
+  padding-bottom: 5em;
+  flex-direction: column;
+  display: flex;
+  align-items: center;
+  justify-content: center; }
+
+/* table related */
+.watch-table {
+  width: 100%; }
+  .watch-table tr.unviewed {
+    font-weight: bold; }
+  .watch-table .error {
+    color: #a00; }
+  .watch-table td {
+    font-size: 80%;
+    white-space: nowrap; }
+  .watch-table td.title-col {
+    word-break: break-all;
+    white-space: normal; }
+  .watch-table th {
+    white-space: nowrap; }
+  .watch-table .title-col a[target="_blank"]::after, .watch-table .current-diff-url::after {
+    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+    margin: 0 3px 0 5px; }
+
+.watch-tag-list {
+  color: #e70069;
+  white-space: nowrap; }
+
+.box {
+  max-width: 80%;
+  flex-direction: column;
+  display: flex;
+  justify-content: center; }
+
+#post-list-buttons {
+  text-align: right;
+  padding: 0px;
+  margin: 0px; }
+  #post-list-buttons li {
+    display: inline-block; }
+  #post-list-buttons a {
+    border-top-left-radius: initial;
+    border-top-right-radius: initial;
+    border-bottom-left-radius: 5px;
+    border-bottom-right-radius: 5px; }
+
+body:after {
+  content: "";
+  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%); }
+
+body:after, body:before {
+  display: block;
+  height: 600px;
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  z-index: -1; }
+
+body::after {
+  opacity: 0.91; }
+
+body::before {
+  content: "";
+  background-image: url(/static/images/gradient-border.png); }
+
+body:before {
+  background-size: cover; }
+
+body:after, body:before {
+  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
+  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%); }
+
+.button-small {
+  font-size: 85%; }
+
+.fetch-error {
+  padding-top: 1em;
+  font-size: 60%;
+  max-width: 400px;
+  display: block; }
+
+.edit-form {
+  background: #fff;
+  padding: 2em;
+  margin: 1em;
+  border-radius: 5px;
+  min-width: 70%; }
+
+.button-secondary {
+  color: white;
+  border-radius: 4px;
+  text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2); }
+
+.button-success {
+  background: #1cb841;
+  /* this is a green */ }
+
+.button-tag {
+  background: #636363;
+  color: #fff;
+  font-size: 65%;
+  border-bottom-left-radius: initial;
+  border-bottom-right-radius: initial; }
+  .button-tag.active {
+    background: #9c9c9c;
+    font-weight: bold; }
+
+.button-error {
+  background: #ca3c3c;
+  /* this is a maroon */ }
+
+.button-warning {
+  background: #df7514;
+  /* this is an orange */ }
+
+.button-secondary {
+  background: #42b8dd;
+  /* this is a light blue */ }
+
+.button-cancel {
+  background: #c8c8c8;
+  /* this is a green */ }
+
+.messages li {
+  list-style: none;
+  padding: 1em;
+  border-radius: 10px;
+  color: #fff;
+  font-weight: bold; }
+  .messages li.message {
+    background: rgba(255, 255, 255, 0.2); }
+  .messages li.error {
+    background: rgba(255, 1, 1, 0.5); }
+  .messages li.notice {
+    background: rgba(255, 255, 255, 0.5); }
+
+#new-watch-form {
+  background: rgba(0, 0, 0, 0.05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em; }
+
+#new-watch-form legend {
+  color: #fff; }
+
+#new-watch-form input {
+  width: auto !important; }
+
+#diff-col {
+  padding-left: 40px; }
+
+#diff-jump {
+  position: fixed;
+  left: 0px;
+  top: 80px;
+  background: #fff;
+  padding: 10px;
+  border-top-right-radius: 5px;
+  border-bottom-right-radius: 5px;
+  box-shadow: 5px 0 5px -2px #888; }
+
+#diff-jump a {
+  color: #1b98f8;
+  cursor: grabbing;
+  -moz-user-select: none;
+  -webkit-user-select: none;
+  -ms-user-select: none;
+  user-select: none;
+  -o-user-select: none; }
+
+footer {
+  padding: 10px;
+  background: #fff;
+  color: #444;
+  text-align: center; }
+
+#feed-icon {
+  vertical-align: middle; }
+
+#version {
+  position: absolute;
+  top: 80px;
+  right: 0px;
+  font-size: 8px;
+  background: #fff;
+  padding: 10px; }
+
+#new-version-text a {
+  color: #e07171; }
+
+.paused-state.state-False img {
+  opacity: 0.2; }
+
+.paused-state.state-False:hover img {
+  opacity: 0.8; }
+
+.monospaced-textarea textarea {
+  width: 100%;
+  font-family: monospace;
+  white-space: pre;
+  overflow-wrap: normal;
+  overflow-x: scroll; }
+
+.pure-form {
+  /* The input fields with errors */
+  /* The list of errors */ }
+  .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
+    padding-bottom: 1em; }
+    .pure-form .pure-control-group dd, .pure-form .pure-group dd, .pure-form .pure-controls dd {
+      margin: 0px; }
+  .pure-form .error input {
+    background-color: #ffebeb; }
+  .pure-form ul.errors {
+    padding: .5em .6em;
+    border: 1px solid #dd0000;
+    border-radius: 4px;
+    vertical-align: middle;
+    -webkit-box-sizing: border-box;
+    box-sizing: border-box; }
+    .pure-form ul.errors li {
+      margin-left: 1em;
+      color: #dd0000; }
+  .pure-form label {
+    font-weight: bold; }
+  .pure-form input[type=url] {
+    width: 100%; }
+  .pure-form textarea {
+    width: 100%;
+    font-size: 14px; }
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+  .box {
+    max-width: 95%; }
+  .edit-form {
+    padding: 0.5em;
+    margin: 0.5em; }
+  #nav-menu {
+    overflow-x: scroll; } }
+
+/*
+Max width before this PARTICULAR table gets nasty
+This query will take effect for any screen smaller than 760px
+and also iPads specifically.
+*/
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+  .watch-table {
+    /* Force table to not be like tables anymore */
+    /* Force table to not be like tables anymore */
+    /* Hide table headers (but not display: none;, for accessibility) */ }
+    .watch-table thead, .watch-table tbody, .watch-table th, .watch-table td, .watch-table tr {
+      display: block; }
+    .watch-table .last-checked::before {
+      color: #555;
+      content: "Last Checked "; }
+    .watch-table .last-changed::before {
+      color: #555;
+      content: "Last Changed "; }
+    .watch-table td.inline {
+      display: inline-block; }
+    .watch-table thead tr {
+      position: absolute;
+      top: -9999px;
+      left: -9999px; }
+    .watch-table .pure-table td, .watch-table .pure-table th {
+      border: none; }
+    .watch-table td {
+      /* Behave  like a "row" */
+      border: none;
+      border-bottom: 1px solid #eee; }
+      .watch-table td:before {
+        /* Top/left values mimic padding */
+        top: 6px;
+        left: 6px;
+        width: 45%;
+        padding-right: 10px;
+        white-space: nowrap; }
+    .watch-table.pure-table-striped tr {
+      background-color: #fff; }
+    .watch-table.pure-table-striped tr:nth-child(2n-1) {
+      background-color: #eee; }
+    .watch-table.pure-table-striped tr:nth-child(2n-1) td {
+      background-color: inherit; } }

backend/static/styles/styles.scss

@@ -0,0 +1,440 @@
+/*
+ * -- BASE STYLES --
+ * Most of these are inherited from Base, but I want to change a few.
+ * npm run scss
+ */
+body {
+  color: #333;
+  background: #262626;
+}
+
+.pure-table-even {
+  background: #fff;
+}
+
+/* Some styles from https://css-tricks.com/ */
+a {
+  text-decoration: none;
+  color: #1b98f8;
+}
+
+a.github-link {
+  color: #fff;
+}
+
+.pure-menu-horizontal {
+  background: #fff;
+  padding: 5px;
+  display: flex;
+  justify-content: space-between;
+  border-bottom: 2px solid #ed5900;
+  align-items: center;
+}
+
+section.content {
+  padding-top: 5em;
+  padding-bottom: 5em;
+  flex-direction: column;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+/* table related */
+.watch-table {
+  width: 100%;
+
+  tr.unviewed {
+    font-weight: bold;
+  }
+
+  .error {
+    color: #a00;
+  }
+
+  td {
+    font-size: 80%;
+    white-space: nowrap;
+  }
+
+  td.title-col {
+    word-break: break-all;
+    white-space: normal;
+  }
+
+  th {
+    white-space: nowrap;
+  }
+
+  .title-col a[target="_blank"]::after, .current-diff-url::after {
+    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
+    margin: 0 3px 0 5px;
+  }
+}
+
+
+.watch-tag-list {
+  color: #e70069;
+  white-space: nowrap;
+}
+
+.box {
+  max-width: 80%;
+  flex-direction: column;
+  display: flex;
+  justify-content: center;
+}
+
+
+#post-list-buttons {
+  text-align: right;
+  padding: 0px;
+  margin: 0px;
+
+  li {
+    display: inline-block;
+  }
+
+  a {
+    border-top-left-radius: initial;
+    border-top-right-radius: initial;
+    border-bottom-left-radius: 5px;
+    border-bottom-right-radius: 5px;
+  }
+}
+
+
+body:after {
+  content: "";
+  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)
+}
+
+body:after, body:before {
+  display: block;
+  height: 600px;
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  z-index: -1;
+}
+
+body::after {
+  opacity: 0.91;
+}
+
+body::before {
+  content: "";
+  background-image: url(/static/images/gradient-border.png);
+}
+
+body:before {
+  background-size: cover
+}
+
+body:after, body:before {
+  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
+  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
+}
+
+
+.button-small {
+  font-size: 85%;
+}
+
+
+.fetch-error {
+  padding-top: 1em;
+  font-size: 60%;
+  max-width: 400px;
+  display: block;
+}
+
+.edit-form {
+  background: #fff;
+  padding: 2em;
+  margin: 1em;
+  border-radius: 5px;
+  min-width: 70%;
+}
+
+.button-secondary {
+  color: white;
+  border-radius: 4px;
+  text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
+}
+
+.button-success {
+  background: rgb(28, 184, 65);
+  /* this is a green */
+}
+
+.button-tag {
+  background: rgb(99, 99, 99);
+  color: #fff;
+  font-size: 65%;
+  border-bottom-left-radius: initial;
+  border-bottom-right-radius: initial;
+
+  &.active {
+    background: #9c9c9c;
+    font-weight: bold;
+  }
+
+}
+
+.button-error {
+  background: rgb(202, 60, 60);
+  /* this is a maroon */
+}
+
+.button-warning {
+  background: rgb(223, 117, 20);
+  /* this is an orange */
+}
+
+.button-secondary {
+  background: rgb(66, 184, 221);
+  /* this is a light blue */
+}
+
+
+.button-cancel {
+  background: rgb(200, 200, 200);
+  /* this is a green */
+}
+
+.messages {
+    li {
+        list-style: none;
+        padding: 1em;
+        border-radius: 10px;
+        color: #fff;
+        font-weight: bold;
+        &.message {
+            background: rgba(255, 255, 255, .2);
+        }
+        &.error {
+            background: rgba(255, 1, 1, .5);
+        }
+        &.notice {
+            background: rgba(255, 255, 255, .5);
+        }
+    }
+
+}
+
+#new-watch-form {
+  background: rgba(0, 0, 0, .05);
+  padding: 1em;
+  border-radius: 10px;
+  margin-bottom: 1em;
+}
+
+#new-watch-form legend {
+  color: #fff;
+}
+
+#new-watch-form input {
+  width: auto !important;
+}
+
+#diff-col {
+  padding-left: 40px;
+}
+
+#diff-jump {
+  position: fixed;
+  left: 0px;
+  top: 80px;
+  background: #fff;
+  padding: 10px;
+  border-top-right-radius: 5px;
+  border-bottom-right-radius: 5px;
+  box-shadow: 5px 0 5px -2px #888;
+}
+
+#diff-jump a {
+  color: #1b98f8;
+  cursor: grabbing;
+  -moz-user-select: none;
+  -webkit-user-select: none;
+  -ms-user-select: none;
+  user-select: none;
+  -o-user-select: none;
+}
+
+footer {
+  padding: 10px;
+  background: #fff;
+  color: #444;
+  text-align: center;
+}
+
+#feed-icon {
+  vertical-align: middle;
+}
+
+#version {
+  position: absolute;
+  top: 80px;
+  right: 0px;
+  font-size: 8px;
+  background: #fff;
+  padding: 10px;
+}
+
+#new-version-text a {
+  color: #e07171;
+}
+
+.paused-state {
+  &.state-False img {
+    opacity: 0.2;
+  }
+
+  &.state-False:hover img {
+    opacity: 0.8;
+  }
+}
+
+.monospaced-textarea {
+    textarea {
+        width: 100%;
+        font-family: monospace;
+        white-space: pre;
+        overflow-wrap: normal;
+        overflow-x: scroll;
+    }
+}
+
+
+.pure-form {
+    .pure-control-group, .pure-group, .pure-controls {
+        padding-bottom: 1em;
+        dd {
+            margin: 0px;
+        }
+    }
+  /* The input fields with errors */
+  .error {
+    input {
+        background-color: #ffebeb;
+    }
+  }
+   /* The list of errors */
+  ul.errors {
+    padding: .5em .6em;
+    border: 1px solid #dd0000;
+    border-radius: 4px;
+    vertical-align: middle;
+    -webkit-box-sizing: border-box;
+    box-sizing: border-box;
+    li {
+        margin-left: 1em;
+        color: #dd0000;
+    }
+  }
+
+  label {
+    font-weight: bold;
+  }
+
+  input[type=url] {
+    width: 100%;
+  }
+
+  textarea {
+    width: 100%;
+    font-size: 14px;
+  }
+}
+
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+  .box {
+    max-width: 95%
+  }
+  .edit-form {
+    padding: 0.5em;
+    margin: 0.5em;
+  }
+  #nav-menu {
+    overflow-x: scroll;
+  }
+
+}
+
+/*
+Max width before this PARTICULAR table gets nasty
+This query will take effect for any screen smaller than 760px
+and also iPads specifically.
+*/
+@media only screen and (max-width: 760px),
+(min-device-width: 768px) and (max-device-width: 1024px) {
+
+  .watch-table {
+    /* Force table to not be like tables anymore */
+    thead, tbody, th, td, tr {
+      display: block;
+    }
+
+    .last-checked::before {
+      color: #555;
+      content: "Last Checked ";
+    }
+
+    .last-changed::before {
+      color: #555;
+      content: "Last Changed ";
+    }
+
+    /* Force table to not be like tables anymore */
+    td.inline {
+      display: inline-block;
+    }
+
+    /* Hide table headers (but not display: none;, for accessibility) */
+    thead tr {
+      position: absolute;
+      top: -9999px;
+      left: -9999px;
+    }
+
+    .pure-table td, .pure-table th {
+      border: none;
+    }
+
+    td {
+      /* Behave  like a "row" */
+      border: none;
+      border-bottom: 1px solid #eee;
+
+      &:before {
+        /* Top/left values mimic padding */
+        top: 6px;
+        left: 6px;
+        width: 45%;
+        padding-right: 10px;
+        white-space: nowrap;
+      }
+    }
+
+    &.pure-table-striped {
+      tr {
+        background-color: #fff;
+      }
+
+      tr:nth-child(2n-1) {
+        background-color: #eee;
+      }
+
+      tr:nth-child(2n-1) td {
+        background-color: inherit;
+      }
+    }
+
+  }
+}
+
+
+

backend/store.py

@@ -1,50 +1,80 @@
+from os import unlink, path, mkdir
 import json
 import uuid as uuid_builder
-import validators
+from threading import Lock
+from copy import deepcopy
+
+import logging
+import time
+import threading
 
 
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
 # https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
 class ChangeDetectionStore:
+    lock = Lock()
 
-    def __init__(self):
+    def __init__(self, datastore_path="/datastore", include_default_watches=True):
         self.needs_write = False
+        self.datastore_path = datastore_path
+        self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
+        self.stop_thread = False
 
         self.__data = {
-            'note' : "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
+            'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
             'watching': {},
             'settings': {
                 'headers': {
                     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
                     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
-                    'Accept-Encoding': 'gzip, deflate, br',
+                    'Accept-Encoding': 'gzip, deflate',  # No support for brolti in python requests yet.
                     'Accept-Language': 'en-GB,en-US;q=0.9,en;'
                 },
                 'requests': {
-                    'timeout': 15, # Default 15 seconds
-                    'minutes_between_check': 3 * 60 # Default 3 hours
+                    'timeout': 15,  # Default 15 seconds
+                    'minutes_between_check': 3 * 60,  # Default 3 hours
+                    'workers': 10  # Number of threads, lower is better for slow connections
+                },
+                'application': {
+                    'password': False,
+                    'extract_title_as_title': False,
+                    'notification_urls': [] # Apprise URL list
                 }
             }
         }
 
-
         # Base definition for all watchers
         self.generic_definition = {
             'url': None,
             'tag': None,
             'last_checked': 0,
             'last_changed': 0,
+            'paused': False,
+            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
+            'newest_history_key': "",
             'title': None,
-            'previous_md5': None,
+            # Re #110, so then if this is set to None, we know to use the default value instead
+            # Requires setting to None on submit if it's the same as the default
+            'minutes_between_check': None,
+            'previous_md5': "",
             'uuid': str(uuid_builder.uuid4()),
-            'headers' : {}, # Extra headers to send
-            'history' : {} # Dict of timestamp and output stripped filename
+            'headers': {},  # Extra headers to send
+            'history': {},  # Dict of timestamp and output stripped filename
+            'ignore_text': [], # List of text to ignore when calculating the comparison checksum
+            'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise)
+            'css_filter': "",
         }
 
+        if path.isfile('backend/source.txt'):
+            with open('backend/source.txt') as f:
+                # Should be set in Dockerfile to look for /source.txt , this will give us the git commit #
+                # So when someone gives us a backup file to examine, we know exactly what code they were running.
+                self.__data['build_sha'] = f.read()
 
         try:
-            with open('/datastore/url-watches.json') as json_file:
+            # @todo retest with ", encoding='utf-8'"
+            with open(self.json_store_path) as json_file:
                 from_disk = json.load(json_file)
 
                 # @todo isnt there a way todo this dict.update recursively?
@@ -52,6 +82,9 @@ class ChangeDetectionStore:
                 if 'watching' in from_disk:
                     self.__data['watching'].update(from_disk['watching'])
 
+                if 'app_guid' in from_disk:
+                    self.__data['app_guid'] = from_disk['app_guid']
+
                 if 'settings' in from_disk:
                     if 'headers' in from_disk['settings']:
                         self.__data['settings']['headers'].update(from_disk['settings']['headers'])
@@ -59,59 +92,151 @@ class ChangeDetectionStore:
                     if 'requests' in from_disk['settings']:
                         self.__data['settings']['requests'].update(from_disk['settings']['requests'])
 
+                    if 'application' in from_disk['settings']:
+                        self.__data['settings']['application'].update(from_disk['settings']['application'])
 
                 # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
                 # @todo pretty sure theres a python we todo this with an abstracted(?) object!
-                i = 0
-                for uuid, watch in self.data['watching'].items():
-                    _blank = self.generic_definition.copy()
+                for uuid, watch in self.__data['watching'].items():
+                    _blank = deepcopy(self.generic_definition)
                     _blank.update(watch)
                     self.__data['watching'].update({uuid: _blank})
-                    print("Watching:", uuid, _blank['url'])
+                    self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
+                    print("Watching:", uuid, self.__data['watching'][uuid]['url'])
 
         # First time ran, doesnt exist.
         except (FileNotFoundError, json.decoder.JSONDecodeError):
-            print("Creating JSON store")
-            self.add_watch(url='http://www.quotationspage.com/random.php', tag='test')
-            self.add_watch(url='https://news.ycombinator.com/', tag='Tech news')
-            self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
-            self.add_watch(url='https://changedetection.io', tag='Tech news')
+            if include_default_watches:
+                print("Creating JSON store at", self.datastore_path)
 
-            
-#        self.entryVariable.get()
-    def update_watch(self, uuid, val, var):
+                self.add_watch(url='http://www.quotationspage.com/random.php', tag='test')
+                self.add_watch(url='https://news.ycombinator.com/', tag='Tech news')
+                self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
+                self.add_watch(url='https://changedetection.io', tag='Tech news')
+
+        self.__data['version_tag'] = "0.37"
+
+        # Helper to remove password protection
+        password_reset_lockfile = "{}/removepassword.lock".format(self.datastore_path)
+        if path.isfile(password_reset_lockfile):
+            self.__data['settings']['application']['password'] = False
+            unlink(password_reset_lockfile)
+
+        if not 'app_guid' in self.__data:
+            import sys
+            import os
+            if "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ:
+                self.__data['app_guid'] = "test-" + str(uuid_builder.uuid4())
+            else:
+                self.__data['app_guid'] = str(uuid_builder.uuid4())
 
-        self.__data['watching'][uuid].update({val: var})
         self.needs_write = True
 
+        # Finally start the thread that will manage periodic data saves to JSON
+        save_data_thread = threading.Thread(target=self.save_datastore).start()
+
+    # Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0.
+    def get_newest_history_key(self, uuid):
+        if len(self.__data['watching'][uuid]['history']) == 1:
+            return 0
+
+        dates = list(self.__data['watching'][uuid]['history'].keys())
+        # Convert to int, sort and back to str again
+        dates = [int(i) for i in dates]
+        dates.sort(reverse=True)
+        if len(dates):
+            # always keyed as str
+            return str(dates[0])
+
+        return 0
+
+    def set_last_viewed(self, uuid, timestamp):
+        self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
+        self.needs_write = True
+
+    def update_watch(self, uuid, update_obj):
+
+        # Skip if 'paused' state
+        if self.__data['watching'][uuid]['paused']:
+            return
+
+        with self.lock:
+
+            # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
+            for dict_key, d in self.generic_definition.items():
+                if isinstance(d, dict):
+                    if update_obj is not None and dict_key in update_obj:
+                        self.__data['watching'][uuid][dict_key].update(update_obj[dict_key])
+                        del (update_obj[dict_key])
+
+            self.__data['watching'][uuid].update(update_obj)
+            self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
+
+        self.needs_write = True
 
     @property
     def data(self):
+
+        has_unviewed = False
+        for uuid, v in self.__data['watching'].items():
+            self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
+            if int(v['newest_history_key']) <= int(v['last_viewed']):
+                self.__data['watching'][uuid]['viewed'] = True
+
+            else:
+                self.__data['watching'][uuid]['viewed'] = False
+                has_unviewed = True
+
+            # #106 - Be sure this is None on empty string, False, None, etc
+            if not self.__data['watching'][uuid]['title']:
+                self.__data['watching'][uuid]['title'] = None
+
+        self.__data['has_unviewed'] = has_unviewed
+
         return self.__data
 
     def get_all_tags(self):
-        tags=[]
+        tags = []
         for uuid, watch in self.data['watching'].items():
 
             # Support for comma separated list of tags.
             for tag in watch['tag'].split(','):
                 tag = tag.strip()
-                if not tag in tags:
+                if tag not in tags:
                     tags.append(tag)
 
         tags.sort()
         return tags
 
-    def delete(self, uuid):
-        # Probably their should be dict...
-        del(self.__data['watching'][uuid])
-        self.needs_write = True
+    def unlink_history_file(self, path):
+        try:
+            unlink(path)
+        except (FileNotFoundError, IOError):
+            pass
 
+    # Delete a single watch by UUID
+    def delete(self, uuid):
+        with self.lock:
+            if uuid == 'all':
+                self.__data['watching'] = {}
+
+                # GitHub #30 also delete history records
+                for uuid in self.data['watching']:
+                    for path in self.data['watching'][uuid]['history'].values():
+                        self.unlink_history_file(path)
+
+            else:
+                for path in self.data['watching'][uuid]['history'].values():
+                    self.unlink_history_file(path)
+
+                del self.data['watching'][uuid]
+
+            self.needs_write = True
 
     def url_exists(self, url):
 
         # Probably their should be dict...
-        for watch in self.data['watching']:
+        for watch in self.data['watching'].values():
             if watch['url'] == url:
                 return True
 
@@ -121,27 +246,131 @@ class ChangeDetectionStore:
         # Probably their should be dict...
         return self.data['watching'][uuid].get(val)
 
+    # Remove a watchs data but keep the entry (URL etc)
+    def scrub_watch(self, uuid, limit_timestamp = False):
+
+        import hashlib
+        del_timestamps = []
+
+        changes_removed = 0
+
+        for timestamp, path in self.data['watching'][uuid]['history'].items():
+            if not limit_timestamp or (limit_timestamp is not False and int(timestamp) > limit_timestamp):
+                self.unlink_history_file(path)
+                del_timestamps.append(timestamp)
+                changes_removed += 1
+
+                if not limit_timestamp:
+                    self.data['watching'][uuid]['last_checked'] = 0
+                    self.data['watching'][uuid]['last_changed'] = 0
+                    self.data['watching'][uuid]['previous_md5'] = 0
+
+
+        for timestamp in del_timestamps:
+            del self.data['watching'][uuid]['history'][str(timestamp)]
+
+            # If there was a limitstamp, we need to reset some meta data about the entry
+            # This has to happen after we remove the others from the list
+            if limit_timestamp:
+                newest_key = self.get_newest_history_key(uuid)
+                if newest_key:
+                    self.data['watching'][uuid]['last_checked'] = int(newest_key)
+                    # @todo should be the original value if it was less than newest key
+                    self.data['watching'][uuid]['last_changed'] = int(newest_key)
+                    try:
+                        with open(self.data['watching'][uuid]['history'][str(newest_key)], "rb") as fp:
+                            content = fp.read()
+                        self.data['watching'][uuid]['previous_md5'] = hashlib.md5(content).hexdigest()
+                    except (FileNotFoundError, IOError):
+                        self.data['watching'][uuid]['previous_md5'] = False
+                        pass
+
+        self.needs_write = True
+        return changes_removed
+
     def add_watch(self, url, tag):
+        with self.lock:
+            # @todo use a common generic version of this
+            new_uuid = str(uuid_builder.uuid4())
+            _blank = deepcopy(self.generic_definition)
+            _blank.update({
+                'url': url,
+                'tag': tag,
+                'uuid': new_uuid
+            })
 
-        # @todo deal with exception
-        validators.url(url)
+            self.data['watching'][new_uuid] = _blank
 
-        # @todo use a common generic version of this
+        # Get the directory ready
+        output_path = "{}/{}".format(self.datastore_path, new_uuid)
+        try:
+            mkdir(output_path)
+        except FileExistsError:
+            print(output_path, "already exists.")
 
-        _blank = self.generic_definition.copy()
-        _blank.update({
-            'url': url,
-            'tag': tag,
-            'uuid': str(uuid_builder.uuid4())
-        })
+        self.sync_to_json()
+        return new_uuid
 
-        self.data['watching'].update({_blank['uuid']: _blank})
+    # Save some text file to the appropriate path and bump the history
+    # result_obj from fetch_site_status.run()
+    def save_history_text(self, uuid, result_obj, contents):
 
+        output_path = "{}/{}".format(self.datastore_path, uuid)
+        fname = "{}/{}-{}.stripped.txt".format(output_path, result_obj['previous_md5'], str(time.time()))
+        with open(fname, 'w') as f:
+            f.write(contents)
+            f.close()
+
+        # Update history with the stripped text for future reference, this will also mean we save the first
+        # Should always be keyed by string(timestamp)
+        self.update_watch(uuid, {"history": {str(result_obj["last_checked"]): fname}})
+
+        return fname
 
     def sync_to_json(self):
-        print ("Saving index")
-        with open('/datastore/url-watches.json', 'w') as json_file:
-            json.dump(self.data, json_file, indent=4)
-        self.needs_write = False
+        print("Saving..")
+        data ={}
 
-# body of the constructor
+        try:
+            data = deepcopy(self.__data)
+        except RuntimeError:
+            time.sleep(0.5)
+            print ("! Data changed when writing to JSON, trying again..")
+            self.sync_to_json()
+            return
+        else:
+            with open(self.json_store_path, 'w') as json_file:
+                json.dump(data, json_file, indent=4)
+                logging.info("Re-saved index")
+
+            self.needs_write = False
+
+    # Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON
+    # by just running periodically in one thread, according to python, dict updates are threadsafe.
+    def save_datastore(self):
+
+        while True:
+            if self.stop_thread:
+                print("Shutting down datastore thread")
+                return
+            
+            if self.needs_write:
+                self.sync_to_json()
+            time.sleep(3)
+
+    # Go through the datastore path and remove any snapshots that are not mentioned in the index
+    # This usually is not used, but can be handy.
+    def remove_unused_snapshots(self):
+        print ("Removing snapshots from datastore that are not in the index..")
+
+        index=[]
+        for uuid in self.data['watching']:
+            for id in self.data['watching'][uuid]['history']:
+                index.append(self.data['watching'][uuid]['history'][str(id)])
+
+        import pathlib
+        # Only in the sub-directories
+        for item in pathlib.Path(self.datastore_path).rglob("*/*txt"):
+            if not str(item) in index:
+                print ("Removing",item)
+                unlink(item)

backend/templates/_helpers.jinja

@@ -0,0 +1,12 @@
+{% macro render_field(field) %}
+  <dt {% if field.errors %} class="error" {% endif %}>{{ field.label }}
+  <dd {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </dd>
+{% endmacro %}

backend/templates/base.html

@@ -5,60 +5,88 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta name="description" content="Self hosted website change detection.">
     <title>Change Detection</title>
-    <link rel="stylesheet" href="/static/css/pure-min.css">
-    <link rel="stylesheet" href="/static/css/styles.css">
+    <link rel="stylesheet" href="/static/styles/pure-min.css">
+    <link rel="stylesheet" href="/static/styles/styles.css?ver=1000">
     {% if extra_stylesheets %}
         {% for m in extra_stylesheets %}
-        <link rel="stylesheet" href="{{ m }}">
+        <link rel="stylesheet" href="{{ m }}?ver=1000">
         {% endfor %}
     {% endif %}
 </head>
 <body>
 
 <div class="header">
-    <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed">
-        <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
+
+    <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
+        {% if has_password and not current_user.is_authenticated %}
+            <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener"><strong>Change</strong>Detection.io</a>
+        {% else %}
+            <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
+        {% endif %}
+        {% if current_diff_url %}
+        <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</span></a>
+        {% else %}
+        {% if new_version_available %}
+        <span id="new-version-text" class="pure-menu-heading"><a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a></span>
+        {% endif %}
+        {% endif %}
 
         <ul class="pure-menu-list">
-
+        {% if current_user.is_authenticated or not has_password %}
+            {% if not current_diff_url %}
+            <li class="pure-menu-item">
+                <a href="/backup" class="pure-menu-link">BACKUP</a>
+            </li>
             <li class="pure-menu-item">
                 <a href="/import" class="pure-menu-link">IMPORT</a>
             </li>
             <li class="pure-menu-item">
                 <a href="/settings" class="pure-menu-link">SETTINGS</a>
             </li>
+            {% else %}
+            <li class="pure-menu-item">
+                <a href="{{ url_for('edit_page', uuid=uuid) }}" class="pure-menu-link">EDIT</a>
+            </li>
+            {% endif %}
+        {% else %}
+            <li class="pure-menu-item">
+                <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
+            </li>
+        {% endif %}
+
+        {% if current_user.is_authenticated %}
+            <li class="pure-menu-item"><a href="/logout" class="pure-menu-link">LOG OUT</a></li>
+        {% endif %}
             <li class="pure-menu-item"><a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
-                <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16" version="1.1"
+                <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16"
+                     version="1.1"
                      width="32" aria-hidden="true">
                     <path fill-rule="evenodd"
                           d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
                 </svg>
             </a></li>
-            <!--
-            <li class="pure-menu-item"><a href="#" class="pure-menu-link">Tour</a></li>
-            <li class="pure-menu-item"><a href="#" class="pure-menu-link">Sign Up</a></li>
-            -->
         </ul>
     </div>
 </div>
-
+<div id="version">v{{ version }}</div>
 <section class="content">
     <header>
         {% block header %}{% endblock %}
     </header>
 
-    {% if messages %}
-    <div class="messages">
-        {% for message in messages %}
-        <div class="flash-message {{ message['class'] }}">{{ message['message'] }}</div>
+    {% with messages = get_flashed_messages(with_categories=true) %}
+      {% if messages %}
+        <ul class=messages>
+        {% for category, message in messages %}
+          <li class="{{ category }}">{{ message }}</li>
         {% endfor %}
-    </div>
-    {% endif %}
-
-
+        </ul>
+      {% endif %}
+    {% endwith %}
     {% block content %}
 
     {% endblock %}
 </section>
+
 </body>
 </html>

backend/templates/diff.html

@@ -2,34 +2,61 @@
 
 {% block content %}
 
-<div id="diff-ui">
+<div id="settings">
+    <h1>Differences</h1>
+    <form class="pure-form " action="" method="GET">
+        <fieldset>
+
+            <label for="diffWords" class="pure-checkbox">
+                <input type="radio" name="diff_type" id="diffWords" value="diffWords"/> Words</label>
+            <label for="diffLines" class="pure-checkbox">
+                <input type="radio" name="diff_type" id="diffLines" value="diffLines" checked=""/> Lines</label>
+
+            <label for="diffChars" class="pure-checkbox">
+                <input type="radio" name="diff_type" id="diffChars" value="diffChars"/> Chars</label>
+
+            {% if versions|length >= 1 %}
+            <label for="diff-version">Compare newest (<span id="current-v-date"></span>) with</label>
+            <select id="diff-version" name="previous_version">
+                {% for version in versions %}
+                <option value="{{version}}" {% if version== current_previous_version %} selected="" {% endif %}>
+                    {{version}}
+                </option>
+                {% endfor %}
+            </select>
+            <button type="submit" class="pure-button pure-button-primary">Go</button>
+            {% endif %}
+        </fieldset>
+    </form>
     <del>Removed text</del>
     <ins>Inserted Text</ins>
-
-<div id="settings">
-    <h3>Diff</h3>
-    <label><input type="radio" name="diff_type" value="diffChars"> Chars</label>
-    <label><input type="radio" name="diff_type" value="diffWords" > Words</label>
-    <label><input type="radio" name="diff_type" value="diffLines" checked=""> Lines</label>
+    <a href="{{ url_for('preview_page', uuid=uuid) }}">Show current snapshot</a>
 </div>
 
+<div id="diff-jump">
+    <a onclick="next_diff();">Jump</a>
+</div>
+<div id="diff-ui">
     <table>
         <tbody>
         <tr>
-			<!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
-            <td id="a" style="display: none;">{{left}}</td>
-            <td id="b" style="display: none;">{{right}}</td>
-            <td>
-                <pre id="result"></pre>
+            <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
+            <td id="a" style="display: none;">{{previous}}</td>
+            <td id="b" style="display: none;">{{newest}}</td>
+            <td id="diff-col">
+                <span id="result"></span>
             </td>
         </tr>
         </tbody>
     </table>
-Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff" >github.com/kpdecker/jsdiff</a>
+    Diff algorithm from the amazing <a href="https://github.com/kpdecker/jsdiff">github.com/kpdecker/jsdiff</a>
+
 </div>
 
 <script src="/static/js/diff.js"></script>
 <script defer="">
+
+
 var a = document.getElementById('a');
 var b = document.getElementById('b');
 var result = document.getElementById('result');
@@ -48,9 +75,12 @@ function changed() {
 		var node;
 		if (diff[i].removed) {
 			node = document.createElement('del');
+			node.classList.add("change");
 			node.appendChild(document.createTextNode(diff[i].value));
+
 		} else if (diff[i].added) {
 			node = document.createElement('ins');
+			node.classList.add("change");
 			node.appendChild(document.createTextNode(diff[i].value));
 		} else {
 			node = document.createTextNode(diff[i].value);
@@ -60,11 +90,33 @@ function changed() {
 
 	result.textContent = '';
 	result.appendChild(fragment);
+
+	// Jump at start
+	inputs.current=0;
+    next_diff();
 }
 
 window.onload = function() {
+
+
+    /* Convert what is options from UTC time.time() to local browser time */
+    var diffList=document.getElementById("diff-version");
+    if (typeof(diffList) != 'undefined' && diffList != null) {
+        for (var option of diffList.options) {
+          var dateObject = new Date(option.value*1000);
+          option.label=dateObject.toLocaleString();
+        }
+    }
+
+    /* Set current version date as local time in the browser also */
+    var current_v = document.getElementById("current-v-date");
+    var dateObject = new Date({{ newest_version_timestamp }}*1000);
+    current_v.innerHTML=dateObject.toLocaleString();
+
+
 	onDiffTypeChange(document.querySelector('#settings [name="diff_type"]:checked'));
 	changed();
+
 };
 
 a.onpaste = a.onchange =
@@ -89,10 +141,32 @@ for (var i = 0; i < radio.length; i++) {
 	}
 }
 
+
+var inputs = document.getElementsByClassName('change');
+inputs.current=0;
+
+
+function next_diff() {
+
+    var element = inputs[inputs.current];
+    var headerOffset = 80;
+    var elementPosition = element.getBoundingClientRect().top;
+    var offsetPosition = elementPosition - headerOffset +  window.scrollY;
+
+    window.scrollTo({
+         top: offsetPosition,
+         behavior: "smooth"
+    });
+
+    inputs.current++;
+    if(inputs.current >= inputs.length) {
+      inputs.current=0;
+    }
+}
+
+
+
 </script>
 
 
-
-
-
 {% endblock %}

backend/templates/edit.html

@@ -1,51 +1,72 @@
 {% extends 'base.html' %}
-
 {% block content %}
-<div class="edit-form">
-
-
-    <form class="pure-form pure-form-stacked" action="/api/update?uuid={{uuid}}" method="POST">
+{% from '_helpers.jinja' import render_field %}
+<div class="edit-form monospaced-textarea">
+    <form class="pure-form pure-form-stacked" action="/edit/{{uuid}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
-                <label for="url">URL</label>
-                <input type="url" id="url" required="" placeholder="https://..." name="url" value="{{ watch.url}}"
-                       size="50"/>
-                <span class="pure-form-message-inline">This is a required field.</span>
+                {{ render_field(form.url, placeholder="https://...", size=30, required=true) }}
             </div>
             <div class="pure-control-group">
-                <label for="tag">Tag</label>
-                <input type="text" placeholder="tag" size="10" id="tag" name="tag" value="{{ watch.tag}}"/>
-                <span class="pure-form-message-inline">Grouping tags, can be a comma separated list.</span>
+                {{ render_field(form.title, size=30) }}
             </div>
-
+            <div class="pure-control-group">
+                {{ render_field(form.tag, size=10) }}
+            </div>
+            <div class="pure-control-group">
+                {{ render_field(form.minutes_between_check, size=5) }}
+                {% if using_default_minutes %}
+                    <span class="pure-form-message-inline">Currently using the <a href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
+                {% else %}
+                    <span class="pure-form-message-inline">Set to blank to use the <a href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
+                {% endif %}
+            </div>
+            <div class="pure-control-group">
+                {{ render_field(form.css_filter, size=25, placeholder=".class-name or #some-id, or other CSS selector rule.") }}
+                <span class="pure-form-message-inline">Limit text to this CSS rule, only text matching this CSS rule is included.<br/>
+                    Please be sure that you thoroughly understand how to write CSS selector rules before filing an issue on GitHub!<br/>
+                    Go <a href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>
+                </span>
+            </div>
+            <!-- @todo: move to tabs --->
             <fieldset class="pure-group">
-                <label for="headers">Extra request headers</label>
-
-                <textarea id=headers name="headers" class="pure-input-1-2" placeholder="Example
-Cookie: foobar
-User-Agent: wonderbra 1.0"
-                          style="width: 100%;
-                            font-family:monospace;
-                            white-space: pre;
-                            overflow-wrap: normal;
-                            overflow-x: scroll;" rows="5">{% for key, value in watch.headers.items() %}{{ key }}: {{ value }}
-{% endfor %}</textarea>
-                <br/>
+                {{ render_field(form.ignore_text, rows=5,  placeholder="Some text to ignore in a line
+/some.regex\d{2}/   for case-INsensitive regex
+") }}
+                <span class="pure-form-message-inline">
+                    Each line processed separately, any line matching will be ignored.<br/>
+                    Regular Expression support, wrap the line in forward slash <b>/regex/</b>.
+                </span>
 
             </fieldset>
+
+            <fieldset class="pure-group">
+                {{ render_field(form.headers, rows=5, placeholder="Example
+Cookie: foobar
+User-Agent: wonderbra 1.0") }}
+            </fieldset>
+
+            <div class="pure-control-group">
+                {{ render_field(form.notification_urls, rows=5, placeholder="Gitter - gitter://token/room
+Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
+SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
+") }}
+                <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
+            </div>
+
+            <div class="pure-controls">
+                {{ render_field(form.trigger_check, rows=5) }}
+            </div>
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Save</button>
             </div>
             <br/>
-
             <div class="pure-control-group">
                 <a href="/" class="pure-button button-small button-cancel">Cancel</a>
                 <a href="/api/delete?uuid={{uuid}}"
                    class="pure-button button-small button-error ">Delete</a>
-
             </div>
-
-
         </fieldset>
     </form>
 

backend/templates/login.html

@@ -0,0 +1,20 @@
+{% extends 'base.html' %}
+
+{% block content %}
+<div class="edit-form">
+    <form class="pure-form pure-form-stacked" action="/login" method="POST">
+        <fieldset>
+            <div class="pure-control-group">
+                <label for="password">Password</label>
+                <input type="password" id="password" required="" name="password" value=""
+                       size="15"/>
+                <input type="hidden" id="email" name="email" value="defaultuser@changedetection.io" />
+            </div>
+            <div class="pure-control-group">
+                <button type="submit" class="pure-button pure-button-primary">Login</button>
+            </div>
+        </fieldset>
+    </form>
+</div>
+
+{% endblock %}

backend/templates/preview.html

@@ -0,0 +1,26 @@
+{% extends 'base.html' %}
+
+{% block content %}
+
+<div id="settings">
+    <h1>Current</h1>
+</div>
+
+
+<div id="diff-ui">
+
+    <table>
+        <tbody>
+        <tr>
+            <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
+
+            <td id="diff-col">
+                <span id="result">{% for row in content %}<pre>{{row}}</pre>{% endfor %}</span>
+            </td>
+        </tr>
+        </tbody>
+    </table>
+</div>
+
+
+{% endblock %}

backend/templates/scrub.html

@@ -0,0 +1,35 @@
+{% extends 'base.html' %}
+
+{% block content %}
+<div class="edit-form">
+    <form class="pure-form pure-form-stacked" action="/scrub" method="POST">
+        <fieldset>
+            <div class="pure-control-group">
+                This will remove all version snapshots/data, but keep your list of URLs. <br/>
+                You may like to use the <strong>BACKUP</strong> link first.<br/>
+            </div>
+            <br/>
+            <div class="pure-control-group">
+                <label for="confirmtext">Confirmation text</label>
+                <input type="text" id="confirmtext" required="" name="confirmtext" value="" size="10"/>
+                <span class="pure-form-message-inline">Type in the word <strong>scrub</strong> to confirm that you understand!</span>
+            </div>
+            <br/>
+            <div class="pure-control-group">
+                <label for="confirmtext">Optional: Limit deletion of snapshots to snapshots <i>newer</i> than date/time</label>
+                <input type="datetime-local" id="limit_date" name="limit_date"  />
+                <span class="pure-form-message-inline">dd/mm/yyyy hh:mm (24 hour format)</span>
+            </div>
+            <br/>
+            <div class="pure-control-group">
+                <button type="submit" class="pure-button pure-button-primary">Scrub!</button>
+            </div>
+            <br/>
+            <div class="pure-control-group">
+                <a href="/" class="pure-button button-small button-cancel">Cancel</a>
+            </div>
+        </fieldset>
+    </form>
+</div>
+
+{% endblock %}

backend/templates/settings.html

@@ -1,26 +1,49 @@
 {% extends 'base.html' %}
 
 {% block content %}
+{% from '_helpers.jinja' import render_field %}
+
 <div class="edit-form">
-
-
-    <form class="pure-form pure-form-stacked" action="/settings" method="POST">
+    <form class="pure-form pure-form-stacked settings" action="/settings" method="POST">
         <fieldset>
             <div class="pure-control-group">
-                <label for="minutes">Maximum time in minutes until recheck.</label>
-                <input type="text" id="minutes" required="" name="minutes" value="{{minutes}}"
-                       size="5"/>
-                <span class="pure-form-message-inline">This is a required field.</span>
+                {{ render_field(form.minutes_between_check, size=5) }}
+                 <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
             </div>
+            <div class="pure-control-group">
+                {% if current_user.is_authenticated %}
+                    <a href="/settings?removepassword=true" class="pure-button pure-button-primary">Remove password</a>
+                {% else %}
+                    {{ render_field(form.password, size=10) }}
+                    <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
+                {% endif %}
+            </div>
+            <div class="pure-control-group">
+                {{ render_field(form.extract_title_as_title) }}
+                  <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
+            </div>
+            <div class="pure-control-group">
+                {{ render_field(form.notification_urls, rows=5, placeholder="Gitter - gitter://token/room
+Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
+SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
+") }}
+                <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
+            </div>
+                <div class="pure-controls">
+                    <span class="pure-form-message-inline"><label for="trigger-test-notification" class="pure-checkbox">
+                        <input type="checkbox" id="trigger-test-notification" name="trigger-test-notification"> Send test notification on save.</label></span>
+                </div>
 
-
+            <br/>
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Save</button>
             </div>
             <br/>
 
             <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Cancel</a>
+                <a href="/" class="pure-button button-small button-cancel">Back</a>
+                <a href="/scrub" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
             </div>
 
 

backend/templates/watch-overview.html

@@ -15,23 +15,21 @@
         <!-- user/pass r = requests.get('https://api.github.com/user', auth=('user', 'pass')) -->
     </form>
     <div>
-
+        <a href="/" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
         {% for tag in tags %}
-            {% if tag == "" %}
-            <a  href="/" class="pure-button button-tag {{'active' if active_tag == tag }}">All</a>
-            {% else %}
-            <a  href="/?tag={{ tag}}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
+            {% if tag != "" %}
+                <a href="/?tag={{ tag}}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
             {% endif %}
         {% endfor %}
     </div>
 
     <div id="watch-table-wrapper">
-
         <table class="pure-table pure-table-striped watch-table">
             <thead>
             <tr>
                 <th>#</th>
                 <th></th>
+                <th></th>
                 <th>Last Checked</th>
                 <th>Last Changed</th>
                 <th></th>
@@ -42,10 +40,14 @@
 
             {% for watch in watches %}
             <tr id="{{ watch.uuid }}"
-                class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }} {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}">
-                <td>{{ loop.index }}</td>
-                <td class="title-col">{{watch.title if watch.title is not none else watch.url}}
-                    <a class="external" target=_blank href="{{ watch.url }}"></a>
+                class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
+                {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
+                {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
+                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
+                <td class="inline">{{ loop.index }}</td>
+                <td class="inline paused-state state-{{watch.paused}}"><a href="/?pause={{ watch.uuid}}{% if active_tag %}&tag={{active_tag}}{% endif %}"><img src="/static/images/pause.svg" alt="Pause"/></a></td>
+                <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
+                    <a class="external" target="_blank" rel="noopener" href="{{ watch.url }}"></a>
                     {% if watch.last_error is defined and watch.last_error != False %}
                     <div class="fetch-error">{{ watch.last_error }}</div>
                     {% endif %}
@@ -53,20 +55,43 @@
                     <span class="watch-tag-list">{{ watch.tag}}</span>
                     {% endif %}
                 </td>
-                <td>{{watch|format_last_checked_time}}</td>
-                <td>{{watch.last_changed|format_timestamp_timeago}}</td>
-                <td><a href="/api/checknow?uuid={{ watch.uuid}}" class="pure-button button-small pure-button-primary">Recheck</a>
-                    <a href="/edit?uuid={{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
+                <td class="last-checked">{{watch|format_last_checked_time}}</td>
+                <td class="last-changed">{% if watch.history|length >= 2 and watch.last_changed %}
+                    {{watch.last_changed|format_timestamp_timeago}}
+                    {% else %}
+                    Not yet
+                    {% endif %}
+                </td>
+                <td>
+                    <a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
+                       class="pure-button button-small pure-button-primary">Recheck</a>
+                    <a href="/edit/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
                     {% if watch.history|length >= 2 %}
-                    <a href="/diff/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
+                    <a href="/diff/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
+                    {% else %}
+                        {% if watch.history|length == 1 %}
+                            <a href="/preview/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
+                        {% endif %}
                     {% endif %}
                 </td>
             </tr>
             {% endfor %}
-
-
             </tbody>
         </table>
+        <ul id="post-list-buttons">
+            {% if has_unviewed %}
+            <li>
+                <a href="/api/mark-all-viewed" class="pure-button button-tag ">Mark all viewed</a>
+            </li>
+            {% endif %}
+            <li>
+               <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
+                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
+            </li>
+            <li>
+                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="/static/images/Generic_Feed-icon.svg" height="15px"></a>
+            </li>
+        </ul>
     </div>
 </div>
 {% endblock %}

backend/tests/__init__.py

@@ -0,0 +1,2 @@
+"""Tests for the app."""
+

backend/tests/conftest.py

@@ -0,0 +1,47 @@
+#!/usr/bin/python3
+
+import pytest
+from backend import changedetection_app
+from backend import store
+import os
+
+
+# https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
+# Much better boilerplate than the docs
+# https://www.python-boilerplate.com/py3+flask+pytest/
+
+global app
+
+@pytest.fixture(scope='session')
+def app(request):
+    """Create application for the tests."""
+    datastore_path = "./test-datastore"
+
+    try:
+        os.mkdir(datastore_path)
+    except FileExistsError:
+        pass
+
+    try:
+        os.unlink("{}/url-watches.json".format(datastore_path))
+    except FileNotFoundError:
+        pass
+
+    app_config = {'datastore_path': datastore_path}
+    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
+    app = changedetection_app(app_config, datastore)
+    app.config['STOP_THREADS'] = True
+
+    def teardown():
+        datastore.stop_thread = True
+        app.config.exit.set()
+        for fname in ["url-watches.json", "count.txt", "output.txt"]:
+            try:
+                os.unlink("{}/{}".format(datastore_path, fname))
+            except FileNotFoundError:
+                # This is fine in the case of a failure.
+                pass
+
+    request.addfinalizer(teardown)
+    yield app
+

backend/tests/test_access_control.py

@@ -0,0 +1,102 @@
+from flask import url_for
+
+
+def test_check_access_control(app, client):
+    # Still doesnt work, but this is closer.
+
+    with app.test_client() as c:
+        # Check we dont have any password protection enabled yet.
+        res = c.get(url_for("settings_page"))
+        assert b"Remove password" not in res.data
+
+        # Enable password check.
+        res = c.post(
+            url_for("settings_page"),
+            data={"password": "foobar", "minutes_between_check": 180},
+            follow_redirects=True
+        )
+
+        assert b"Password protection enabled." in res.data
+        assert b"LOG OUT" not in res.data
+
+        # Check we hit the login
+        res = c.get(url_for("index"), follow_redirects=True)
+
+        assert b"Login" in res.data
+
+        # Menu should not be available yet
+        #        assert b"SETTINGS" not in res.data
+        #        assert b"BACKUP" not in res.data
+        #        assert b"IMPORT" not in res.data
+
+        # defaultuser@changedetection.io is actually hardcoded for now, we only use a single password
+        res = c.post(
+            url_for("login"),
+            data={"password": "foobar"},
+            follow_redirects=True
+        )
+
+        assert b"LOG OUT" in res.data
+        res = c.get(url_for("settings_page"))
+
+        # Menu should be available now
+        assert b"SETTINGS" in res.data
+        assert b"BACKUP" in res.data
+        assert b"IMPORT" in res.data
+        assert b"LOG OUT" in res.data
+
+        # Now remove the password so other tests function, @todo this should happen before each test automatically
+        res = c.get(url_for("settings_page", removepassword="true"),
+              follow_redirects=True)
+        assert b"Password protection removed." in res.data
+
+        res = c.get(url_for("index"))
+        assert b"LOG OUT" not in res.data
+
+
+# There was a bug where saving the settings form would submit a blank password
+def test_check_access_control_no_blank_password(app, client):
+    # Still doesnt work, but this is closer.
+
+    with app.test_client() as c:
+        # Check we dont have any password protection enabled yet.
+        res = c.get(url_for("settings_page"))
+        assert b"Remove password" not in res.data
+
+        # Enable password check.
+        res = c.post(
+            url_for("settings_page"),
+            data={"password": "", "minutes_between_check": 180},
+            follow_redirects=True
+        )
+
+        assert b"Password protection enabled." not in res.data
+        assert b"Login" not in res.data
+
+
+# There was a bug where saving the settings form would submit a blank password
+def test_check_access_no_remote_access_to_remove_password(app, client):
+    # Still doesnt work, but this is closer.
+
+    with app.test_client() as c:
+        # Check we dont have any password protection enabled yet.
+        res = c.get(url_for("settings_page"))
+        assert b"Remove password" not in res.data
+
+        # Enable password check.
+        res = c.post(
+            url_for("settings_page"),
+            data={"password": "password", "minutes_between_check": 180},
+            follow_redirects=True
+        )
+
+        assert b"Password protection enabled." in res.data
+        assert b"Login" in res.data
+
+        res = c.get(url_for("settings_page", removepassword="true"),
+              follow_redirects=True)
+        assert b"Password protection removed." not in res.data
+
+        res = c.get(url_for("index"),
+              follow_redirects=True)
+        assert b"watch-table-wrapper" not in res.data

backend/tests/test_backend.py

@@ -0,0 +1,107 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from urllib.request import urlopen
+from . util import set_original_response, set_modified_response, live_server_setup
+
+sleep_time_for_fetch_thread = 3
+
+
+def test_check_basic_change_detection_functionality(client, live_server):
+    set_original_response()
+    live_server_setup(live_server)
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": url_for('test_endpoint', _external=True)},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Do this a few times.. ensures we dont accidently set the status
+    for n in range(3):
+        client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+        # Give the thread time to pick it up
+        time.sleep(sleep_time_for_fetch_thread)
+
+        # It should report nothing found (no new 'unviewed' class)
+        res = client.get(url_for("index"))
+        assert b'unviewed' not in res.data
+        assert b'test-endpoint' in res.data
+
+        # Default no password set, this stuff should be always available.
+
+        assert b"SETTINGS" in res.data
+        assert b"BACKUP" in res.data
+        assert b"IMPORT" in res.data
+
+    #####################
+
+    # Make a change
+    set_modified_response()
+
+    res = urlopen(url_for('test_endpoint', _external=True))
+    assert b'which has this one new line' in res.read()
+
+    # Force recheck
+    res = client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    assert b'1 watches are rechecking.' in res.data
+
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Now something should be ready, indicated by having a 'unviewed' class
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    # #75, and it should be in the RSS feed
+    res = client.get(url_for("index", rss="true"))
+    expected_url = url_for('test_endpoint', _external=True)
+    assert b'<rss' in res.data
+    assert expected_url.encode('utf-8') in res.data
+
+    # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
+    res = client.get(url_for("diff_history_page", uuid="first"))
+    assert b'Compare newest' in res.data
+
+    time.sleep(2)
+
+    # Do this a few times.. ensures we dont accidently set the status
+    for n in range(2):
+        client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+        # Give the thread time to pick it up
+        time.sleep(sleep_time_for_fetch_thread)
+
+        # It should report nothing found (no new 'unviewed' class)
+        res = client.get(url_for("index"))
+        assert b'unviewed' not in res.data
+        assert b'head title' not in res.data # Should not be present because this is off by default
+        assert b'test-endpoint' in res.data
+
+    set_original_response()
+
+    # Enable auto pickup of <title> in settings
+    res = client.post(
+        url_for("settings_page"),
+        data={"extract_title_as_title": "1", "minutes_between_check": 180},
+        follow_redirects=True
+    )
+
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+    # It should have picked up the <title>
+    assert b'head title' in res.data
+
+    #
+    # Cleanup everything
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+

backend/tests/test_css_selector.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+from ..html_tools import *
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that will change</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     <div id="sametext">Some text thats the same</div>
+     <div id="changetext">Some text that changes</div>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+# Test that the CSS extraction works how we expect, important here is the right placing of new lines \n's
+def test_css_filter_output():
+    from backend import fetch_site_status
+    from inscriptis import get_text
+
+    # Check text with sub-parts renders correctly
+    content = """<html> <body><div id="thingthing" >  Some really <b>bold</b> text  </div> </body> </html>"""
+    html_blob = css_filter(css_filter="#thingthing", html_content=content)
+    text = get_text(html_blob)
+    assert text == "  Some really bold text"
+
+    content = """<html> <body>
+    <p>foo bar blah</p>
+    <div class="parts">Block A</div> <div class="parts">Block B</div></body> 
+    </html>
+"""
+    html_blob = css_filter(css_filter=".parts", html_content=content)
+    text = get_text(html_blob)
+
+    # Divs are converted to 4 whitespaces by inscriptis
+    assert text == "    Block A\n    Block B"
+
+
+# Tests the whole stack works with the CSS Filter
+def test_check_markup_css_filter_restriction(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    css_filter = "#sametext"
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": ""},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(css_filter.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+    #  Make a change
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should have 'unviewed' still
+    # Because it should be looking at only that 'sametext' id
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data

backend/tests/test_ignore_regex_text.py

@@ -0,0 +1,31 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+# Unit test of the stripper
+# Always we are dealing in utf-8
+def test_strip_regex_text_func():
+    from backend import fetch_site_status
+
+    test_content = """
+    but sometimes we want to remove the lines.
+    
+    but 1 lines
+    but including 1234 lines
+    igNORe-cAse text we dont want to keep    
+    but not always."""
+
+    ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"]
+
+    fetcher = fetch_site_status.perform_site_check(datastore=False)
+    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
+
+    assert b"but 1 lines" in stripped_content
+    assert b"igNORe-cAse text" not in stripped_content
+    assert b"but 1234 lines" not in stripped_content
+

backend/tests/test_ignore_text.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+# Unit test of the stripper
+# Always we are dealing in utf-8
+def test_strip_text_func():
+    from backend import fetch_site_status
+
+    test_content = """
+    Some content
+    is listed here
+
+    but sometimes we want to remove the lines.
+
+    but not always."""
+
+    ignore_lines = ["sometimes"]
+
+    fetcher = fetch_site_status.perform_site_check(datastore=False)
+    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
+
+    assert b"sometimes" not in stripped_content
+    assert b"Some content" in stripped_content
+
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_modified_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some NEW nice initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+
+# Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
+def set_modified_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     <P>ZZZZZ</P>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_check_ignore_text_functionality(client, live_server):
+    sleep_time_for_fetch_thread = 3
+
+    ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
+    set_original_ignore_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"ignore_text": ignore_text, "url": test_url},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(ignore_text.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+    #  Make a change
+    set_modified_ignore_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+    # Just to be sure.. set a regular modified change..
+    set_modified_original_ignore_response()
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

backend/tests/test_notification.py

@@ -0,0 +1,72 @@
+
+import time
+from flask import url_for
+from . util import set_original_response, set_modified_response, live_server_setup
+
+# Hard to just add more live server URLs when one test is already running (I think)
+# So we add our test here (was in a different file)
+def test_check_notification(client, live_server):
+
+    live_server_setup(live_server)
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(3)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    url = url_for('test_notification_endpoint', _external=True)
+    notification_url = url.replace('http', 'json')
+
+    print (">>>> Notification URL: "+notification_url)
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"notification_urls": notification_url, "url": test_url, "tag": "", "headers": ""},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Hit the edit page, be sure that we saved it
+    res = client.get(
+        url_for("edit_page", uuid="first"))
+    assert bytes(notification_url.encode('utf-8')) in res.data
+
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Did the front end see it?
+    res = client.get(
+        url_for("index"))
+
+    assert bytes("just now".encode('utf-8')) in res.data
+
+
+    # Check it triggered
+    res = client.get(
+        url_for("test_notification_counter"),
+    )
+
+    assert bytes("we hit it".encode('utf-8')) in res.data
+
+    # Did we see the URL that had a change, in the notification?
+    assert bytes("test-endpoint".encode('utf-8')) in res.data
+
+    # Re #65 - did we see our foobar.com BASE_URL ?
+    assert bytes("https://foobar.com".encode('utf-8')) in res.data

backend/tests/test_watch_fields_storage.py

@@ -0,0 +1,145 @@
+import time
+from flask import url_for
+from urllib.request import urlopen
+from . util import set_original_response, set_modified_response, live_server_setup
+
+
+def test_check_watch_field_storage(client, live_server):
+    set_original_response()
+    live_server_setup(live_server)
+
+    test_url = "http://somerandomsitewewatch.com"
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={ "notification_urls": "http://myapi.com",
+               "minutes_between_check": 126,
+               "css_filter" : ".fooclass",
+               "title" : "My title",
+               "ignore_text" : "ignore this",
+               "url": test_url,
+               "tag": "woohoo",
+               "headers": "curl:foo",
+
+               },
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"http://myapi.com" in res.data
+    assert b"126" in res.data
+    assert b".fooclass" in res.data
+    assert b"My title" in res.data
+    assert b"ignore this" in res.data
+    assert b"http://somerandomsitewewatch.com" in res.data
+    assert b"woohoo" in res.data
+    assert b"curl: foo" in res.data
+
+
+
+# Re https://github.com/dgtlmoon/changedetection.io/issues/110
+def test_check_recheck_global_setting(client, live_server):
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 1566,
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Now add a record
+
+    test_url = "http://somerandomsitewewatch.com"
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Now visit the edit page, it should have the default minutes
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should show the default minutes
+    assert b"change to another value if you want to be specific" in res.data
+    assert b"1566" in res.data
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 222,
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should show the default minutes
+    assert b"change to another value if you want to be specific" in res.data
+    assert b"222" in res.data
+
+    # Now change it specifically, it should show the new minutes
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url,
+              "minutes_between_check": 55,
+              },
+        follow_redirects=True
+    )
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"55" in res.data
+
+    # Now submit an empty field, it should give back the default global minutes
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 666,
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url,
+              "minutes_between_check": "",
+              },
+        follow_redirects=True
+    )
+
+    assert b"Updated watch." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"666" in res.data
+

backend/tests/util.py

@@ -0,0 +1,69 @@
+#!/usr/bin/python3
+
+
+def set_original_response():
+    test_return_data = """<html>
+    <head><title>head title</title></head>
+    <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """<html>
+    <head><title>modified head title</title></head>
+    <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+def live_server_setup(live_server):
+
+    @live_server.app.route('/test-endpoint')
+    def test_endpoint():
+        # Tried using a global var here but didn't seem to work, so reading from a file instead.
+        with open("test-datastore/output.txt", "r") as f:
+            return f.read()
+
+    @live_server.app.route('/test_notification_endpoint', methods=['POST'])
+    def test_notification_endpoint():
+        from flask import request
+
+        with open("test-datastore/count.txt", "w") as f:
+            f.write("we hit it\n")
+            # Debug method, dump all POST to file also, used to prove #65
+            data = request.stream.read()
+            if data != None:
+                f.write(str(data))
+
+        print("\n>> Test notification endpoint was hit.\n")
+        return "Text was set"
+
+    # And this should return not zero.
+    @live_server.app.route('/test_notification_counter')
+    def test_notification_counter():
+        try:
+            with open("test-datastore/count.txt", "r") as f:
+                return f.read()
+        except FileNotFoundError:
+            return "nope :("
+
+    live_server.start()

backend/update_worker.py

@@ -0,0 +1,69 @@
+import threading
+import queue
+
+# Requests for checking on the site use a pool of thread Workers managed by a Queue.
+class update_worker(threading.Thread):
+    current_uuid = None
+
+    def __init__(self, q, notification_q, app, datastore, *args, **kwargs):
+        self.q = q
+        self.app = app
+        self.notification_q = notification_q
+        self.datastore = datastore
+        super().__init__(*args, **kwargs)
+
+    def run(self):
+        from backend import fetch_site_status
+
+        update_handler = fetch_site_status.perform_site_check(datastore=self.datastore)
+
+        while not self.app.config.exit.is_set():
+
+            try:
+                uuid = self.q.get(block=False)
+            except queue.Empty:
+                pass
+
+            else:
+                self.current_uuid = uuid
+
+                if uuid in list(self.datastore.data['watching'].keys()):
+                    try:
+                        changed_detected, result, contents = update_handler.run(uuid)
+
+                    except PermissionError as e:
+                        self.app.logger.error("File permission error updating", uuid, str(e))
+                    except Exception as e:
+                        self.app.logger.error("Exception reached", uuid, str(e))
+                    else:
+                        if result:
+                            try:
+                                self.datastore.update_watch(uuid=uuid, update_obj=result)
+                                if changed_detected:
+                                    # A change was detected
+                                    self.datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
+
+                                    watch = self.datastore.data['watching'][uuid]
+
+                                    # Did it have any notification alerts to hit?
+                                    if len(watch['notification_urls']):
+                                        print("Processing notifications for UUID: {}".format(uuid))
+                                        n_object = {'watch_url': self.datastore.data['watching'][uuid]['url'],
+                                                    'notification_urls': watch['notification_urls']}
+                                        self.notification_q.put(n_object)
+
+
+                                    # No? maybe theres a global setting, queue them all
+                                    elif len(self.datastore.data['settings']['application']['notification_urls']):
+                                        print("Processing GLOBAL notifications for UUID: {}".format(uuid))
+                                        n_object = {'watch_url': self.datastore.data['watching'][uuid]['url'],
+                                                    'notification_urls': self.datastore.data['settings']['application'][
+                                                        'notification_urls']}
+                                        self.notification_q.put(n_object)
+                            except Exception as e:
+                                print("!!!! Exception in update_worker !!!\n", e)
+
+                self.current_uuid = None  # Done
+                self.q.task_done()
+
+            self.app.config.exit.wait(1)

changedetection.py

@@ -0,0 +1,80 @@
+#!/usr/bin/python3
+
+# Launch as a eventlet.wsgi server instance.
+
+import getopt
+import os
+import sys
+
+import eventlet
+import eventlet.wsgi
+import backend
+
+from backend import store
+
+def main(argv):
+    ssl_mode = False
+    port = 5000
+    do_cleanup = False
+
+    # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
+    datastore_path = os.path.join(os.getcwd(), "datastore")
+
+    try:
+        opts, args = getopt.getopt(argv, "csd:p:", "port")
+    except getopt.GetoptError:
+        print('backend.py -s SSL enable -p [port] -d [datastore path]')
+        sys.exit(2)
+
+    for opt, arg in opts:
+        #        if opt == '--purge':
+        # Remove history, the actual files you need to delete manually.
+        #            for uuid, watch in datastore.data['watching'].items():
+        #                watch.update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': None})
+
+        if opt == '-s':
+            ssl_mode = True
+
+        if opt == '-p':
+            port = int(arg)
+
+        if opt == '-d':
+            datastore_path = arg
+
+        # Cleanup (remove text files that arent in the index)
+        if opt == '-c':
+            do_cleanup = True
+
+    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
+    app_config = {'datastore_path': datastore_path}
+
+    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'])
+    app = backend.changedetection_app(app_config, datastore)
+
+    # Go into cleanup mode
+    if do_cleanup:
+        datastore.remove_unused_snapshots()
+
+    app.config['datastore_path'] = datastore_path
+
+
+    @app.context_processor
+    def inject_version():
+        return dict(version=datastore.data['version_tag'],
+                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
+                    has_password=datastore.data['settings']['application']['password'] != False
+                    )
+
+    if ssl_mode:
+        # @todo finalise SSL config, but this should get you in the right direction if you need it.
+        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen(('', port)),
+                                               certfile='cert.pem',
+                                               keyfile='privkey.pem',
+                                               server_side=True), app)
+
+    else:
+        eventlet.wsgi.server(eventlet.listen(('', port)), app)
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

datastore/README.md

@@ -1,2 +0,0 @@
-Empty dir, please keep, this is used to store your data!
-

docker-compose-development.yml

@@ -1,22 +0,0 @@
-version: "2"
-services:
-
-# I have a feeling we can get rid of this, and just use one docker-compose.yml, and just set a ENV var if
-# we want dev mode (just gives a docker shell) or not.
-
-  backend:
-    build: ./backend/dev-docker
-    image: dgtlmoon/changedetection.io:dev
-    container_name: changedetection.io-dev
-    volumes:
-      - ./backend:/app
-      - ./datastore:/datastore
-
-    ports:
-      - "127.0.0.1:5001:5000"
-
-    networks:
-      - changenet
-
-networks:
-  changenet:

docker-compose.yml

@@ -0,0 +1,26 @@
+version: '2'
+services:
+    changedetection.io:
+      image: dgtlmoon/changedetection.io
+      container_name: changedetection.io
+      hostname: changedetection.io
+      volumes:
+        - changedetection-data:/datastore
+  #    environment:
+  #      - PUID=1000
+  #      - PGID=1000
+  #        Proxy support example.
+  #      - HTTP_PROXY="socks5h://10.10.1.10:1080"
+  #      - HTTPS_PROXY="socks5h://10.10.1.10:1080"
+  #        An exclude list (useful for notification URLs above) can be specified by with
+  #      - NO_PROXY="localhost,192.168.0.0/24"
+  #        Base URL of your changedetection.io install (Added to notification alert
+  #      - BASE_URL="https://mysite.com"
+
+      ports:
+        - 5000:5000
+      restart: always
+
+volumes:
+  changedetection-data:
+

requirements.txt

@@ -1,21 +1,23 @@
-aiohttp
-async-timeout
 chardet==2.3.0
-multidict
-python-engineio
-six==1.10.0
-yarl
-flask
-
-eventlet
-requests
+flask~= 1.0
+pytest ~=6.2
+pytest-flask ~=1.2
+eventlet>=0.31.0
+requests[socks] ~= 2.15
 validators
+timeago ~=1.0
+inscriptis ~= 1.1
+feedgen ~= 0.9
+flask-login ~= 0.5
+pytz
+urllib3
+wtforms ~= 2.3.3
 
-bleach==3.2.1
-html5lib==0.9999999       # via bleach
-timeago
-html2text
-inscriptis
 
-# @notes
-# - Dont install socketio, it interferes with flask_socketio
+
+# Notification library
+apprise ~= 0.9
+
+# Used for CSS filtering, replace with soupsieve and lxml for xpath
+bs4
+