Re #23 - always check value of interval time, not just on start

Proof of concept, view as a stream
2026-03-29 15:18:07 +00:00 · 2021-03-29 15:03:33 +02:00 · 2021-03-15 22:26:50 +01:00
52 changed files with 873 additions and 4225 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,2 +0,0 @@
 .git
 .github
--- a/.github/FUNDING.yml
+++ b/.github/FUNDING.yml
@@ -1,3 +1,12 @@
 # These are supported funding model platforms
 github: dgtlmoon
 patreon: # Replace with a single Patreon username
 open_collective: # Replace with a single Open Collective username
 ko_fi: # Replace with a single Ko-fi username
 tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
 community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
 liberapay: # Replace with a single Liberapay username
 issuehunt: # Replace with a single IssueHunt username
 otechie: # Replace with a single Otechie username
 custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -1,62 +0,0 @@
 # For most projects, this workflow file will not need changing; you simply need
 # to commit it to your repository.
 #
 # You may wish to alter this file to override the set of languages analyzed,
 # or to provide custom queries or build logic.
 #
 # ******** NOTE ********
 # We have attempted to detect the languages in your repository. Please check
 # the `language` matrix defined below to confirm you have the correct set of
 # supported CodeQL languages.
 #
 name: "CodeQL"
 on:
  schedule:
    - cron: '27 9 * * 4'
 jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        language: [ 'javascript', 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
        # Learn more:
        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
    steps:
    - name: Checkout repository
      uses: actions/checkout@v2
    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v1
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.
        # queries: ./path/to/local/query, your-org/your-repo/queries@main
    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v1
    # ℹ️ Command-line programs to run using the OS shell.
    # 📚 https://git.io/JvXDl
    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
    #    and modify them (or add more) to build your code if your project
    #    uses a compiled language
    #- run: |
    #   make bootstrap
    #   make release
    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v1
--- a/.github/workflows/image.yml
+++ b/.github/workflows/image.yml
@@ -1,88 +0,0 @@
 name: Test, build and push to Docker Hub
 on:
  push:
    branches: [ master, arm-build ]
 jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.9
        uses: actions/setup-python@v2
        with:
          python-version: 3.9
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install flake8 pytest
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
      - name: Lint with flake8
        run: |
          # stop the build if there are Python syntax errors or undefined names
          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
      - name: Create release metadata
        run: |
          # COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py
          echo ${{ github.sha }} > backend/source.txt
          echo ${{ github.ref }} > backend/tag.txt
      - name: Test with pytest
        run: |
          # Each test is totally isolated and performs its own cleanup/reset
          cd backend; ./run_all_tests.sh
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1
        with:
          image: tonistiigi/binfmt:latest
          platforms: all
      - name: Login to Docker Hub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v1
        with:
          install: true
          version: latest
          driver-opts: image=moby/buildkit:master
      - name: Build and push
        id: docker_build
        uses: docker/build-push-action@v2
        with:
          context: ./
          file: ./Dockerfile
          push: true
          tags: |
            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
          #                ${{ secrets.DOCKER_HUB_USERNAME }}:/changedetection.io:${{ env.RELEASE_VERSION }}
          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
 #          platforms: linux/amd64
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,dest=/tmp/.buildx-cache
      - name: Image digest
        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
 # failed: Cache service responded with 503
 #      - name: Cache Docker layers
 #        uses: actions/cache@v2
 #        with:
 #          path: /tmp/.buildx-cache
 #          key: ${{ runner.os }}-buildx-${{ github.sha }}
 #          restore-keys: |
 #            ${{ runner.os }}-buildx-
--- a/.github/workflows/python-app.yml
+++ b/.github/workflows/python-app.yml
@@ -0,0 +1,33 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 name: changedetection.io
 on: [push, pull_request]
 jobs:
  build:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python 3.9
      uses: actions/setup-python@v2
      with:
        python-version: 3.9
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install flake8 pytest
        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
    - name: Test with pytest
      run: |
        cd backend; pytest
--- a/.github/workflows/test-only.yml
+++ b/.github/workflows/test-only.yml
@@ -1,33 +0,0 @@
 name: ChangeDetection.io Test
 # Triggers the workflow on push or pull request events
 on: [push, pull_request]
 jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.9
        uses: actions/setup-python@v2
        with:
          python-version: 3.9
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install flake8 pytest
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
      - name: Lint with flake8
        run: |
          # stop the build if there are Python syntax errors or undefined names
          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
      - name: Test with pytest
        run: |
          # Each test is totally isolated and performs its own cleanup/reset
          cd backend; ./run_all_tests.sh
--- a/58
+++ b/58
@@ -1,56 +1,28 @@
 # pip dependencies install stage
 FROM python:3.8-slim as builder
 # rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 RUN apt-get update && apt-get install -y --no-install-recommends \
    libssl-dev \
    libffi-dev \
    gcc \
    libc-dev \
    libxslt-dev \
    zlib1g-dev \
    g++
 RUN mkdir /install
 WORKDIR /install
 COPY requirements.txt /requirements.txt
 RUN pip install --target=/dependencies -r /requirements.txt
 # Final image stage
 FROM python:3.8-slim
 COPY requirements.txt /tmp/requirements.txt
 RUN pip3 install -r /tmp/requirements.txt
 # Actual packages needed at runtime, usually due to the notification (apprise) backend
 # rustc compiler would be needed on ARM type devices but theres an issue with some deps not building..
 ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
 # Re #93, #73, excluding rustc (adds another 430Mb~)
 RUN apt-get update && apt-get install -y --no-install-recommends \
    libssl-dev \
    libffi-dev \
    gcc \
    libc-dev \
    libxslt-dev \
    zlib1g-dev \
    g++
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1
 RUN [ ! -d "/app" ] && mkdir /app
 RUN [ ! -d "/datastore" ] && mkdir /datastore
 # Copy modules over to the final image and add their dir to PYTHONPATH
 COPY --from=builder /dependencies /usr/local
 ENV PYTHONPATH=/usr/local
 # The actual flask app
 COPY backend /app/backend
 # The eventlet server wrapper
 COPY changedetection.py /app/changedetection.py
 WORKDIR /app
 # https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops
 ENV PYTHONUNBUFFERED=1
 # Attempt to store the triggered commit
 ARG SOURCE_COMMIT
 ARG SOURCE_BRANCH
 RUN echo "commit: $SOURCE_COMMIT branch: $SOURCE_BRANCH" >/source.txt
 CMD [ "python", "./changedetection.py" , "-d", "/datastore"]
--- a/README.md
+++ b/README.md
@@ -1,21 +1,19 @@
 #  changedetection.io
-![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
+![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/python-app.yml/badge.svg?branch=master)
 <a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
  <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
 </a>
 <a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
-  <img src="https://img.shields.io/github/v/release/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
+  <img src="https://img.shields.io/docker/v/dgtlmoon/changedetection.io/0.27" alt="Change detection latest tag version"/> 
 </a>
 ## Self-hosted change monitoring of web pages.
 _Know when web pages change! Stay ontop of new information!_
-Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
+![Self-hosted web page change monitoring application screenshot](screenshot.png?raw=true "Self-hosted web page change monitoring screenshot")
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
 #### Example use cases
 Know when ...
@@ -25,7 +23,6 @@ Know when ...
 - New software releases, security advisories when you're not on their mailing list.
 - Festivals with changes
 - Realestate listing changes
 - COVID related news from government websites
 **Get monitoring now! super simple, one command!**
@@ -51,69 +48,16 @@ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/dat
 Examining differences in content.
 ![Self-hosted web page change monitoring context difference screenshot](screenshot-diff.png?raw=true "Self-hosted web page change monitoring context difference screenshot")
 ### Future plans
 - Greater configuration of check interval times, page request headers.
 - ~~General options for timeout, default headers~~
 - On change detection, callout to another API (handy for notices/issue trackers)
 - ~~Explore the differences that were detected~~ 
 - Add more options to explore versions of differences
 - Use a graphic/rendered page difference instead of text (see the experimental `selenium-screenshot-diff` branch)
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
 ### Notifications
 ChangeDetection.io supports a massive amount of notifications (including email, office365, custom APIs, etc) when a web-page has a change detected thanks to the <a href="https://github.com/caronc/apprise">apprise</a> library.
 Simply set one or more notification URL's in the _[edit]_ tab of that watch.
 Just some examples
    discord://webhook_id/webhook_token
    flock://app_token/g:channel_id
    gitter://token/room
    gchat://workspace/key/token
    msteams://TokenA/TokenB/TokenC/
    o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
    rocket://user:password@hostname/#Channel
    mailto://user:pass@example.com?to=receivingAddress@example.com
    json://someserver.com/custom-api
    syslog://
 <a href="https://github.com/caronc/apprise">And everything else in this list!</a>
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
 ### Proxy
 A proxy for ChangeDectection.io can be configured by setting environment the 
 `HTTP_PROXY`, `HTTPS_PROXY` variables, examples are also in the `docker-compose.yml`
 `NO_PROXY` exclude list can be specified by following `"localhost,192.168.0.0/24"`
 as `docker run` with `-e`
 ```
 docker run -d --restart always -e HTTPS_PROXY="socks5h://10.10.1.10:1080" -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
 ```
 With `docker-compose`, see the `Proxy support example` in <a href="https://github.com/dgtlmoon/changedetection.io/blob/master/docker-compose.yml">docker-compose.yml</a>.
 For more information see https://docs.python-requests.org/en/master/user/advanced/#proxies
 This proxy support also extends to the notifications https://github.com/caronc/apprise/issues/387#issuecomment-841718867
 ### Notes
 - Does not yet support Javascript
 - Wont work with Cloudfare type "Please turn on javascript" protected pages
 - You can use the 'headers' section to monitor password protected web page changes
 ### RaspberriPi support?
 RaspberriPi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! 
 ### Support us
 Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 Please support us, even small amounts help a LOT.
 BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />
--- a/backend/README-pytest.md
+++ b/backend/README-pytest.md
@@ -0,0 +1 @@
 Note: run `pytest` from this directory.
--- a/backend/init.py
+++ b/backend/init.py
@@ -6,6 +6,7 @@
 # @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option?
 # @todo option for interval day/6 hour/etc
 # @todo on change detected, config for calling some API
 # @todo make tables responsive!
 # @todo fetch title into json
 # https://distill.io/features
 # proxy per check
@@ -14,15 +15,13 @@
 import time
 import os
 import timeago
 import flask_login
 from flask_login import login_required
 import threading
 from threading import Event
 import queue
-from flask import Flask, render_template, request, send_from_directory, abort, redirect, url_for, flash
+from flask import Flask, render_template, request, send_file, send_from_directory, abort, redirect, url_for
 from feedgen.feed import FeedGenerator
 from flask import make_response
@@ -35,13 +34,12 @@ datastore = None
 running_update_threads = []
 ticker_thread = None
 messages = []
 extra_stylesheets = []
 update_q = queue.Queue()
-notification_q = queue.Queue()
+app = Flask(__name__, static_url_path="/var/www/change-detection/backen/static")
 app = Flask(__name__, static_url_path="/var/www/change-detection/backend/static")
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
@@ -50,42 +48,10 @@ app.config.exit = Event()
 app.config['NEW_VERSION_AVAILABLE'] = False
 app.config['LOGIN_DISABLED'] = False
 #app.config["EXPLAIN_TEMPLATE_LOADING"] = True
 # Disables caching of the templates
 app.config['TEMPLATES_AUTO_RELOAD'] = True
 def init_app_secret(datastore_path):
    secret = ""
    path = "{}/secret.txt".format(datastore_path)
    try:
        with open(path, "r") as f:
            secret = f.read()
    except FileNotFoundError:
        import secrets
        with open(path, "w") as f:
            secret = secrets.token_hex(32)
            f.write(secret)
    return secret
 # Remember python is by reference
 # populate_form in wtfors didnt work for me. (try using a setattr() obj type on datastore.watch?)
 def populate_form_from_watch(form, watch):
    for i in form.__dict__.keys():
        if i[0] != '_':
            p = getattr(form, i)
            if hasattr(p, 'data') and i in watch:
                if not p.data:
                    setattr(p, "data", watch[i])
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
 # running or something similar.
@app.template_filter('format_last_checked_time')
@@ -114,127 +80,27 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
    # return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
 class User(flask_login.UserMixin):
    id=None
    def set_password(self, password):
        return True
    def get_user(self, email="defaultuser@changedetection.io"):
        return self
    def is_authenticated(self):
        return True
    def is_active(self):
        return True
    def is_anonymous(self):
        return False
    def get_id(self):
        return str(self.id)
    def check_password(self, password):
        import hashlib
        import base64
        # Getting the values back out
        raw_salt_pass = base64.b64decode(datastore.data['settings']['application']['password'])
        salt_from_storage = raw_salt_pass[:32]  # 32 is the length of the salt
        # Use the exact same setup you used to generate the key, but this time put in the password to check
        new_key = hashlib.pbkdf2_hmac(
            'sha256',
            password.encode('utf-8'),  # Convert the password to bytes
            salt_from_storage,
            100000
        )
        new_key =  salt_from_storage + new_key
        return new_key == raw_salt_pass
    pass
 def changedetection_app(config=None, datastore_o=None):
    global datastore
    datastore = datastore_o
    app.config.update(dict(DEBUG=True))
-    #app.config.update(config or {})
+    app.config.update(config or {})
    login_manager = flask_login.LoginManager(app)
    login_manager.login_view = 'login'
    app.secret_key = init_app_secret(config['datastore_path'])
    # Setup cors headers to allow all domains
    # https://flask-cors.readthedocs.io/en/latest/
    #    CORS(app)
    @login_manager.user_loader
    def user_loader(email):
        user = User()
        user.get_user(email)
        return user
    @login_manager.unauthorized_handler
    def unauthorized_handler():
        # @todo validate its a URL of this host and use that
        return redirect(url_for('login', next=url_for('index')))
    @app.route('/logout')
    def logout():
        flask_login.logout_user()
        return redirect(url_for('index'))
    # https://github.com/pallets/flask/blob/93dd1709d05a1cf0e886df6223377bdab3b077fb/examples/tutorial/flaskr/__init__.py#L39
    # You can divide up the stuff like this
    @app.route('/login', methods=['GET', 'POST'])
    def login():
        if not datastore.data['settings']['application']['password']:
            flash("Login not required, no password enabled.", "notice")
            return redirect(url_for('index'))
        if request.method == 'GET':
            output = render_template("login.html")
            return output
        user = User()
        user.id = "defaultuser@changedetection.io"
        password = request.form.get('password')
        if (user.check_password(password)):
            flask_login.login_user(user, remember=True)
            next = request.args.get('next')
            #            if not is_safe_url(next):
            #                return flask.abort(400)
            return redirect(next or url_for('index'))
        else:
            flash('Incorrect password', 'error')
        return redirect(url_for('login'))
    @app.before_request
    def do_something_whenever_a_request_comes_in():
        # Disable password  loginif there is not one set
        app.config['LOGIN_DISABLED'] = datastore.data['settings']['application']['password'] == False
    @app.route("/", methods=['GET'])
    @login_required
    def index():
        global messages
        limit_tag = request.args.get('tag')
-
+        rss = request.args.get('rss')
-        pause_uuid = request.args.get('pause')
+        mode = request.args.get('mode')
        if pause_uuid:
            try:
                datastore.data['watching'][pause_uuid]['paused'] ^= True
                datastore.needs_write = True
                return redirect(url_for('index', tag = limit_tag))
            except KeyError:
                pass
        # Sort by last_changed and add the uuid which is usually the key..
        sorted_watches = []
@@ -255,7 +121,63 @@ def changedetection_app(config=None, datastore_o=None):
        sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)
        existing_tags = datastore.get_all_tags()
-        rss = request.args.get('rss')
+
        if mode == 'stream':
            import difflib
            import pprint
            streams = []
            extra_stylesheets = ['/static/css/diff.css']
            for watch in sorted_watches:
                if not watch['viewed']:
                    # get last two date keys
                    dates = list(watch['history'].keys())
                    # Convert to int, sort and back to str again
                    dates = [int(i) for i in dates]
                    dates.sort(reverse=True)
                    dates = [str(i) for i in dates]
                    print ("OK", watch['uuid'])
                    if len(dates) < 2:
                        print ("Skipping", watch['url'])
                        continue
                    else:
                        try:
                            path = datastore.data['watching'][watch['uuid']]['history'][str(dates[1])]
                            with open(path,
                                      encoding='utf-8') as file:
                                txt1=[line.rstrip() for line in file.readlines()]
                            path = datastore.data['watching'][watch['uuid']]['history'][str(dates[0])]
                            with open(path,
                                      encoding='utf-8') as file:
                                txt2 = [line.rstrip() for line in file.readlines()]
                        except FileNotFoundError:
                            print ("Skipping", watch['url'])
                            continue
                        df = list(difflib.unified_diff(txt1, txt2,n=1))
                        diff_entry=[]
                        for line in df:
                            if line[0] == '-' or line[0] == '+':
                                diff_entry.append(line)
                        # pprint(df)
                        #s = pprint.pformat(df)
                        streams.append(diff_entry)
            print ("###########", len(streams))
            output = render_template("watch-diff-stream.html",
                                     streams=streams,
                                     extra_stylesheets=extra_stylesheets
                                     )
            return output
        if rss:
            fg = FeedGenerator()
@@ -279,59 +201,47 @@ def changedetection_app(config=None, datastore_o=None):
            return response
        else:
-            output = render_template("watch-overview.html",
+            #table = render_template('watch-table.html', watches=sorted_watches)
            output = render_template("watch-table.html",
                                     watches=sorted_watches,
                                     messages=messages,
                                     tags=existing_tags,
                                     active_tag=limit_tag,
                                     has_unviewed=datastore.data['has_unviewed'])
            # Show messages but once.
            messages = []
        return output
    @app.route("/scrub", methods=['GET', 'POST'])
    @login_required
    def scrub_page():
        from pathlib import Path
-        import re
+        global messages
        if request.method == 'POST':
            confirmtext = request.form.get('confirmtext')
            limit_date = request.form.get('limit_date')
            try:
                limit_date = limit_date.replace('T', ' ')
                # I noticed chrome will show '/' but actually submit '-'
                limit_date = limit_date.replace('-', '/')
                # In the case that :ss seconds are supplied
                limit_date = re.sub('(\d\d:\d\d)(:\d\d)', '\\1', limit_date)
                str_to_dt = datetime.datetime.strptime(limit_date, '%Y/%m/%d %H:%M')
                limit_timestamp = int(str_to_dt.timestamp())
                if limit_timestamp > time.time():
                    flash("Timestamp is in the future, cannot continue.", 'error')
                    return redirect(url_for('scrub_page'))
            except ValueError:
                flash('Incorrect date format, cannot continue.', 'error')
                return redirect(url_for('scrub_page'))
            if confirmtext == 'scrub':
                changes_removed = 0
                for uuid, watch in datastore.data['watching'].items():
                    if limit_timestamp:
                        changes_removed += datastore.scrub_watch(uuid, limit_timestamp=limit_timestamp)
                    else:
                        changes_removed += datastore.scrub_watch(uuid)
-                flash("Cleared snapshot history ({} snapshots removed)".format(changes_removed))
+                for txt_file_path in Path(app.config['datastore_path']).rglob('*.txt'):
                    os.unlink(txt_file_path)
                for uuid, watch in datastore.data['watching'].items():
                    watch['last_checked'] = 0
                    watch['last_changed'] = 0
                    watch['previous_md5'] = None
                    watch['history'] = {}
                datastore.needs_write = True
                messages.append({'class': 'ok', 'message': 'Cleaned all version history.'})
            else:
-                flash('Incorrect confirmation text.', 'error')
+                messages.append({'class': 'error', 'message': 'Wrong confirm text.'})
            return redirect(url_for('index'))
-        output =  render_template("scrub.html")
+        return render_template("scrub.html")
        return output
    # If they edited an existing watch, we need to know to reset the current/previous md5 to include
    # the excluded text.
@@ -361,146 +271,92 @@ def changedetection_app(config=None, datastore_o=None):
        return datastore.data['watching'][uuid]['previous_md5']
    @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
    @login_required
    def edit_page(uuid):
-        from backend import forms
+        global messages
-        form = forms.watchForm(request.form)
+        import validators
        # More for testing, possible to return the first/only
        if uuid == 'first':
            uuid = list(datastore.data['watching'].keys()).pop()
-        if request.method == 'GET':
+        if request.method == 'POST':
            if not uuid in datastore.data['watching']:
                flash("No watch with the UUID %s found." % (uuid), "error")
                return redirect(url_for('index'))
-            populate_form_from_watch(form, datastore.data['watching'][uuid])
+            url = request.form.get('url').strip()
            tag = request.form.get('tag').strip()
-        if request.method == 'POST' and form.validate():
+            # Extra headers
-            update_obj = {'url': form.url.data.strip(),
+            form_headers = request.form.get('headers').strip().split("\n")
-                          'minutes_between_check': form.minutes_between_check.data,
+            extra_headers = {}
-                          'tag': form.tag.data.strip(),
+            if form_headers:
-                          'title': form.title.data.strip(),
+                for header in form_headers:
-                          'headers': form.headers.data
+                    if len(header):
                        parts = header.split(':', 1)
                        if len(parts) == 2:
                            extra_headers.update({parts[0].strip(): parts[1].strip()})
            update_obj = {'url': url,
                          'tag': tag,
                          'headers': extra_headers
                          }
            # Notification URLs
            datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data
            # Ignore text
-            form_ignore_text = form.ignore_text.data
+            form_ignore_text = request.form.get('ignore-text').strip()
-            datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
+            ignore_text = []
            if len(form_ignore_text):
                for text in form_ignore_text.split("\n"):
                    text = text.strip()
                    if len(text):
                        ignore_text.append(text)
-            # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
+                datastore.data['watching'][uuid]['ignore_text'] = ignore_text
-            if form_ignore_text:
+
                # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
                if len(datastore.data['watching'][uuid]['history']):
                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
-
+            validators.url(url)  # @todo switch to prop/attr/observer
            datastore.data['watching'][uuid]['css_filter'] = form.css_filter.data.strip()
            # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
            if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']:
                if len(datastore.data['watching'][uuid]['history']):
                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
            datastore.data['watching'][uuid].update(update_obj)
            datastore.needs_write = True
            flash("Updated watch.")
-            # Queue the watch for immediate recheck
+            messages.append({'class': 'ok', 'message': 'Updated watch.'})
            update_q.put(uuid)
            if form.trigger_check.data:
                n_object = {'watch_url': form.url.data.strip(),
                            'notification_urls': form.notification_urls.data}
                notification_q.put(n_object)
                flash('Notifications queued.')
            return redirect(url_for('index'))
        else:
-            if request.method == 'POST' and not form.validate():
+            output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], messages=messages)
                flash("An error occurred, please see below.", "error")
            output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], form=form)
        return output
    @app.route("/settings", methods=['GET', "POST"])
    @login_required
    def settings_page():
        global messages
        if request.method == 'POST':
            try:
                minutes = int(request.values.get('minutes').strip())
            except ValueError:
                messages.append({'class': 'error', 'message': "Invalid value given, use an integer."})
-        from backend import forms
+            else:
-        form = forms.globalSettingsForm(request.form)
+                if minutes >= 5:
                    datastore.data['settings']['requests']['minutes_between_check'] = minutes
                    datastore.needs_write = True
-        if request.method == 'GET':
+                    messages.append({'class': 'ok', 'message': "Updated"})
            form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check'])
            form.notification_urls.data = datastore.data['settings']['application']['notification_urls']
            # Password unset is a GET
            if request.values.get('removepassword') == 'true':
                from pathlib import Path
                datastore.data['settings']['application']['password'] = False
                flash("Password protection removed.", 'notice')
                flask_login.logout_user()
        if request.method == 'POST' and form.validate():
            datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
            datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data
            if len(form.notification_urls.data):
                import apprise
                apobj = apprise.Apprise()
                apobj.debug = True
                # Add each notification
                for n in datastore.data['settings']['application']['notification_urls']:
                    apobj.add(n)
                outcome = apobj.notify(
                    body='Hello from the worlds best and simplest web page change detection and monitoring service!',
                    title='Changedetection.io Notification Test',
                )
                if outcome:
                    flash("{} Notification URLs reached.".format(len(form.notification_urls.data)), "notice")
                else:
-                    flash("One or more Notification URLs failed", 'error')
+                    messages.append(
                        {'class': 'error', 'message': "Must be atleast 5 minutes."})
        output = render_template("settings.html", messages=messages,
                                 minutes=datastore.data['settings']['requests']['minutes_between_check'])
        messages = []
            datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
            datastore.needs_write = True
            if form.trigger_check.data:
                n_object = {'watch_url': "Test from changedetection.io!",
                            'notification_urls': form.notification_urls.data}
                notification_q.put(n_object)
                flash('Notifications queued.')
            if form.password.encrypted_password:
                datastore.data['settings']['application']['password'] = form.password.encrypted_password
                flash("Password protection enabled.", 'notice')
                flask_login.logout_user()
                return redirect(url_for('index'))
            flash("Settings updated.")
        if request.method == 'POST' and not form.validate():
            flash("An error occurred, please see below.", "error")
        output = render_template("settings.html", form=form)
        return output
    @app.route("/import", methods=['GET', "POST"])
    @login_required
    def import_page():
        import validators
        global messages
        remaining_urls = []
        good = 0
@@ -518,7 +374,7 @@ def changedetection_app(config=None, datastore_o=None):
                    if len(url):
                        remaining_urls.append(url)
-            flash("{} Imported, {} Skipped.".format(good, len(remaining_urls)))
+            messages.append({'class': 'ok', 'message': "{} Imported, {} Skipped.".format(good, len(remaining_urls))})
            if len(remaining_urls) == 0:
                # Looking good, redirect to index.
@@ -526,35 +382,37 @@ def changedetection_app(config=None, datastore_o=None):
        # Could be some remaining, or we could be on GET
        output = render_template("import.html",
                                 messages=messages,
                                 remaining="\n".join(remaining_urls)
                                 )
        messages = []
        return output
    # Clear all statuses, so we do not see the 'unviewed' class
    @app.route("/api/mark-all-viewed", methods=['GET'])
    @login_required
    def mark_all_viewed():
        # Save the current newest history as the most recently viewed
        for watch_uuid, watch in datastore.data['watching'].items():
            datastore.set_last_viewed(watch_uuid, watch['newest_history_key'])
-        flash("Cleared all statuses.")
+        messages.append({'class': 'ok', 'message': "Cleared all statuses."})
        return redirect(url_for('index'))
    @app.route("/diff/<string:uuid>", methods=['GET'])
    @login_required
    def diff_history_page(uuid):
        global messages
        # More for testing, possible to return the first/only
        if uuid == 'first':
            uuid = list(datastore.data['watching'].keys()).pop()
-        extra_stylesheets = ['/static/styles/diff.css']
+        extra_stylesheets = ['/static/css/diff.css']
        try:
            watch = datastore.data['watching'][uuid]
        except KeyError:
-            flash("No history found for the specified link, bad link?", "error")
+            messages.append({'class': 'error', 'message': "No history found for the specified link, bad link?"})
            return redirect(url_for('index'))
        dates = list(watch['history'].keys())
@@ -564,7 +422,8 @@ def changedetection_app(config=None, datastore_o=None):
        dates = [str(i) for i in dates]
        if len(dates) < 2:
-            flash("Not enough saved change detection snapshots to produce a report.", "error")
+            messages.append(
                {'class': 'error', 'message': "Not enough saved change detection snapshots to produce a report."})
            return redirect(url_for('index'))
        # Save the current newest history as the most recently viewed
@@ -586,104 +445,52 @@ def changedetection_app(config=None, datastore_o=None):
            previous_version_file_contents = f.read()
        output = render_template("diff.html", watch_a=watch,
                                 messages=messages,
                                 newest=newest_version_file_contents,
                                 previous=previous_version_file_contents,
                                 extra_stylesheets=extra_stylesheets,
                                 versions=dates[1:],
                                 uuid=uuid,
                                 newest_version_timestamp=dates[0],
                                 current_previous_version=str(previous_version),
                                 current_diff_url=watch['url'])
        return output
    @app.route("/preview/<string:uuid>", methods=['GET'])
    @login_required
    def preview_page(uuid):
        # More for testing, possible to return the first/only
        if uuid == 'first':
            uuid = list(datastore.data['watching'].keys()).pop()
        extra_stylesheets = ['/static/styles/diff.css']
        try:
            watch = datastore.data['watching'][uuid]
        except KeyError:
            flash("No history found for the specified link, bad link?", "error")
            return redirect(url_for('index'))
        newest = list(watch['history'].keys())[-1]
        with open(watch['history'][newest], 'r') as f:
            content = f.readlines()
        output = render_template("preview.html",
                                 content=content,
                                 extra_stylesheets=extra_stylesheets,
                                 current_diff_url=watch['url'],
                                 uuid=uuid)
        return output
    @app.route("/favicon.ico", methods=['GET'])
    def favicon():
        return send_from_directory("/app/static/images", filename="favicon.ico")
    # We're good but backups are even better!
    @app.route("/backup", methods=['GET'])
    @login_required
    def get_backup():
        import zipfile
        from pathlib import Path
        # Remove any existing backup file, for now we just keep one file
        for previous_backup_filename in Path(app.config['datastore_path']).rglob('changedetection-backup-*.zip'):
            os.unlink(previous_backup_filename)
        # create a ZipFile object
        backupname = "changedetection-backup-{}.zip".format(int(time.time()))
        # We only care about UUIDS from the current index file
        uuids = list(datastore.data['watching'].keys())
        backup_filepath = os.path.join(app.config['datastore_path'], backupname)
-        with zipfile.ZipFile(backup_filepath, "w",
+        with zipfile.ZipFile(os.path.join(app.config['datastore_path'], backupname), 'w',
                             compression=zipfile.ZIP_DEFLATED,
-                             compresslevel=8) as zipObj:
+                             compresslevel=6) as zipObj:
            # Be sure we're written fresh
            datastore.sync_to_json()
            # Add the index
-            zipObj.write(os.path.join(app.config['datastore_path'], "url-watches.json"), arcname="url-watches.json")
+            zipObj.write(os.path.join(app.config['datastore_path'], "url-watches.json"))
-
+            # Add any snapshot data we find
            # Add the flask app secret
            zipObj.write(os.path.join(app.config['datastore_path'], "secret.txt"), arcname="secret.txt")
            # Add any snapshot data we find, use the full path to access the file, but make the file 'relative' in the Zip.
            for txt_file_path in Path(app.config['datastore_path']).rglob('*.txt'):
                parent_p = txt_file_path.parent
                if parent_p.name in uuids:
-                    zipObj.write(txt_file_path,
+                    zipObj.write(txt_file_path)
                                 arcname=str(txt_file_path).replace(app.config['datastore_path'], ''),
                                 compress_type=zipfile.ZIP_DEFLATED,
                                 compresslevel=8)
-            # Create a list file with just the URLs, so it's easier to port somewhere else in the future
+        return send_file(os.path.join(app.config['datastore_path'], backupname),
-            list_file = os.path.join(app.config['datastore_path'], "url-list.txt")
+                         as_attachment=True,
-            with open(list_file, "w") as f:
+                         mimetype="application/zip",
-                for uuid in datastore.data['watching']:
+                         attachment_filename=backupname)
                    url = datastore.data['watching'][uuid]['url']
                    f.write("{}\r\n".format(url))
            # Add it to the Zip
            zipObj.write(list_file,
                         arcname="url-list.txt",
                         compress_type=zipfile.ZIP_DEFLATED,
                         compresslevel=8)
        return send_from_directory(app.config['datastore_path'], backupname, as_attachment=True)
    @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
    def static_content(group, filename):
@@ -697,36 +504,31 @@ def changedetection_app(config=None, datastore_o=None):
            abort(404)
    @app.route("/api/add", methods=['POST'])
    @login_required
    def api_watch_add():
-
+        global messages
        url = request.form.get('url').strip()
        if datastore.url_exists(url):
            flash('The URL {} already exists'.format(url), "error")
            return redirect(url_for('index'))
        # @todo add_watch should throw a custom Exception for validation etc
-        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
+        new_uuid = datastore.add_watch(url=request.form.get('url').strip(), tag=request.form.get('tag').strip())
        # Straight into the queue.
        update_q.put(new_uuid)
-        flash("Watch added.")
+        messages.append({'class': 'ok', 'message': 'Watch added.'})
        return redirect(url_for('index'))
    @app.route("/api/delete", methods=['GET'])
    @login_required
    def api_delete():
-
+        global messages
        uuid = request.args.get('uuid')
        datastore.delete(uuid)
-        flash('Deleted.')
+        messages.append({'class': 'ok', 'message': 'Deleted.'})
        return redirect(url_for('index'))
    @app.route("/api/checknow", methods=['GET'])
    @login_required
    def api_watch_checknow():
        global messages
        tag = request.args.get('tag')
        uuid = request.args.get('uuid')
        i = 0
@@ -746,25 +548,22 @@ def changedetection_app(config=None, datastore_o=None):
            # Items that have this current tag
            for watch_uuid, watch in datastore.data['watching'].items():
                if (tag != None and tag in watch['tag']):
-                    if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
+                    i += 1
                    if watch_uuid not in running_uuids:
                        update_q.put(watch_uuid)
                        i += 1
        else:
            # No tag, no uuid, add everything.
            for watch_uuid, watch in datastore.data['watching'].items():
-
+                i += 1
-                if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
+                if watch_uuid not in running_uuids:
                    update_q.put(watch_uuid)
-                    i += 1
+
-        flash("{} watches are rechecking.".format(i))
+        messages.append({'class': 'ok', 'message': "{} watches are rechecking.".format(i)})
        return redirect(url_for('index', tag=tag))
    # @todo handle ctrl break
    ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
    threading.Thread(target=notification_runner).start()
    # Check for new release version
    threading.Thread(target=check_for_new_version).start()
    return app
@@ -796,79 +595,74 @@ def check_for_new_version():
        # Check daily
        app.config.exit.wait(86400)
 def notification_runner():
-    while not app.config.exit.is_set():
+# Requests for checking on the site use a pool of thread Workers managed by a Queue.
-        try:
+class Worker(threading.Thread):
-            # At the moment only one thread runs (single runner)
+    current_uuid = None
            n_object = notification_q.get(block=False)
        except queue.Empty:
            time.sleep(1)
            pass
-        else:
+    def __init__(self, q, *args, **kwargs):
-            import apprise
+        self.q = q
        super().__init__(*args, **kwargs)
    def run(self):
        from backend import fetch_site_status
        update_handler = fetch_site_status.perform_site_check(datastore=datastore)
        while not app.config.exit.is_set():
            # Create an Apprise instance
            try:
-                apobj = apprise.Apprise()
+                uuid = self.q.get(block=False)
-                for url in n_object['notification_urls']:
+            except queue.Empty:
-                    apobj.add(url.strip())
+                pass
-                n_body = n_object['watch_url']
+            else:
                self.current_uuid = uuid
-                # 65 - Append URL of instance to the notification if it is set.
+                if uuid in list(datastore.data['watching'].keys()):
                base_url = os.getenv('BASE_URL')
                if base_url != None:
                    n_body += "\n" + base_url
-                apobj.notify(
+                    try:
-                    body=n_body,
+                        changed_detected, result, contents = update_handler.run(uuid)
                    # @todo This should be configurable.
                    title="ChangeDetection.io Notification - {}".format(n_object['watch_url'])
                )
-            except Exception as e:
+                    except PermissionError as s:
-                print("Watch URL: {}  Error {}".format(n_object['watch_url'],e))
+                        app.logger.error("File permission error updating", uuid, str(s))
                    else:
                        if result:
                            datastore.update_watch(uuid=uuid, update_obj=result)
                            if changed_detected:
                                # A change was detected
                                datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
                self.current_uuid = None  # Done
                self.q.task_done()
            app.config.exit.wait(1)
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
    from backend import update_worker
    # Spin up Workers.
    for _ in range(datastore.data['settings']['requests']['workers']):
-        new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
+        new_worker = Worker(update_q)
        running_update_threads.append(new_worker)
        new_worker.start()
    while not app.config.exit.is_set():
        # Get a list of watches by UUID that are currently fetching data
        running_uuids = []
        for t in running_update_threads:
-            if t.current_uuid:
+            running_uuids.append(t.current_uuid)
                running_uuids.append(t.current_uuid)
-        # Check for watches outside of the time threshold to put in the thread queue.
+        # Look at the dataset, find a stale watch to process
        # Every minute check for new UUIDs to follow up on, should be inside the loop incase it changes.
        minutes = datastore.data['settings']['requests']['minutes_between_check']
        threshold = time.time() - (minutes * 60)
        for uuid, watch in datastore.data['watching'].items():
-
+            if watch['last_checked'] <= threshold:
            # If they supplied an individual entry minutes to threshold.
            if 'minutes_between_check' in watch:
                max_time = watch['minutes_between_check'] * 60
            else:
                # Default system wide.
                max_time = datastore.data['settings']['requests']['minutes_between_check'] * 60
            threshold = time.time() - max_time
            # Yeah, put it in the queue, it's more than time.
            if not watch['paused'] and watch['last_checked'] <= threshold:
                if not uuid in running_uuids and uuid not in update_q.queue:
                    update_q.put(uuid)
        # Wait a few seconds before checking the list again
        time.sleep(3)
        # Should be low so we can break this out in testing
        app.config.exit.wait(1)
--- a/backend/fetch_site_status.py
+++ b/backend/fetch_site_status.py
@@ -13,34 +13,18 @@ class perform_site_check():
        self.datastore = datastore
    def strip_ignore_text(self, content, list_ignore_text):
        import re
        ignore = []
        ignore_regex = []
        for k in list_ignore_text:
-
+            ignore.append(k.encode('utf8'))
            # Is it a regex?
            if k[0] == '/':
                ignore_regex.append(k.strip(" /"))
            else:
                ignore.append(k)
        output = []
        for line in content.splitlines():
            line = line.encode('utf8')
            # Always ignore blank lines in this mode. (when this function gets called)
            if len(line.strip()):
-                regex_matches = False
+                if not any(skip_text in line for skip_text in ignore):
-
+                    output.append(line)
                # if any of these match, skip
                for regex in ignore_regex:
                    try:
                        if re.search(regex, line, re.IGNORECASE):
                            regex_matches = True
                    except Exception as e:
                        continue
                if not regex_matches and not any(skip_text in line for skip_text in ignore):
                    output.append(line.encode('utf8'))
        return "\n".encode('utf8').join(output)
@@ -82,36 +66,25 @@ class perform_site_check():
                             timeout=timeout,
                             verify=False)
-            # CSS Filter
+            stripped_text_from_html = get_text(r.text)
            css_filter = self.datastore.data['watching'][uuid]['css_filter']
            if css_filter and len(css_filter.strip()):
                from bs4 import BeautifulSoup
                soup = BeautifulSoup(r.content, "html.parser")
                stripped_text_from_html = ""
                for item in soup.select(css_filter):
                    text = str(item.get_text()).strip() + '\n'
                    stripped_text_from_html += text
            else:
                stripped_text_from_html = get_text(r.text)
        # Usually from networkIO/requests level
        except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
            update_obj["last_error"] = str(e)
            print(str(e))
        except requests.exceptions.MissingSchema:
            print("Skipping {} due to missing schema/bad url".format(uuid))
        # Usually from html2text level
-        except Exception as e:
+        except UnicodeDecodeError as e:
-            #        except UnicodeDecodeError as e:
+
            update_obj["last_error"] = str(e)
            print(str(e))
            # figure out how to deal with this cleaner..
            # 'utf-8' codec can't decode byte 0xe9 in position 480: invalid continuation byte
        else:
            # We rely on the actual text in the html output.. many sites have random script vars etc,
            # in the future we'll implement other mechanisms.
--- a/backend/forms.py
+++ b/backend/forms.py
@@ -1,131 +0,0 @@
 from wtforms import Form, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \
    Field
 from wtforms import widgets
 from wtforms.validators import ValidationError
 from wtforms.fields import html5
 class StringListField(StringField):
    widget = widgets.TextArea()
    def _value(self):
        if self.data:
            return "\r\n".join(self.data)
        else:
            return u''
    # incoming
    def process_formdata(self, valuelist):
        if valuelist:
            # Remove empty strings
            cleaned = list(filter(None, valuelist[0].split("\n")))
            self.data = [x.strip() for x in cleaned]
            p = 1
        else:
            self.data = []
 class SaltyPasswordField(StringField):
    widget = widgets.PasswordInput()
    encrypted_password = ""
    def build_password(self, password):
        import hashlib
        import base64
        import secrets
        # Make a new salt on every new password and store it with the password
        salt = secrets.token_bytes(32)
        key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
        store = base64.b64encode(salt + key).decode('ascii')
        return store
    # incoming
    def process_formdata(self, valuelist):
        if valuelist:
            # Be really sure it's non-zero in length
            if len(valuelist[0].strip()) > 0:
                self.encrypted_password = self.build_password(valuelist[0])
                self.data = ""
        else:
            self.data = False
 # Separated by  key:value
 class StringDictKeyValue(StringField):
    widget = widgets.TextArea()
    def _value(self):
        if self.data:
            output = u''
            for k in self.data.keys():
                output += "{}: {}\r\n".format(k, self.data[k])
            return output
        else:
            return u''
    # incoming
    def process_formdata(self, valuelist):
        if valuelist:
            self.data = {}
            # Remove empty strings
            cleaned = list(filter(None, valuelist[0].split("\n")))
            for s in cleaned:
                parts = s.strip().split(':')
                if len(parts) == 2:
                    self.data.update({parts[0].strip(): parts[1].strip()})
        else:
            self.data = {}
 class ListRegex(object):
    """
    Validates that anything that looks like a regex passes as a regex
    """
    def __init__(self, message=None):
        self.message = message
    def __call__(self, form, field):
        import re
        for line in field.data:
            if line[0] == '/' and line[-1] == '/':
                # Because internally we dont wrap in /
                line = line.strip('/')
                try:
                    re.compile(line)
                except re.error:
                    message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
                    raise ValidationError(message % (line))
 class watchForm(Form):
    # https://wtforms.readthedocs.io/en/2.3.x/fields/#module-wtforms.fields.html5
    # `require_tld` = False is needed even for the test harness "http://localhost:5005.." to run
    url = html5.URLField('URL', [validators.URL(require_tld=False)])
    tag = StringField('Tag', [validators.Optional(), validators.Length(max=35)])
    minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
                                               [validators.Optional(), validators.NumberRange(min=1)])
    css_filter = StringField('CSS Filter')
    title = StringField('Title')
    ignore_text = StringListField('Ignore Text', [ListRegex()])
    notification_urls = StringListField('Notification URL List')
    headers = StringDictKeyValue('Request Headers')
    trigger_check = BooleanField('Send test notification on save')
 class globalSettingsForm(Form):
    password = SaltyPasswordField()
    minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
                                               [validators.NumberRange(min=1)])
    notification_urls = StringListField('Notification URL List')
    trigger_check = BooleanField('Send test notification on save')
--- a/backend/pytest.ini
+++ b/backend/pytest.ini
@@ -1,7 +1,7 @@
 [pytest]
 addopts = --no-start-live-server --live-server-port=5005
 #testpaths = tests pytest_invenio
-#live_server_scope = function
+#live_server_scope = session
 filterwarnings =
    ignore::DeprecationWarning:urllib3.*:
--- a/backend/run_all_tests.sh
+++ b/backend/run_all_tests.sh
@@ -1,19 +0,0 @@
 #!/bin/bash
 # live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions
 # and I like to restart the server for each test (and have the test cleanup after each test)
 # merge request welcome :)
 # exit when any command fails
 set -e
 # Re #65 - Ability to include a link back to the installation, in the notification.
 export BASE_URL="https://foobar.com"
 find tests/test_*py -type f|while read test_name
 do
  echo "TEST RUNNING $test_name"
  pytest $test_name
 done
--- a/backend/static/styles/diff.scss
+++ b/backend/static/styles/diff.scss
@@ -1,24 +1,15 @@
-#diff-ui {
+table {
-
+	table-layout: fixed;
-    background: #fff;
+	width: 100%;
-    padding: 2em;
+}
-    margin: 1em;
+td {
-    border-radius: 5px;
+	width: 33%;
-    font-size: 9px;
+	padding: 3px 4px;
-
+	border: 1px solid transparent;
-    table {
+	vertical-align: top;
-        table-layout: fixed;
+	font: 1em monospace;
-        width: 100%;
+	text-align: left;
-    }
+	white-space: pre-wrap;
    td {
        padding: 3px 4px;
        border: 1px solid transparent;
        vertical-align: top;
        font: 1em monospace;
        text-align: left;
        white-space: pre-wrap;
    }
 }
 h1 {
 	display: inline;
@@ -42,16 +33,16 @@ ins {
 #settings {
    background: rgba(0,0,0,.05);
-    padding: 1em;
+        padding: 1em;
    border-radius: 10px;
    margin-bottom: 1em;
    color: #fff;
    font-size: 80%;
-    label {
+}
-	    margin-left: 1em;
+#settings label {
-	    display: inline-block;
+	margin-left: 1em;
-	    font-weight: normal;
+	display: inline-block;
-    }
+	font-weight: normal;
 }
 .source {
@@ -65,3 +56,11 @@ ins {
 		height: 99%; /* Hide scroll bar in Firefox */
 	}
 }
 #diff-ui {
    background: #fff;
    padding: 2em;
    margin: 1em;
    border-radius: 5px;
    font-size: 9px;
 }
--- a/backend/static/styles/pure-min.css
+++ b/backend/static/styles/pure-min.css
--- a/backend/static/css/styles.css
+++ b/backend/static/css/styles.css
@@ -0,0 +1,273 @@
 /*
 * -- BASE STYLES --
 * Most of these are inherited from Base, but I want to change a few.
 */
 body {
  color: #333;
  background: #262626;
 }
 .pure-table-even {
  background: #fff;
 }
 /* Some styles from https://css-tricks.com/ */
 a {
  text-decoration: none;
  color: #1b98f8;
 }
 a.github-link {
  color: #fff;
 }
 .pure-menu-horizontal {
  background: #fff;
  padding: 5px;
  display: flex;
  justify-content: space-between;
  border-bottom: 2px solid #ed5900;
  align-items: center;
 }
 section.content {
  padding-top: 5em;
  padding-bottom: 5em;
  flex-direction: column;
  display: flex;
  align-items: center;
  justify-content: center;
 }
 .pure-table.watch-table td {
  font-size: 80%;
 }
 /* table related */
 .watch-table {
  width: 100%;
 }
 .watch-table tr.unviewed {
    font-weight: bold;
 }
 .watch-tag-list {
  color: #e70069;
  white-space: nowrap;
 }
 .box {
  max-width: 80%;
  flex-direction: column;
  display: flex;
  justify-content: center;
 }
 .watch-table .error {
  color: #a00;
 }
 .watch-table td {
  white-space: nowrap;
 }
 .watch-table td.title-col {
  word-break: break-all;
  white-space: normal;
 }
 .watch-table th {
  white-space: nowrap;
 }
 .watch-table .title-col a[target="_blank"]::after, .current-diff-url::after {
  content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
  margin: 0 3px 0 5px;
 }
 #post-list-buttons {
  text-align: right;
  padding: 0px;
  margin: 0px;
 }
 #post-list-buttons li {
  display: inline-block;
 }
 #post-list-buttons a {
   border-top-left-radius: initial;
   border-top-right-radius: initial;
   border-bottom-left-radius: 5px;
   border-bottom-right-radius: 5px;
 }
 body:after {
  content: "";
  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)
 }
 body:after, body:before {
  display: block;
  height: 600px;
  position: absolute;
  top: 0;
  left: 0;
  width: 100%;
  z-index: -1;
 }
 body::after {
  opacity: 0.91;
 }
 body::before {
  content: "";
  background-image: url(/static/images/gradient-border.png);
 }
 body:before {
  background-size: cover
 }
 body:after, body:before {
  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
 }
 .button-small {
  font-size: 85%;
 }
 .fetch-error {
  padding-top: 1em;
  font-size: 60%;
  max-width: 400px;
  display: block;
 }
 .edit-form {
   background: #fff;
   padding: 2em;
   margin: 1em;
   border-radius: 5px;
 }
 .button-secondary {
    color: white;
    border-radius: 4px;
    text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
 }
 .button-success {
    background: rgb(28, 184, 65);
    /* this is a green */
 }
 .button-tag {
    background: rgb(99, 99, 99);
    color: #fff;
    font-size: 65%;
    border-bottom-left-radius: initial;
    border-bottom-right-radius: initial;
 }
 .button-tag.active {
    background: #9c9c9c;
    font-weight: bold;
 }
 .button-error {
    background: rgb(202, 60, 60);
    /* this is a maroon */
 }
 .button-warning {
    background: rgb(223, 117, 20);
    /* this is an orange */
 }
 .button-secondary {
    background: rgb(66, 184, 221);
    /* this is a light blue */
 }
 .button-cancel {
    background: rgb(200, 200, 200);
    /* this is a green */
 }
 .messages {
    padding: 1em;
    background: rgba(255,255,255,.2);
    border-radius: 10px;
    color: #fff;
    font-weight: bold;
 }
 .pure-form label  {
  font-weight: bold;
 }
 #new-watch-form {
    background: rgba(0,0,0,.05);
    padding: 1em;
    border-radius: 10px;
    margin-bottom: 1em;
 }
 #new-watch-form legend {
    color: #fff;
 }
 #diff-col {
    padding-left:40px;
 }
 #diff-jump {
    position: fixed;
    left: 0px;
    top: 80px;
    background: #fff;
    padding: 10px;
    border-top-right-radius: 5px;
    border-bottom-right-radius: 5px;
    box-shadow: 5px 0 5px -2px #888;
 }
 #diff-jump a {
    color: #1b98f8;
    cursor: grabbing;
    -moz-user-select: none;
    -webkit-user-select: none;
    -ms-user-select:none;
    user-select:none;
    -o-user-select:none;
 }
 footer {
    padding: 10px;
    background: #fff;
    color: #444;
    text-align: center;
 }
 #feed-icon {
    vertical-align: middle;
 }
 #version {
    position: absolute;
    top: 80px;
    right: 0px;
    font-size: 8px;
    background: #fff;
    padding: 10px;
 }
 #new-version-text a{
    color: #e07171;
 }
 #diff-stream {
    font-size: 10px;
    white-space: pre-wrap;
 }
--- a/backend/static/images/pause.svg
+++ b/backend/static/images/pause.svg
@@ -1,84 +0,0 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg
   xmlns:dc="http://purl.org/dc/elements/1.1/"
   xmlns:cc="http://creativecommons.org/ns#"
   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
   xmlns:svg="http://www.w3.org/2000/svg"
   xmlns="http://www.w3.org/2000/svg"
   version="1.1"
   id="Capa_1"
   x="0px"
   y="0px"
   viewBox="0 0 15 14.998326"
   xml:space="preserve"
   width="15"
   height="14.998326"><metadata
   id="metadata39"><rdf:RDF><cc:Work
       rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
         rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
   id="defs37" />
 <path
   id="path2"
   style="fill:#1b98f8;fill-opacity:1;stroke-width:0.0292893"
   d="M 7.4975161,6.5052867e-4 C 4.549072,-0.04028702 1.7055675,1.8548221 0.58868606,4.5801341 -0.57739762,7.2574642 0.02596981,10.583326 2.069916,12.671949 4.0364753,14.788409 7.2763651,15.56067 9.989207,14.57284 12.801145,13.617602 14.87442,10.855325 14.985833,7.8845744 15.172496,4.9966544 13.49856,2.1100704 10.911002,0.8209349 9.8598067,0.28073592 8.6791261,-0.00114855 7.4975161,6.5052867e-4 Z M 6.5602569,10.251923 c -0.00509,0.507593 -0.5693885,0.488472 -0.9352002,0.468629 -0.3399386,0.0018 -0.8402048,0.07132 -0.9297965,-0.374189 -0.015842,-1.8973128 -0.015872,-3.7979649 0,-5.6952784 0.1334405,-0.5224315 0.7416869,-0.3424086 1.1377562,-0.374189 0.3969969,-0.084515 0.8245634,0.1963256 0.7272405,0.6382917 0,1.7789118 0,3.5578239 0,5.3367357 z m 3.7490371,0 c -0.0051,0.507593 -0.5693888,0.488472 -0.9352005,0.468629 -0.3399386,0.0018 -0.8402048,0.07132 -0.9297965,-0.374189 -0.015842,-1.8973128 -0.015872,-3.7979649 0,-5.6952784 0.1334405,-0.5224315 0.7416869,-0.3424086 1.1377562,-0.374189 0.3969969,-0.084515 0.8245638,0.1963256 0.7272408,0.6382917 0,1.7789118 0,3.5578239 0,5.3367357 z" />
 <g
   id="g4"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 <g
   id="g6"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 <g
   id="g8"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 <g
   id="g10"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 <g
   id="g12"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 <g
   id="g14"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 <g
   id="g16"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 <g
   id="g18"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 <g
   id="g20"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 <g
   id="g22"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 <g
   id="g24"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 <g
   id="g26"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 <g
   id="g28"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 <g
   id="g30"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 <g
   id="g32"
   transform="translate(-0.01903604,0.02221043)">
 </g>
 </svg>
--- a/backend/static/styles/.gitignore
+++ b/backend/static/styles/.gitignore
@@ -1 +0,0 @@
 node_modules
--- a/backend/static/styles/diff.css
+++ b/backend/static/styles/diff.css
@@ -1,55 +0,0 @@
 #diff-ui {
  background: #fff;
  padding: 2em;
  margin: 1em;
  border-radius: 5px;
  font-size: 9px; }
  #diff-ui table {
    table-layout: fixed;
    width: 100%; }
  #diff-ui td {
    padding: 3px 4px;
    border: 1px solid transparent;
    vertical-align: top;
    font: 1em monospace;
    text-align: left;
    white-space: pre-wrap; }
 h1 {
  display: inline;
  font-size: 100%; }
 del {
  text-decoration: none;
  color: #b30000;
  background: #fadad7; }
 ins {
  background: #eaf2c2;
  color: #406619;
  text-decoration: none; }
 #result {
  white-space: pre-wrap; }
 #settings {
  background: rgba(0, 0, 0, 0.05);
  padding: 1em;
  border-radius: 10px;
  margin-bottom: 1em;
  color: #fff;
  font-size: 80%; }
  #settings label {
    margin-left: 1em;
    display: inline-block;
    font-weight: normal; }
 .source {
  position: absolute;
  right: 1%;
  top: .2em; }
@-moz-document url-prefix() {
  body {
    height: 99%;
    /* Hide scroll bar in Firefox */ } }
--- a/backend/static/styles/package-lock.json
+++ b/backend/static/styles/package-lock.json
--- a/backend/static/styles/package.json
+++ b/backend/static/styles/package.json
@@ -1,15 +0,0 @@
 {
  "name": "changedetection.io-theme",
  "version": "0.0.3",
  "description": "",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1",
    "scss": "node-sass --watch styles.scss diff.scss -o ."
  },
  "author": "",
  "license": "ISC",
  "dependencies": {
    "node-sass": "^6.0.0"
  }
 }
--- a/backend/static/styles/styles.css
+++ b/backend/static/styles/styles.css
@@ -1,313 +0,0 @@
 /*
 * -- BASE STYLES --
 * Most of these are inherited from Base, but I want to change a few.
 * npm run scss
 */
 body {
  color: #333;
  background: #262626; }
 .pure-table-even {
  background: #fff; }
 /* Some styles from https://css-tricks.com/ */
 a {
  text-decoration: none;
  color: #1b98f8; }
 a.github-link {
  color: #fff; }
 .pure-menu-horizontal {
  background: #fff;
  padding: 5px;
  display: flex;
  justify-content: space-between;
  border-bottom: 2px solid #ed5900;
  align-items: center; }
 section.content {
  padding-top: 5em;
  padding-bottom: 5em;
  flex-direction: column;
  display: flex;
  align-items: center;
  justify-content: center; }
 /* table related */
 .watch-table {
  width: 100%; }
  .watch-table tr.unviewed {
    font-weight: bold; }
  .watch-table .error {
    color: #a00; }
  .watch-table td {
    font-size: 80%;
    white-space: nowrap; }
  .watch-table td.title-col {
    word-break: break-all;
    white-space: normal; }
  .watch-table th {
    white-space: nowrap; }
  .watch-table .title-col a[target="_blank"]::after, .watch-table .current-diff-url::after {
    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
    margin: 0 3px 0 5px; }
 .watch-tag-list {
  color: #e70069;
  white-space: nowrap; }
 .box {
  max-width: 80%;
  flex-direction: column;
  display: flex;
  justify-content: center; }
 #post-list-buttons {
  text-align: right;
  padding: 0px;
  margin: 0px; }
  #post-list-buttons li {
    display: inline-block; }
  #post-list-buttons a {
    border-top-left-radius: initial;
    border-top-right-radius: initial;
    border-bottom-left-radius: 5px;
    border-bottom-right-radius: 5px; }
 body:after {
  content: "";
  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%); }
 body:after, body:before {
  display: block;
  height: 600px;
  position: absolute;
  top: 0;
  left: 0;
  width: 100%;
  z-index: -1; }
 body::after {
  opacity: 0.91; }
 body::before {
  content: "";
  background-image: url(/static/images/gradient-border.png); }
 body:before {
  background-size: cover; }
 body:after, body:before {
  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%); }
 .button-small {
  font-size: 85%; }
 .fetch-error {
  padding-top: 1em;
  font-size: 60%;
  max-width: 400px;
  display: block; }
 .edit-form {
  background: #fff;
  padding: 2em;
  margin: 1em;
  border-radius: 5px; }
 .button-secondary {
  color: white;
  border-radius: 4px;
  text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2); }
 .button-success {
  background: #1cb841;
  /* this is a green */ }
 .button-tag {
  background: #636363;
  color: #fff;
  font-size: 65%;
  border-bottom-left-radius: initial;
  border-bottom-right-radius: initial; }
  .button-tag.active {
    background: #9c9c9c;
    font-weight: bold; }
 .button-error {
  background: #ca3c3c;
  /* this is a maroon */ }
 .button-warning {
  background: #df7514;
  /* this is an orange */ }
 .button-secondary {
  background: #42b8dd;
  /* this is a light blue */ }
 .button-cancel {
  background: #c8c8c8;
  /* this is a green */ }
 .messages li {
  list-style: none;
  padding: 1em;
  border-radius: 10px;
  color: #fff;
  font-weight: bold; }
  .messages li.message {
    background: rgba(255, 255, 255, 0.2); }
  .messages li.error {
    background: rgba(255, 1, 1, 0.5); }
  .messages li.notice {
    background: rgba(255, 255, 255, 0.5); }
 #new-watch-form {
  background: rgba(0, 0, 0, 0.05);
  padding: 1em;
  border-radius: 10px;
  margin-bottom: 1em; }
 #new-watch-form legend {
  color: #fff; }
 #new-watch-form input {
  width: auto !important; }
 #diff-col {
  padding-left: 40px; }
 #diff-jump {
  position: fixed;
  left: 0px;
  top: 80px;
  background: #fff;
  padding: 10px;
  border-top-right-radius: 5px;
  border-bottom-right-radius: 5px;
  box-shadow: 5px 0 5px -2px #888; }
 #diff-jump a {
  color: #1b98f8;
  cursor: grabbing;
  -moz-user-select: none;
  -webkit-user-select: none;
  -ms-user-select: none;
  user-select: none;
  -o-user-select: none; }
 footer {
  padding: 10px;
  background: #fff;
  color: #444;
  text-align: center; }
 #feed-icon {
  vertical-align: middle; }
 #version {
  position: absolute;
  top: 80px;
  right: 0px;
  font-size: 8px;
  background: #fff;
  padding: 10px; }
 #new-version-text a {
  color: #e07171; }
 .paused-state.state-False img {
  opacity: 0.2; }
 .paused-state.state-False:hover img {
  opacity: 0.8; }
 .monospaced-textarea textarea {
  width: 100%;
  font-family: monospace;
  white-space: pre;
  overflow-wrap: normal;
  overflow-x: scroll; }
 .pure-form {
  /* The input fields with errors */
  /* The list of errors */ }
  .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
    padding-bottom: 1em; }
    .pure-form .pure-control-group dd, .pure-form .pure-group dd, .pure-form .pure-controls dd {
      margin: 0px; }
  .pure-form .error input {
    background-color: #ffebeb; }
  .pure-form ul.errors {
    padding: .5em .6em;
    border: 1px solid #dd0000;
    border-radius: 4px;
    vertical-align: middle;
    -webkit-box-sizing: border-box;
    box-sizing: border-box; }
    .pure-form ul.errors li {
      margin-left: 1em;
      color: #dd0000; }
  .pure-form label {
    font-weight: bold; }
  .pure-form input[type=url] {
    width: 100%; }
  .pure-form textarea {
    width: 100%;
    font-size: 14px; }
@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
  .box {
    max-width: 95%; }
  .edit-form {
    padding: 0.5em;
    margin: 0.5em; }
  #nav-menu {
    overflow-x: scroll; } }
 /*
 Max width before this PARTICULAR table gets nasty
 This query will take effect for any screen smaller than 760px
 and also iPads specifically.
 */
@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
  .watch-table {
    /* Force table to not be like tables anymore */
    /* Force table to not be like tables anymore */
    /* Hide table headers (but not display: none;, for accessibility) */ }
    .watch-table thead, .watch-table tbody, .watch-table th, .watch-table td, .watch-table tr {
      display: block; }
    .watch-table .last-checked::before {
      color: #555;
      content: "Last Checked "; }
    .watch-table .last-changed::before {
      color: #555;
      content: "Last Changed "; }
    .watch-table td.inline {
      display: inline-block; }
    .watch-table thead tr {
      position: absolute;
      top: -9999px;
      left: -9999px; }
    .watch-table .pure-table td, .watch-table .pure-table th {
      border: none; }
    .watch-table td {
      /* Behave  like a "row" */
      border: none;
      border-bottom: 1px solid #eee; }
      .watch-table td:before {
        /* Top/left values mimic padding */
        top: 6px;
        left: 6px;
        width: 45%;
        padding-right: 10px;
        white-space: nowrap; }
    .watch-table.pure-table-striped tr {
      background-color: #fff; }
    .watch-table.pure-table-striped tr:nth-child(2n-1) {
      background-color: #eee; }
    .watch-table.pure-table-striped tr:nth-child(2n-1) td {
      background-color: inherit; } }
--- a/backend/static/styles/styles.scss
+++ b/backend/static/styles/styles.scss
@@ -1,439 +0,0 @@
 /*
 * -- BASE STYLES --
 * Most of these are inherited from Base, but I want to change a few.
 * npm run scss
 */
 body {
  color: #333;
  background: #262626;
 }
 .pure-table-even {
  background: #fff;
 }
 /* Some styles from https://css-tricks.com/ */
 a {
  text-decoration: none;
  color: #1b98f8;
 }
 a.github-link {
  color: #fff;
 }
 .pure-menu-horizontal {
  background: #fff;
  padding: 5px;
  display: flex;
  justify-content: space-between;
  border-bottom: 2px solid #ed5900;
  align-items: center;
 }
 section.content {
  padding-top: 5em;
  padding-bottom: 5em;
  flex-direction: column;
  display: flex;
  align-items: center;
  justify-content: center;
 }
 /* table related */
 .watch-table {
  width: 100%;
  tr.unviewed {
    font-weight: bold;
  }
  .error {
    color: #a00;
  }
  td {
    font-size: 80%;
    white-space: nowrap;
  }
  td.title-col {
    word-break: break-all;
    white-space: normal;
  }
  th {
    white-space: nowrap;
  }
  .title-col a[target="_blank"]::after, .current-diff-url::after {
    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
    margin: 0 3px 0 5px;
  }
 }
 .watch-tag-list {
  color: #e70069;
  white-space: nowrap;
 }
 .box {
  max-width: 80%;
  flex-direction: column;
  display: flex;
  justify-content: center;
 }
 #post-list-buttons {
  text-align: right;
  padding: 0px;
  margin: 0px;
  li {
    display: inline-block;
  }
  a {
    border-top-left-radius: initial;
    border-top-right-radius: initial;
    border-bottom-left-radius: 5px;
    border-bottom-right-radius: 5px;
  }
 }
 body:after {
  content: "";
  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)
 }
 body:after, body:before {
  display: block;
  height: 600px;
  position: absolute;
  top: 0;
  left: 0;
  width: 100%;
  z-index: -1;
 }
 body::after {
  opacity: 0.91;
 }
 body::before {
  content: "";
  background-image: url(/static/images/gradient-border.png);
 }
 body:before {
  background-size: cover
 }
 body:after, body:before {
  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
 }
 .button-small {
  font-size: 85%;
 }
 .fetch-error {
  padding-top: 1em;
  font-size: 60%;
  max-width: 400px;
  display: block;
 }
 .edit-form {
  background: #fff;
  padding: 2em;
  margin: 1em;
  border-radius: 5px;
 }
 .button-secondary {
  color: white;
  border-radius: 4px;
  text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
 }
 .button-success {
  background: rgb(28, 184, 65);
  /* this is a green */
 }
 .button-tag {
  background: rgb(99, 99, 99);
  color: #fff;
  font-size: 65%;
  border-bottom-left-radius: initial;
  border-bottom-right-radius: initial;
  &.active {
    background: #9c9c9c;
    font-weight: bold;
  }
 }
 .button-error {
  background: rgb(202, 60, 60);
  /* this is a maroon */
 }
 .button-warning {
  background: rgb(223, 117, 20);
  /* this is an orange */
 }
 .button-secondary {
  background: rgb(66, 184, 221);
  /* this is a light blue */
 }
 .button-cancel {
  background: rgb(200, 200, 200);
  /* this is a green */
 }
 .messages {
    li {
        list-style: none;
        padding: 1em;
        border-radius: 10px;
        color: #fff;
        font-weight: bold;
        &.message {
            background: rgba(255, 255, 255, .2);
        }
        &.error {
            background: rgba(255, 1, 1, .5);
        }
        &.notice {
            background: rgba(255, 255, 255, .5);
        }
    }
 }
 #new-watch-form {
  background: rgba(0, 0, 0, .05);
  padding: 1em;
  border-radius: 10px;
  margin-bottom: 1em;
 }
 #new-watch-form legend {
  color: #fff;
 }
 #new-watch-form input {
  width: auto !important;
 }
 #diff-col {
  padding-left: 40px;
 }
 #diff-jump {
  position: fixed;
  left: 0px;
  top: 80px;
  background: #fff;
  padding: 10px;
  border-top-right-radius: 5px;
  border-bottom-right-radius: 5px;
  box-shadow: 5px 0 5px -2px #888;
 }
 #diff-jump a {
  color: #1b98f8;
  cursor: grabbing;
  -moz-user-select: none;
  -webkit-user-select: none;
  -ms-user-select: none;
  user-select: none;
  -o-user-select: none;
 }
 footer {
  padding: 10px;
  background: #fff;
  color: #444;
  text-align: center;
 }
 #feed-icon {
  vertical-align: middle;
 }
 #version {
  position: absolute;
  top: 80px;
  right: 0px;
  font-size: 8px;
  background: #fff;
  padding: 10px;
 }
 #new-version-text a {
  color: #e07171;
 }
 .paused-state {
  &.state-False img {
    opacity: 0.2;
  }
  &.state-False:hover img {
    opacity: 0.8;
  }
 }
 .monospaced-textarea {
    textarea {
        width: 100%;
        font-family: monospace;
        white-space: pre;
        overflow-wrap: normal;
        overflow-x: scroll;
    }
 }
 .pure-form {
    .pure-control-group, .pure-group, .pure-controls {
        padding-bottom: 1em;
        dd {
            margin: 0px;
        }
    }
  /* The input fields with errors */
  .error {
    input {
        background-color: #ffebeb;
    }
  }
   /* The list of errors */
  ul.errors {
    padding: .5em .6em;
    border: 1px solid #dd0000;
    border-radius: 4px;
    vertical-align: middle;
    -webkit-box-sizing: border-box;
    box-sizing: border-box;
    li {
        margin-left: 1em;
        color: #dd0000;
    }
  }
  label {
    font-weight: bold;
  }
  input[type=url] {
    width: 100%;
  }
  textarea {
    width: 100%;
    font-size: 14px;
  }
 }
@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
  .box {
    max-width: 95%
  }
  .edit-form {
    padding: 0.5em;
    margin: 0.5em;
  }
  #nav-menu {
    overflow-x: scroll;
  }
 }
 /*
 Max width before this PARTICULAR table gets nasty
 This query will take effect for any screen smaller than 760px
 and also iPads specifically.
 */
@media only screen and (max-width: 760px),
 (min-device-width: 768px) and (max-device-width: 1024px) {
  .watch-table {
    /* Force table to not be like tables anymore */
    thead, tbody, th, td, tr {
      display: block;
    }
    .last-checked::before {
      color: #555;
      content: "Last Checked ";
    }
    .last-changed::before {
      color: #555;
      content: "Last Changed ";
    }
    /* Force table to not be like tables anymore */
    td.inline {
      display: inline-block;
    }
    /* Hide table headers (but not display: none;, for accessibility) */
    thead tr {
      position: absolute;
      top: -9999px;
      left: -9999px;
    }
    .pure-table td, .pure-table th {
      border: none;
    }
    td {
      /* Behave  like a "row" */
      border: none;
      border-bottom: 1px solid #eee;
      &:before {
        /* Top/left values mimic padding */
        top: 6px;
        left: 6px;
        width: 45%;
        padding-right: 10px;
        white-space: nowrap;
      }
    }
    &.pure-table-striped {
      tr {
        background-color: #fff;
      }
      tr:nth-child(2n-1) {
        background-color: #eee;
      }
      tr:nth-child(2n-1) td {
        background-color: inherit;
      }
    }
  }
 }
--- a/backend/store.py
+++ b/backend/store.py
@@ -1,7 +1,9 @@
 from os import unlink, path, mkdir
 import json
 import uuid as uuid_builder
 import os.path
 from os import path
 from threading import Lock
 from copy import deepcopy
 import logging
@@ -35,10 +37,6 @@ class ChangeDetectionStore:
                    'timeout': 15,  # Default 15 seconds
                    'minutes_between_check': 3 * 60,  # Default 3 hours
                    'workers': 10  # Number of threads, lower is better for slow connections
                },
                'application': {
                    'password': False,
                    'notification_urls': [] # Apprise URL list
                }
            }
        }
@@ -49,22 +47,18 @@ class ChangeDetectionStore:
            'tag': None,
            'last_checked': 0,
            'last_changed': 0,
            'paused': False,
            'last_viewed': 0,  # history key value of the last viewed via the [diff] link
            'newest_history_key': "",
            'title': None,
            'minutes_between_check': 3 * 60,  # Default 3 hours
            'previous_md5': "",
            'uuid': str(uuid_builder.uuid4()),
            'headers': {},  # Extra headers to send
            'history': {},  # Dict of timestamp and output stripped filename
-            'ignore_text': [], # List of text to ignore when calculating the comparison checksum
+            'ignore_text': [] # List of text to ignore when calculating the comparison checksum
            'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise)
            'css_filter': "",
        }
-        if path.isfile('backend/source.txt'):
+        if path.isfile('/source.txt'):
-            with open('backend/source.txt') as f:
+            with open('/source.txt') as f:
                # Should be set in Dockerfile to look for /source.txt , this will give us the git commit #
                # So when someone gives us a backup file to examine, we know exactly what code they were running.
                self.__data['build_sha'] = f.read()
@@ -89,9 +83,6 @@ class ChangeDetectionStore:
                    if 'requests' in from_disk['settings']:
                        self.__data['settings']['requests'].update(from_disk['settings']['requests'])
                    if 'application' in from_disk['settings']:
                        self.__data['settings']['application'].update(from_disk['settings']['application'])
                # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
                # @todo pretty sure theres a python we todo this with an abstracted(?) object!
                for uuid, watch in self.__data['watching'].items():
@@ -111,21 +102,11 @@ class ChangeDetectionStore:
                self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
                self.add_watch(url='https://changedetection.io', tag='Tech news')
        self.__data['version_tag'] = "0.36"
-        # Helper to remove password protection
+        self.__data['version_tag'] = "0.27"
        password_reset_lockfile = "{}/removepassword.lock".format(self.datastore_path)
        if path.isfile(password_reset_lockfile):
            self.__data['settings']['application']['password'] = False
            unlink(password_reset_lockfile)
        if not 'app_guid' in self.__data:
-            import sys
+            self.__data['app_guid'] = str(uuid_builder.uuid4())
            import os
            if "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ:
                self.__data['app_guid'] = "test-" + str(uuid_builder.uuid4())
            else:
                self.__data['app_guid'] = str(uuid_builder.uuid4())
        self.needs_write = True
@@ -153,10 +134,6 @@ class ChangeDetectionStore:
    def update_watch(self, uuid, update_obj):
        # Skip if 'paused' state
        if self.__data['watching'][uuid]['paused']:
            return
        with self.lock:
            # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
@@ -175,6 +152,7 @@ class ChangeDetectionStore:
    def data(self):
        has_unviewed = False
        for uuid, v in self.__data['watching'].items():
            self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
            if int(v['newest_history_key']) <= int(v['last_viewed']):
@@ -201,35 +179,19 @@ class ChangeDetectionStore:
        tags.sort()
        return tags
    def unlink_history_file(self, path):
        try:
            unlink(path)
        except (FileNotFoundError, IOError):
            pass
    # Delete a single watch by UUID
    def delete(self, uuid):
        with self.lock:
            if uuid == 'all':
                self.__data['watching'] = {}
                # GitHub #30 also delete history records
                for uuid in self.data['watching']:
                    for path in self.data['watching'][uuid]['history'].values():
                        self.unlink_history_file(path)
            else:
-                for path in self.data['watching'][uuid]['history'].values():
+                del (self.__data['watching'][uuid])
                    self.unlink_history_file(path)
                del self.data['watching'][uuid]
            self.needs_write = True
    def url_exists(self, url):
        # Probably their should be dict...
-        for watch in self.data['watching'].values():
+        for watch in self.data['watching']:
            if watch['url'] == url:
                return True
@@ -239,48 +201,6 @@ class ChangeDetectionStore:
        # Probably their should be dict...
        return self.data['watching'][uuid].get(val)
    # Remove a watchs data but keep the entry (URL etc)
    def scrub_watch(self, uuid, limit_timestamp = False):
        import hashlib
        del_timestamps = []
        changes_removed = 0
        for timestamp, path in self.data['watching'][uuid]['history'].items():
            if not limit_timestamp or (limit_timestamp is not False and int(timestamp) > limit_timestamp):
                self.unlink_history_file(path)
                del_timestamps.append(timestamp)
                changes_removed += 1
                if not limit_timestamp:
                    self.data['watching'][uuid]['last_checked'] = 0
                    self.data['watching'][uuid]['last_changed'] = 0
                    self.data['watching'][uuid]['previous_md5'] = 0
        for timestamp in del_timestamps:
            del self.data['watching'][uuid]['history'][str(timestamp)]
            # If there was a limitstamp, we need to reset some meta data about the entry
            # This has to happen after we remove the others from the list
            if limit_timestamp:
                newest_key = self.get_newest_history_key(uuid)
                if newest_key:
                    self.data['watching'][uuid]['last_checked'] = int(newest_key)
                    # @todo should be the original value if it was less than newest key
                    self.data['watching'][uuid]['last_changed'] = int(newest_key)
                    try:
                        with open(self.data['watching'][uuid]['history'][str(newest_key)], "rb") as fp:
                            content = fp.read()
                        self.data['watching'][uuid]['previous_md5'] = hashlib.md5(content).hexdigest()
                    except (FileNotFoundError, IOError):
                        self.data['watching'][uuid]['previous_md5'] = False
                        pass
        self.needs_write = True
        return changes_removed
    def add_watch(self, url, tag):
        with self.lock:
            # @todo use a common generic version of this
@@ -297,7 +217,7 @@ class ChangeDetectionStore:
        # Get the directory ready
        output_path = "{}/{}".format(self.datastore_path, new_uuid)
        try:
-            mkdir(output_path)
+            os.mkdir(output_path)
        except FileExistsError:
            print(output_path, "already exists.")
@@ -308,6 +228,12 @@ class ChangeDetectionStore:
    # result_obj from fetch_site_status.run()
    def save_history_text(self, uuid, result_obj, contents):
        output_path = "{}/{}".format(self.datastore_path, uuid)
        try:
            os.mkdir(output_path)
        except FileExistsError:
            pass
        output_path = "{}/{}".format(self.datastore_path, uuid)
        fname = "{}/{}-{}.stripped.txt".format(output_path, result_obj['previous_md5'], str(time.time()))
        with open(fname, 'w') as f:
@@ -322,21 +248,11 @@ class ChangeDetectionStore:
    def sync_to_json(self):
        print("Saving..")
-        data ={}
+        with open(self.json_store_path, 'w') as json_file:
            json.dump(self.__data, json_file, indent=4)
            logging.info("Re-saved index")
-        try:
+        self.needs_write = False
            data = deepcopy(self.__data)
        except RuntimeError:
            time.sleep(0.5)
            print ("! Data changed when writing to JSON, trying again..")
            self.sync_to_json()
            return
        else:
            with open(self.json_store_path, 'w') as json_file:
                json.dump(data, json_file, indent=4)
                logging.info("Re-saved index")
            self.needs_write = False
    # Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON
    # by just running periodically in one thread, according to python, dict updates are threadsafe.
@@ -346,24 +262,8 @@ class ChangeDetectionStore:
            if self.stop_thread:
                print("Shutting down datastore thread")
                return
            if self.needs_write:
                self.sync_to_json()
-            time.sleep(3)
+            time.sleep(1)
-    # Go through the datastore path and remove any snapshots that are not mentioned in the index
+# body of the constructor
    # This usually is not used, but can be handy.
    def remove_unused_snapshots(self):
        print ("Removing snapshots from datastore that are not in the index..")
        index=[]
        for uuid in self.data['watching']:
            for id in self.data['watching'][uuid]['history']:
                index.append(self.data['watching'][uuid]['history'][str(id)])
        import pathlib
        # Only in the sub-directories
        for item in pathlib.Path(self.datastore_path).rglob("*/*txt"):
            if not str(item) in index:
                print ("Removing",item)
                unlink(item)
--- a/backend/templates/_helpers.jinja
+++ b/backend/templates/_helpers.jinja
@@ -1,12 +0,0 @@
 {% macro render_field(field) %}
  <dt {% if field.errors %} class="error" {% endif %}>{{ field.label }}
  <dd {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
  {% if field.errors %}
    <ul class=errors>
    {% for error in field.errors %}
      <li>{{ error }}</li>
    {% endfor %}
    </ul>
  {% endif %}
  </dd>
 {% endmacro %}
--- a/backend/templates/base.html
+++ b/backend/templates/base.html
@@ -5,8 +5,8 @@
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <meta name="description" content="Self hosted website change detection.">
    <title>Change Detection</title>
-    <link rel="stylesheet" href="/static/styles/pure-min.css">
+    <link rel="stylesheet" href="/static/css/pure-min.css">
-    <link rel="stylesheet" href="/static/styles/styles.css?ver=1000">
+    <link rel="stylesheet" href="/static/css/styles.css?ver=1000">
    {% if extra_stylesheets %}
        {% for m in extra_stylesheets %}
        <link rel="stylesheet" href="{{ m }}?ver=1000">
@@ -16,13 +16,8 @@
 <body>
 <div class="header">
-
+    <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed">
-    <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
+        <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
        {% if has_password and not current_user.is_authenticated %}
            <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener"><strong>Change</strong>Detection.io</a>
        {% else %}
            <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
        {% endif %}
        {% if current_diff_url %}
        <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</span></a>
        {% else %}
@@ -32,8 +27,7 @@
        {% endif %}
        <ul class="pure-menu-list">
-        {% if current_user.is_authenticated or not has_password %}
+
            {% if not current_diff_url %}
            <li class="pure-menu-item">
                <a href="/backup" class="pure-menu-link">BACKUP</a>
            </li>
@@ -43,20 +37,6 @@
            <li class="pure-menu-item">
                <a href="/settings" class="pure-menu-link">SETTINGS</a>
            </li>
            {% else %}
            <li class="pure-menu-item">
                <a href="{{ url_for('edit_page', uuid=uuid) }}" class="pure-menu-link">EDIT</a>
            </li>
            {% endif %}
        {% else %}
            <li class="pure-menu-item">
                <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
            </li>
        {% endif %}
        {% if current_user.is_authenticated %}
            <li class="pure-menu-item"><a href="/logout" class="pure-menu-link">LOG OUT</a></li>
        {% endif %}
            <li class="pure-menu-item"><a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
                <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16"
                     version="1.1"
@@ -65,6 +45,10 @@
                          d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
                </svg>
            </a></li>
            <!--
            <li class="pure-menu-item"><a href="#" class="pure-menu-link">Tour</a></li>
            <li class="pure-menu-item"><a href="#" class="pure-menu-link">Sign Up</a></li>
            -->
        </ul>
    </div>
 </div>
@@ -74,15 +58,15 @@
        {% block header %}{% endblock %}
    </header>
-    {% with messages = get_flashed_messages(with_categories=true) %}
+    {% if messages %}
-      {% if messages %}
+    <div class="messages">
-        <ul class=messages>
+        {% for message in messages %}
-        {% for category, message in messages %}
+        <div class="flash-message {{ message['class'] }}">{{ message['message'] }}</div>
          <li class="{{ category }}">{{ message }}</li>
        {% endfor %}
-        </ul>
+    </div>
-      {% endif %}
+    {% endif %}
-    {% endwith %}
+
    {% block content %}
    {% endblock %}
--- a/backend/templates/diff.html
+++ b/backend/templates/diff.html
@@ -8,7 +8,7 @@
        <fieldset>
            <label for="diffWords" class="pure-checkbox">
-                <input type="radio" name="diff_type" id="diffWords" value="diffWords"/> Words</label>
+                <input type="radio" name="diff_type" id="diffWords" value="diffWords" /> Words</label>
            <label for="diffLines" class="pure-checkbox">
                <input type="radio" name="diff_type" id="diffLines" value="diffLines" checked=""/> Lines</label>
@@ -19,9 +19,9 @@
            <label for="diff-version">Compare newest (<span id="current-v-date"></span>) with</label>
            <select id="diff-version" name="previous_version">
                {% for version in versions %}
-                <option value="{{version}}" {% if version== current_previous_version %} selected="" {% endif %}>
+                    <option value="{{version}}" {% if version== current_previous_version %} selected="" {% endif %}>
-                    {{version}}
+                        {{version}}
-                </option>
+                    </option>
                {% endfor %}
            </select>
            <button type="submit" class="pure-button pure-button-primary">Go</button>
@@ -30,13 +30,13 @@
    </form>
    <del>Removed text</del>
    <ins>Inserted Text</ins>
    <a href="{{ url_for('preview_page', uuid=uuid) }}">Show current snapshot</a>
 </div>
 <div id="diff-jump">
    <a onclick="next_diff();">Jump</a>
 </div>
 <div id="diff-ui">
    <table>
        <tbody>
        <tr>
@@ -90,10 +90,6 @@ function changed() {
 	result.textContent = '';
 	result.appendChild(fragment);
 	// Jump at start
 	inputs.current=0;
    next_diff();
 }
 window.onload = function() {
@@ -116,7 +112,6 @@ window.onload = function() {
 	onDiffTypeChange(document.querySelector('#settings [name="diff_type"]:checked'));
 	changed();
 };
 a.onpaste = a.onchange =
@@ -145,7 +140,6 @@ for (var i = 0; i < radio.length; i++) {
 var inputs = document.getElementsByClassName('change');
 inputs.current=0;
 function next_diff() {
    var element = inputs[inputs.current];
@@ -165,7 +159,6 @@ function next_diff() {
 }
 </script>
--- a/backend/templates/edit.html
+++ b/backend/templates/edit.html
@@ -1,67 +1,69 @@
 {% extends 'base.html' %}
 {% block content %}
-{% from '_helpers.jinja' import render_field %}
+<div class="edit-form">
-<div class="edit-form monospaced-textarea">
+
    <form class="pure-form pure-form-stacked" action="/edit/{{uuid}}" method="POST">
        <fieldset>
            <div class="pure-control-group">
-                {{ render_field(form.url, placeholder="https://...", size=30, required=true) }}
+                <label for="url">URL</label>
                <input type="url" id="url" required="" placeholder="https://..." name="url" value="{{ watch.url}}"
                       size="50"/>
                <span class="pure-form-message-inline">This is a required field.</span>
            </div>
            <div class="pure-control-group">
-                {{ render_field(form.title, size=30) }}
+                <label for="tag">Tag</label>
-            </div>
+                <input type="text" placeholder="tag" size="10" id="tag" name="tag" value="{{ watch.tag}}"/>
-            <div class="pure-control-group">
+                <span class="pure-form-message-inline">Grouping tags, can be a comma separated list.</span>
                {{ render_field(form.tag, size=10) }}
            </div>
            <div class="pure-control-group">
                {{ render_field(form.minutes_between_check, size=5) }}
            </div>
            <div class="pure-control-group">
                {{ render_field(form.css_filter, size=25, placeholder=".class-name or #some-id, or other CSS selector rule.") }}
                <span class="pure-form-message-inline">Limit text to this CSS rule, only text matching this CSS rule is included.<br/>
                    Please be sure that you thoroughly understand how to write CSS selector rules before filing an issue on GitHub!<br/>
                    Go <a href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>
                </span>
            </div>
            <!-- @todo: move to tabs --->
            <fieldset class="pure-group">
-                {{ render_field(form.ignore_text, rows=5,  placeholder="Some text to ignore in a line
+                <label for="ignore-text">Ignore text</label>
-/some.regex\d{2}/   for case-INsensitive regex
+
-") }}
+                <textarea id="ignore-text" name="ignore-text" class="pure-input-1-2" placeholder=""
-                <span class="pure-form-message-inline">
+                          style="width: 100%;
-                    Each line processed separately, any line matching will be ignored.<br/>
+                            font-family:monospace;
-                    Regular Expression support, wrap the line in forward slash <b>/regex/</b>.
+                            white-space: pre;
-                </span>
+                            overflow-wrap: normal;
                            overflow-x: scroll;" rows="5">{% for value in watch.ignore_text %}{{ value }}
 {% endfor %}</textarea>
                <span class="pure-form-message-inline">Each line will be processed separately as an ignore rule.</span>
            </fieldset>
            <!-- @todo: move to tabs --->
            <fieldset class="pure-group">
-                {{ render_field(form.headers, rows=5, placeholder="Example
+                <label for="headers">Extra request headers</label>
                <textarea id="headers" name="headers" class="pure-input-1-2" placeholder="Example
 Cookie: foobar
-User-Agent: wonderbra 1.0") }}
+User-Agent: wonderbra 1.0"
                          style="width: 100%;
                            font-family:monospace;
                            white-space: pre;
                            overflow-wrap: normal;
                            overflow-x: scroll;" rows="5">{% for key, value in watch.headers.items() %}{{ key }}: {{ value }}
 {% endfor %}</textarea>
                <br/>
            </fieldset>
            <div class="pure-control-group">
                {{ render_field(form.notification_urls, rows=5, placeholder="Gitter - gitter://token/room
 Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
 AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
 SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
 ") }}
                <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
            </div>
            <div class="pure-controls">
                {{ render_field(form.trigger_check, rows=5) }}
            </div>
            <div class="pure-control-group">
                <button type="submit" class="pure-button pure-button-primary">Save</button>
            </div>
            <br/>
            <div class="pure-control-group">
                <a href="/" class="pure-button button-small button-cancel">Cancel</a>
                <a href="/api/delete?uuid={{uuid}}"
                   class="pure-button button-small button-error ">Delete</a>
            </div>
        </fieldset>
    </form>
--- a/backend/templates/login.html
+++ b/backend/templates/login.html
@@ -1,20 +0,0 @@
 {% extends 'base.html' %}
 {% block content %}
 <div class="edit-form">
    <form class="pure-form pure-form-stacked" action="/login" method="POST">
        <fieldset>
            <div class="pure-control-group">
                <label for="password">Password</label>
                <input type="password" id="password" required="" name="password" value=""
                       size="15"/>
                <input type="hidden" id="email" name="email" value="defaultuser@changedetection.io" />
            </div>
            <div class="pure-control-group">
                <button type="submit" class="pure-button pure-button-primary">Login</button>
            </div>
        </fieldset>
    </form>
 </div>
 {% endblock %}
--- a/backend/templates/preview.html
+++ b/backend/templates/preview.html
@@ -1,26 +0,0 @@
 {% extends 'base.html' %}
 {% block content %}
 <div id="settings">
    <h1>Current</h1>
 </div>
 <div id="diff-ui">
    <table>
        <tbody>
        <tr>
            <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
            <td id="diff-col">
                <span id="result">{% for row in content %}<pre>{{row}}</pre>{% endfor %}</span>
            </td>
        </tr>
        </tbody>
    </table>
 </div>
 {% endblock %}
--- a/backend/templates/scrub.html
+++ b/backend/templates/scrub.html
@@ -2,25 +2,29 @@
 {% block content %}
 <div class="edit-form">
    <form class="pure-form pure-form-stacked" action="/scrub" method="POST">
        <fieldset>
            <div class="pure-control-group">
                This will remove all version snapshots/data, but keep your list of URLs. <br/>
                You may like to use the <strong>BACKUP</strong> link first.<br/>
                Type in the word <strong>scrub</strong> to confirm that you understand!
                <br/>
            </div>
-            <br/>
+
            <div class="pure-control-group">
-                <label for="confirmtext">Confirmation text</label>
+                <br/>
                <label for="confirmtext">Confirm</label><br/>
                <input type="text" id="confirmtext" required="" name="confirmtext" value="" size="10"/>
-                <span class="pure-form-message-inline">Type in the word <strong>scrub</strong> to confirm that you understand!</span>
+                <br/>
            </div>
-            <br/>
+
-            <div class="pure-control-group">
+
                <label for="confirmtext">Optional: Limit deletion of snapshots to snapshots <i>newer</i> than date/time</label>
                <input type="datetime-local" id="limit_date" name="limit_date"  />
                <span class="pure-form-message-inline">dd/mm/yyyy hh:mm (24 hour format)</span>
            </div>
            <br/>
            <div class="pure-control-group">
                <button type="submit" class="pure-button pure-button-primary">Scrub!</button>
            </div>
@@ -28,8 +32,12 @@
            <div class="pure-control-group">
                <a href="/" class="pure-button button-small button-cancel">Cancel</a>
            </div>
        </fieldset>
    </form>
 </div>
 {% endblock %}
--- a/backend/templates/settings.html
+++ b/backend/templates/settings.html
@@ -1,34 +1,18 @@
 {% extends 'base.html' %}
 {% block content %}
 {% from '_helpers.jinja' import render_field %}
 <div class="edit-form">
-    <form class="pure-form pure-form-stacked settings" action="/settings" method="POST">
+
    <form class="pure-form pure-form-stacked" action="/settings" method="POST">
        <fieldset>
            <div class="pure-control-group">
-                {{ render_field(form.minutes_between_check, size=5) }}
+                <label for="minutes">Maximum time in minutes until recheck.</label>
                <input type="text" id="minutes" required="" name="minutes" value="{{minutes}}"
                       size="5"/>
                <span class="pure-form-message-inline">This is a required field.</span>
            </div>
            <div class="pure-control-group">
                {% if current_user.is_authenticated %}
                    <a href="/settings?removepassword=true" class="pure-button pure-button-primary">Remove password</a>
                {% else %}
                    {{ render_field(form.password, size=10) }}
                {% endif %}
            </div>
            <div class="pure-control-group">
                {{ render_field(form.notification_urls, rows=5, placeholder="Gitter - gitter://token/room
 Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
 AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
 SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
 ") }}
                <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
            </div>
                <div class="pure-controls">
                    <span class="pure-form-message-inline"><label for="trigger-test-notification" class="pure-checkbox">
                        <input type="checkbox" id="trigger-test-notification" name="trigger-test-notification"> Send test notification on save.</label></span>
                </div>
            <br/>
            <div class="pure-control-group">
@@ -38,7 +22,7 @@ SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
            <div class="pure-control-group">
                <a href="/" class="pure-button button-small button-cancel">Back</a>
-                <a href="/scrub" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
+                <a href="/scrub" class="pure-button button-small button-cancel">Reset all version data</a>
            </div>
--- a/backend/templates/watch-diff-stream.html
+++ b/backend/templates/watch-diff-stream.html
@@ -0,0 +1,12 @@
 {% extends 'watch-overview.html' %}
 {% block innercontent %}
 Entries: {{ streams|length }}
 <div id="diff-stream" class="edit-form">
    {% for item in streams %}
        {{ loop.index }}
 {% for diff in item %}{% if diff[0] =='+' %}<ins>{{ diff }}</ins>{% endif %}{% if diff[0] =='-' %}<del>{{ diff }}</del>{% endif %}
        {% endfor %}
    {% endfor %}
 </div>
 {% endblock %}
--- a/backend/templates/watch-overview.html
+++ b/backend/templates/watch-overview.html
@@ -24,74 +24,9 @@
    </div>
    <div id="watch-table-wrapper">
-        <table class="pure-table pure-table-striped watch-table">
+    {% block innercontent %}
            <thead>
            <tr>
                <th>#</th>
                <th></th>
                <th></th>
                <th>Last Checked</th>
                <th>Last Changed</th>
                <th></th>
            </tr>
            </thead>
            <tbody>
-
+    {% endblock %}
            {% for watch in watches %}
            <tr id="{{ watch.uuid }}"
                class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
                {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
                {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
                <td class="inline">{{ loop.index }}</td>
                <td class="inline paused-state state-{{watch.paused}}"><a href="/?pause={{ watch.uuid}}{% if active_tag %}&tag={{active_tag}}{% endif %}"><img src="/static/images/pause.svg" alt="Pause"/></a></td>
                <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                    <a class="external" target="_blank" rel="noopener" href="{{ watch.url }}"></a>
                    {% if watch.last_error is defined and watch.last_error != False %}
                    <div class="fetch-error">{{ watch.last_error }}</div>
                    {% endif %}
                    {% if not active_tag %}
                    <span class="watch-tag-list">{{ watch.tag}}</span>
                    {% endif %}
                </td>
                <td class="last-checked">{{watch|format_last_checked_time}}</td>
                <td class="last-changed">{% if watch.history|length >= 2 and watch.last_changed %}
                    {{watch.last_changed|format_timestamp_timeago}}
                    {% else %}
                    Not yet
                    {% endif %}
                </td>
                <td>
                    <a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
                       class="pure-button button-small pure-button-primary">Recheck</a>
                    <a href="/edit/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
                    {% if watch.history|length >= 2 %}
                    <a href="/diff/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
                    {% else %}
                        {% if watch.history|length == 1 %}
                            <a href="/preview/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
                        {% endif %}
                    {% endif %}
                </td>
            </tr>
            {% endfor %}
            </tbody>
        </table>
        <ul id="post-list-buttons">
            {% if has_unviewed %}
            <li>
                <a href="/api/mark-all-viewed" class="pure-button button-tag ">Mark all viewed</a>
            </li>
            {% endif %}
            <li>
               <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
            </li>
            <li>
                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="/static/images/Generic_Feed-icon.svg" height="15px"></a>
            </li>
        </ul>
    </div>
 </div>
 {% endblock %}
--- a/backend/templates/watch-table.html
+++ b/backend/templates/watch-table.html
@@ -0,0 +1,64 @@
 {% extends 'watch-overview.html' %}
 {% block innercontent %}
        <table class="pure-table pure-table-striped watch-table">
            <thead>
            <tr>
                <th>#</th>
                <th></th>
                <th>Last Checked</th>
                <th>Last Changed</th>
                <th></th>
            </tr>
            </thead>
            <tbody>
            {% for watch in watches %}
            <tr id="{{ watch.uuid }}"
                class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
                {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
                <td>{{ loop.index }}</td>
                <td class="title-col">{{watch.title if watch.title is not none else watch.url}}
                    <a class="external" target=_blank href="{{ watch.url }}"></a>
                    {% if watch.last_error is defined and watch.last_error != False %}
                    <div class="fetch-error">{{ watch.last_error }}</div>
                    {% endif %}
                    {% if not active_tag %}
                    <span class="watch-tag-list">{{ watch.tag}}</span>
                    {% endif %}
                </td>
                <td>{{watch|format_last_checked_time}}</td>
                <td>{% if watch.history|length >= 2 and watch.last_changed %}
                    {{watch.last_changed|format_timestamp_timeago}}
                    {% else %}
                    Not yet
                    {% endif %}
                </td>
                <td>
                    <a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
                       class="pure-button button-small pure-button-primary">Recheck</a>
                    <a href="/edit/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
                    {% if watch.history|length >= 2 %}
                    <a href="/diff/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
                    {% endif %}
                </td>
            </tr>
            {% endfor %}
            </tbody>
        </table>
        <ul id="post-list-buttons">
            {% if has_unviewed %}
            <li>
                <a href="/api/mark-all-viewed" class="pure-button button-tag ">Mark all viewed</a>
            </li>
            {% endif %}
            <li>
               <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
            </li>
            <li>
                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="/static/images/Generic_Feed-icon.svg" height="15px"></a>
            </li>
        </ul>
 {% endblock %}
--- a/backend/tests/conftest.py
+++ b/backend/tests/conftest.py
@@ -35,12 +35,13 @@ def app(request):
    def teardown():
        datastore.stop_thread = True
        app.config.exit.set()
-        for fname in ["url-watches.json", "count.txt", "output.txt"]:
+        try:
-            try:
+            os.unlink("{}/url-watches.json".format(datastore_path))
-                os.unlink("{}/{}".format(datastore_path, fname))
+        except FileNotFoundError:
-            except FileNotFoundError:
+            # This is fine in the case of a failure.
-                # This is fine in the case of a failure.
+            pass
-                pass
+
        assert 1 == 1
    request.addfinalizer(teardown)
    yield app
--- a/backend/tests/test_access_control.py
+++ b/backend/tests/test_access_control.py
@@ -1,102 +0,0 @@
 from flask import url_for
 def test_check_access_control(app, client):
    # Still doesnt work, but this is closer.
    with app.test_client() as c:
        # Check we dont have any password protection enabled yet.
        res = c.get(url_for("settings_page"))
        assert b"Remove password" not in res.data
        # Enable password check.
        res = c.post(
            url_for("settings_page"),
            data={"password": "foobar", "minutes_between_check": 180},
            follow_redirects=True
        )
        assert b"Password protection enabled." in res.data
        assert b"LOG OUT" not in res.data
        # Check we hit the login
        res = c.get(url_for("index"), follow_redirects=True)
        assert b"Login" in res.data
        # Menu should not be available yet
        #        assert b"SETTINGS" not in res.data
        #        assert b"BACKUP" not in res.data
        #        assert b"IMPORT" not in res.data
        # defaultuser@changedetection.io is actually hardcoded for now, we only use a single password
        res = c.post(
            url_for("login"),
            data={"password": "foobar"},
            follow_redirects=True
        )
        assert b"LOG OUT" in res.data
        res = c.get(url_for("settings_page"))
        # Menu should be available now
        assert b"SETTINGS" in res.data
        assert b"BACKUP" in res.data
        assert b"IMPORT" in res.data
        assert b"LOG OUT" in res.data
        # Now remove the password so other tests function, @todo this should happen before each test automatically
        res = c.get(url_for("settings_page", removepassword="true"),
              follow_redirects=True)
        assert b"Password protection removed." in res.data
        res = c.get(url_for("index"))
        assert b"LOG OUT" not in res.data
 # There was a bug where saving the settings form would submit a blank password
 def test_check_access_control_no_blank_password(app, client):
    # Still doesnt work, but this is closer.
    with app.test_client() as c:
        # Check we dont have any password protection enabled yet.
        res = c.get(url_for("settings_page"))
        assert b"Remove password" not in res.data
        # Enable password check.
        res = c.post(
            url_for("settings_page"),
            data={"password": "", "minutes_between_check": 180},
            follow_redirects=True
        )
        assert b"Password protection enabled." not in res.data
        assert b"Login" not in res.data
 # There was a bug where saving the settings form would submit a blank password
 def test_check_access_no_remote_access_to_remove_password(app, client):
    # Still doesnt work, but this is closer.
    with app.test_client() as c:
        # Check we dont have any password protection enabled yet.
        res = c.get(url_for("settings_page"))
        assert b"Remove password" not in res.data
        # Enable password check.
        res = c.post(
            url_for("settings_page"),
            data={"password": "password", "minutes_between_check": 180},
            follow_redirects=True
        )
        assert b"Password protection enabled." in res.data
        assert b"Login" in res.data
        res = c.get(url_for("settings_page", removepassword="true"),
              follow_redirects=True)
        assert b"Password protection removed." not in res.data
        res = c.get(url_for("index"),
              follow_redirects=True)
        assert b"watch-table-wrapper" not in res.data
--- a/backend/tests/test_backend.py
+++ b/backend/tests/test_backend.py
@@ -3,16 +3,55 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-from . util import set_original_response, set_modified_response, live_server_setup
+import pytest
 sleep_time_for_fetch_thread = 3
 def test_setup_liveserver(live_server):
    @live_server.app.route('/test-endpoint')
    def test_endpoint():
        # Tried using a global var here but didn't seem to work, so reading from a file instead.
        with open("test-datastore/output.txt", "r") as f:
            return f.read()
    live_server.start()
    assert 1 == 1
 def set_original_response():
    test_return_data = """<html>
       <body>
     Some initial text</br>
     <p>Which is across multiple lines</p>
     </br>
     So let's see what happens.  </br>
     </body>
     </html>
    """
    with open("test-datastore/output.txt", "w") as f:
        f.write(test_return_data)
 def set_modified_response():
    test_return_data = """<html>
       <body>
     Some initial text</br>
     <p>which has this one new line</p>
     </br>
     So let's see what happens.  </br>
     </body>
     </html>
    """
    with open("test-datastore/output.txt", "w") as f:
        f.write(test_return_data)
 def test_check_basic_change_detection_functionality(client, live_server):
    set_original_response()
    live_server_setup(live_server)
    # Add our URL to the import page
    res = client.post(
@@ -36,12 +75,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
        assert b'unviewed' not in res.data
        assert b'test-endpoint' in res.data
        # Default no password set, this stuff should be always available.
        assert b"SETTINGS" in res.data
        assert b"BACKUP" in res.data
        assert b"IMPORT" in res.data
    #####################
    # Make a change
@@ -60,12 +93,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
    res = client.get(url_for("index"))
    assert b'unviewed' in res.data
    # #75, and it should be in the RSS feed
    res = client.get(url_for("index", rss="true"))
    expected_url = url_for('test_endpoint', _external=True)
    assert b'<rss' in res.data
    assert expected_url.encode('utf-8') in res.data
    # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
    res = client.get(url_for("diff_history_page", uuid="first"))
    assert b'Compare newest' in res.data
@@ -94,4 +121,3 @@ def test_check_basic_change_detection_functionality(client, live_server):
    # Cleanup everything
    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
    assert b'Deleted' in res.data
--- a/backend/tests/test_css_selector.py
+++ b/backend/tests/test_css_selector.py
@@ -1,102 +0,0 @@
 #!/usr/bin/python3
 import time
 from flask import url_for
 from . util import live_server_setup
 def test_setup(live_server):
    live_server_setup(live_server)
 def set_original_response():
    test_return_data = """<html>
       <body>
     Some initial text</br>
     <p>Which is across multiple lines</p>
     </br>
     So let's see what happens.  </br>
     <div id="sametext">Some text thats the same</div>
     <div id="changetext">Some text that will change</div>
     </body>
     </html>
    """
    with open("test-datastore/output.txt", "w") as f:
        f.write(test_return_data)
    return None
 def set_modified_response():
    test_return_data = """<html>
       <body>
     Some initial text</br>
     <p>which has this one new line</p>
     </br>
     So let's see what happens.  </br>
     <div id="sametext">Some text thats the same</div>
     <div id="changetext">Some text that changes</div>
     </body>
     </html>
    """
    with open("test-datastore/output.txt", "w") as f:
        f.write(test_return_data)
    return None
 def test_check_markup_css_filter_restriction(client, live_server):
    sleep_time_for_fetch_thread = 3
    css_filter = "#sametext"
    set_original_response()
    # Give the endpoint time to spin up
    time.sleep(1)
    # Add our URL to the import page
    test_url = url_for('test_endpoint', _external=True)
    res = client.post(
        url_for("import_page"),
        data={"urls": test_url},
        follow_redirects=True
    )
    assert b"1 Imported" in res.data
    # Trigger a check
    client.get(url_for("api_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
    # Goto the edit page, add our ignore text
    # Add our URL to the import page
    res = client.post(
        url_for("edit_page", uuid="first"),
        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": ""},
        follow_redirects=True
    )
    assert b"Updated watch." in res.data
    # Check it saved
    res = client.get(
        url_for("edit_page", uuid="first"),
    )
    assert bytes(css_filter.encode('utf-8')) in res.data
    # Trigger a check
    client.get(url_for("api_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
    #  Make a change
    set_modified_response()
    # Trigger a check
    client.get(url_for("api_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)
    # It should have 'unviewed' still
    # Because it should be looking at only that 'sametext' id
    res = client.get(url_for("index"))
    assert b'unviewed' in res.data
--- a/backend/tests/test_ignore_regex_text.py
+++ b/backend/tests/test_ignore_regex_text.py
@@ -1,31 +0,0 @@
 #!/usr/bin/python3
 import time
 from flask import url_for
 from . util import live_server_setup
 def test_setup(live_server):
    live_server_setup(live_server)
 # Unit test of the stripper
 # Always we are dealing in utf-8
 def test_strip_regex_text_func():
    from backend import fetch_site_status
    test_content = """
    but sometimes we want to remove the lines.
    but 1 lines
    but including 1234 lines
    igNORe-cAse text we dont want to keep    
    but not always."""
    ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"]
    fetcher = fetch_site_status.perform_site_check(datastore=False)
    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
    assert b"but 1 lines" in stripped_content
    assert b"igNORe-cAse text" not in stripped_content
    assert b"but 1234 lines" not in stripped_content
--- a/backend/tests/test_ignore_text.py
+++ b/backend/tests/test_ignore_text.py
@@ -2,10 +2,9 @@
 import time
 from flask import url_for
-from . util import live_server_setup
+from urllib.request import urlopen
 import pytest
 def test_setup(live_server):
    live_server_setup(live_server)
 # Unit test of the stripper
 # Always we are dealing in utf-8
@@ -82,7 +81,7 @@ def set_modified_ignore_response():
 def test_check_ignore_text_functionality(client, live_server):
    sleep_time_for_fetch_thread = 3
-    ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
+    ignore_text = "XXXXX\nYYYYY\nZZZZZ"
    set_original_ignore_response()
    # Give the endpoint time to spin up
@@ -107,7 +106,7 @@ def test_check_ignore_text_functionality(client, live_server):
    # Add our URL to the import page
    res = client.post(
        url_for("edit_page", uuid="first"),
-        data={"ignore_text": ignore_text, "url": test_url},
+        data={"ignore-text": ignore_text, "url": test_url, "tag": "", "headers": ""},
        follow_redirects=True
    )
    assert b"Updated watch." in res.data
--- a/backend/tests/test_notification.py
+++ b/backend/tests/test_notification.py
@@ -1,72 +0,0 @@
 import time
 from flask import url_for
 from . util import set_original_response, set_modified_response, live_server_setup
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
 def test_check_notification(client, live_server):
    live_server_setup(live_server)
    set_original_response()
    # Give the endpoint time to spin up
    time.sleep(3)
    # Add our URL to the import page
    test_url = url_for('test_endpoint', _external=True)
    res = client.post(
        url_for("import_page"),
        data={"urls": test_url},
        follow_redirects=True
    )
    assert b"1 Imported" in res.data
    # Give the thread time to pick it up
    time.sleep(3)
    # Goto the edit page, add our ignore text
    # Add our URL to the import page
    url = url_for('test_notification_endpoint', _external=True)
    notification_url = url.replace('http', 'json')
    print (">>>> Notification URL: "+notification_url)
    res = client.post(
        url_for("edit_page", uuid="first"),
        data={"notification_urls": notification_url, "url": test_url, "tag": "", "headers": ""},
        follow_redirects=True
    )
    assert b"Updated watch." in res.data
    # Hit the edit page, be sure that we saved it
    res = client.get(
        url_for("edit_page", uuid="first"))
    assert bytes(notification_url.encode('utf-8')) in res.data
    set_modified_response()
    # Trigger a check
    client.get(url_for("api_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(3)
    # Did the front end see it?
    res = client.get(
        url_for("index"))
    assert bytes("just now".encode('utf-8')) in res.data
    # Check it triggered
    res = client.get(
        url_for("test_notification_counter"),
    )
    assert bytes("we hit it".encode('utf-8')) in res.data
    # Did we see the URL that had a change, in the notification?
    assert bytes("test-endpoint".encode('utf-8')) in res.data
    # Re #65 - did we see our foobar.com BASE_URL ?
    assert bytes("https://foobar.com".encode('utf-8')) in res.data
--- a/backend/tests/test_watch_fields_storage.py
+++ b/backend/tests/test_watch_fields_storage.py
@@ -1,52 +0,0 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
 from . util import set_original_response, set_modified_response, live_server_setup
 def test_check_watch_field_storage(client, live_server):
    set_original_response()
    live_server_setup(live_server)
    test_url = "http://somerandomsitewewatch.com"
    res = client.post(
        url_for("import_page"),
        data={"urls": test_url},
        follow_redirects=True
    )
    assert b"1 Imported" in res.data
    res = client.post(
        url_for("edit_page", uuid="first"),
        data={ "notification_urls": "http://myapi.com",
               "minutes_between_check": 126,
               "css_filter" : ".fooclass",
               "title" : "My title",
               "ignore_text" : "ignore this",
               "url": test_url,
               "tag": "woohoo",
               "headers": "curl:foo",
               },
        follow_redirects=True
    )
    assert b"Updated watch." in res.data
    res = client.get(
        url_for("edit_page", uuid="first"),
        follow_redirects=True
    )
    assert b"http://myapi.com" in res.data
    assert b"126" in res.data
    assert b".fooclass" in res.data
    assert b"My title" in res.data
    assert b"ignore this" in res.data
    assert b"http://somerandomsitewewatch.com" in res.data
    assert b"woohoo" in res.data
    assert b"curl: foo" in res.data
--- a/backend/tests/util.py
+++ b/backend/tests/util.py
@@ -1,67 +0,0 @@
 #!/usr/bin/python3
 def set_original_response():
    test_return_data = """<html>
       <body>
     Some initial text</br>
     <p>Which is across multiple lines</p>
     </br>
     So let's see what happens.  </br>
     </body>
     </html>
    """
    with open("test-datastore/output.txt", "w") as f:
        f.write(test_return_data)
    return None
 def set_modified_response():
    test_return_data = """<html>
       <body>
     Some initial text</br>
     <p>which has this one new line</p>
     </br>
     So let's see what happens.  </br>
     </body>
     </html>
    """
    with open("test-datastore/output.txt", "w") as f:
        f.write(test_return_data)
    return None
 def live_server_setup(live_server):
    @live_server.app.route('/test-endpoint')
    def test_endpoint():
        # Tried using a global var here but didn't seem to work, so reading from a file instead.
        with open("test-datastore/output.txt", "r") as f:
            return f.read()
    @live_server.app.route('/test_notification_endpoint', methods=['POST'])
    def test_notification_endpoint():
        from flask import request
        with open("test-datastore/count.txt", "w") as f:
            f.write("we hit it\n")
            # Debug method, dump all POST to file also, used to prove #65
            data = request.stream.read()
            if data != None:
                f.write(str(data))
        print("\n>> Test notification endpoint was hit.\n")
        return "Text was set"
    # And this should return not zero.
    @live_server.app.route('/test_notification_counter')
    def test_notification_counter():
        try:
            with open("test-datastore/count.txt", "r") as f:
                return f.read()
        except FileNotFoundError:
            return "nope :("
    live_server.start()
--- a/backend/update_worker.py
+++ b/backend/update_worker.py
@@ -1,67 +0,0 @@
 import threading
 import queue
 # Requests for checking on the site use a pool of thread Workers managed by a Queue.
 class update_worker(threading.Thread):
    current_uuid = None
    def __init__(self, q, notification_q, app, datastore, *args, **kwargs):
        self.q = q
        self.app = app
        self.notification_q = notification_q
        self.datastore = datastore
        super().__init__(*args, **kwargs)
    def run(self):
        from backend import fetch_site_status
        update_handler = fetch_site_status.perform_site_check(datastore=self.datastore)
        while not self.app.config.exit.is_set():
            try:
                uuid = self.q.get(block=False)
            except queue.Empty:
                pass
            else:
                self.current_uuid = uuid
                if uuid in list(self.datastore.data['watching'].keys()):
                    try:
                        changed_detected, result, contents = update_handler.run(uuid)
                    except PermissionError as s:
                        self.app.logger.error("File permission error updating", uuid, str(s))
                    else:
                        if result:
                            try:
                                self.datastore.update_watch(uuid=uuid, update_obj=result)
                                if changed_detected:
                                    # A change was detected
                                    self.datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
                                    watch = self.datastore.data['watching'][uuid]
                                    # Did it have any notification alerts to hit?
                                    if len(watch['notification_urls']):
                                        print("Processing notifications for UUID: {}".format(uuid))
                                        n_object = {'watch_url': self.datastore.data['watching'][uuid]['url'],
                                                    'notification_urls': watch['notification_urls']}
                                        self.notification_q.put(n_object)
                                    # No? maybe theres a global setting, queue them all
                                    elif len(self.datastore.data['settings']['application']['notification_urls']):
                                        print("Processing GLOBAL notifications for UUID: {}".format(uuid))
                                        n_object = {'watch_url': self.datastore.data['watching'][uuid]['url'],
                                                    'notification_urls': self.datastore.data['settings']['application'][
                                                        'notification_urls']}
                                        self.notification_q.put(n_object)
                            except Exception as e:
                                print("!!!! Exception in update_worker !!!\n", e)
                self.current_uuid = None  # Done
                self.q.task_done()
            self.app.config.exit.wait(1)
--- a/btc-support.png
+++ b/btc-support.png
--- a/changedetection.py
+++ b/changedetection.py
@@ -3,7 +3,6 @@
 # Launch as a eventlet.wsgi server instance.
 import getopt
 import os
 import sys
 import eventlet
@@ -12,16 +11,14 @@ import backend
 from backend import store
 def main(argv):
    ssl_mode = False
    port = 5000
-    do_cleanup = False
+    datastore_path = "./datastore"
    # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
    datastore_path = os.path.join(os.getcwd(), "datastore")
    try:
-        opts, args = getopt.getopt(argv, "csd:p:", "port")
+        opts, args = getopt.getopt(argv, "sd:p:", "purge")
    except getopt.GetoptError:
        print('backend.py -s SSL enable -p [port] -d [datastore path]')
        sys.exit(2)
@@ -41,9 +38,11 @@ def main(argv):
        if opt == '-d':
            datastore_path = arg
-        # Cleanup (remove text files that arent in the index)
+
-        if opt == '-c':
+
-            do_cleanup = True
+    # threads can read from disk every x seconds right?
    # front end can just save
    # We just need to know which threads are looking at which UUIDs
    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
    app_config = {'datastore_path': datastore_path}
@@ -51,19 +50,13 @@ def main(argv):
    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'])
    app = backend.changedetection_app(app_config, datastore)
    # Go into cleanup mode
    if do_cleanup:
        datastore.remove_unused_snapshots()
    app.config['datastore_path'] = datastore_path
    @app.context_processor
    def inject_version():
-        return dict(version=datastore.data['version_tag'],
+        return dict(version=datastore.data['version_tag'])
-                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
+
-                    has_password=datastore.data['settings']['application']['password'] != False
+    @app.context_processor
-                    )
+    def inject_new_version_available():
        return dict(new_version_available=app.config['NEW_VERSION_AVAILABLE'])
    if ssl_mode:
        # @todo finalise SSL config, but this should get you in the right direction if you need it.
--- a/docker-compose-development.yml
+++ b/docker-compose-development.yml
@@ -0,0 +1,23 @@
 version: "2"
 services:
 # I have a feeling we can get rid of this, and just use one docker-compose.yml, and just set a ENV var if
 # we want dev mode (just gives a docker shell) or not.
  backend:
    build: ./backend/dev-docker
    image: dgtlmoon/changedetection.io:dev
    container_name: changedetection.io-dev
    volumes:
      - .:/app
      - ./requirements.txt:/requirements.txt # Normally COPY'ed in the Dockerfile
      - ./datastore:/datastore
    ports:
      - "127.0.0.1:5001:5000"
    networks:
      - changenet
 networks:
  changenet:
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,26 +0,0 @@
 version: '2'
 services:
    changedetection.io:
      image: dgtlmoon/changedetection.io
      container_name: changedetection.io
      hostname: changedetection.io
      volumes:
        - changedetection-data:/datastore
  #    environment:
  #      - PUID=1000
  #      - PGID=1000
  #        Proxy support example.
  #      - HTTP_PROXY="socks5h://10.10.1.10:1080"
  #      - HTTPS_PROXY="socks5h://10.10.1.10:1080"
  #        An exclude list (useful for notification URLs above) can be specified by with
  #      - NO_PROXY="localhost,192.168.0.0/24"
  #        Base URL of your changedetection.io install (Added to notification alert
  #      - BASE_URL="https://mysite.com"
      ports:
        - 5000:5000
      restart: always
 volumes:
  changedetection-data:
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,23 +1,13 @@
 chardet==2.3.0
 yarl
 flask~= 1.0
 pytest ~=6.2
-pytest-flask ~=1.2
+pytest-flask ~=1.1
-eventlet>=0.31.0
+eventlet ~= 0.30
-requests[socks] ~= 2.15
+requests
 validators
 timeago ~=1.0
 inscriptis ~= 1.1
 feedgen ~= 0.9
 flask-login ~= 0.5
 pytz
 urllib3
 wtforms ~= 2.3.3
 # Notification library
 apprise ~= 0.9
 # Used for CSS filtering
 bs4
--- a/screenshot-notifications.png
+++ b/screenshot-notifications.png
Author	SHA1	Message	Date
dgtlmoon	14d88c249e	Re #23 - always check value of interval time, not just on start	2021-03-29 15:03:33 +02:00
Leigh Morresi	0fa443c3f2	Proof of concept, view as a stream	2021-03-15 22:26:50 +01:00