Re #23 - always check value of interval time, not just on start

.dockerignore

@@ -1,2 +0,0 @@
-.git
-.github

.github/FUNDING.yml

@@ -1,3 +1,12 @@
 # These are supported funding model platforms
 
 github: dgtlmoon
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/codeql-analysis.yml

@@ -1,67 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ master ]
-  schedule:
-    - cron: '27 9 * * 4'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'javascript', 'python' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
-        # Learn more:
-        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.github/workflows/image.yml

@@ -1,77 +0,0 @@
-name: Test, build and push to Docker Hub
-
-on:
-  push:
-    branches: [ master ]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install flake8 pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-      - name: Test with pytest
-        run: |
-          cd backend; pytest
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          image: tonistiigi/binfmt:latest
-          platforms: all
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-          version: latest
-          driver-opts: image=moby/buildkit:master
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./
-          file: ./Dockerfile
-          push: true
-          tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
-          #                ${{ secrets.DOCKER_HUB_USERNAME }}:/changedetection.io:${{ env.RELEASE_VERSION }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
-
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
-
-      - name: Cache Docker layers
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-
-

.github/workflows/python-app.yml

@@ -0,0 +1,33 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: changedetection.io
+
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        cd backend; pytest

.github/workflows/test-only.yml

@@ -1,33 +0,0 @@
-name: Test only
-
-# Triggers the workflow on push or pull request events
-on: [push, pull_request]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install flake8 pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-      - name: Test with pytest
-        run: |
-          # Each test is totally isolated and performs its own cleanup/reset
-          cd backend; ./run_all_tests.sh
-

Dockerfile

@@ -1,8 +1,6 @@
 FROM python:3.8-slim
 COPY requirements.txt /tmp/requirements.txt
-RUN apt-get update && apt-get install -y gcc libc-dev libxslt-dev zlib1g-dev g++ --no-install-recommends && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
-
-RUN pip3 install --no-cache-dir -r /tmp/requirements.txt
+RUN pip3 install -r /tmp/requirements.txt
 
 
 RUN [ ! -d "/app" ] && mkdir /app

README.md

@@ -1,21 +1,19 @@
 #  changedetection.io
-![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/image.yml/badge.svg?branch=master)
+![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/python-app.yml/badge.svg?branch=master)
 <a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
   <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
 </a>
 <a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
-  <img src="https://img.shields.io/github/v/release/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
+  <img src="https://img.shields.io/docker/v/dgtlmoon/changedetection.io/0.27" alt="Change detection latest tag version"/> 
 </a>
 
 ## Self-hosted change monitoring of web pages.
 
-_Know when web pages change! Stay ontop of new information!_ 
+_Know when web pages change! Stay ontop of new information!_
 
-Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
+![Self-hosted web page change monitoring application screenshot](screenshot.png?raw=true "Self-hosted web page change monitoring screenshot")
 
 
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
-
 #### Example use cases
 
 Know when ...
@@ -50,36 +48,16 @@ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/dat
 
 Examining differences in content.
 
+![Self-hosted web page change monitoring context difference screenshot](screenshot-diff.png?raw=true "Self-hosted web page change monitoring context difference screenshot")
 
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
+### Future plans
 
-Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
+- Greater configuration of check interval times, page request headers.
+- ~~General options for timeout, default headers~~
+- On change detection, callout to another API (handy for notices/issue trackers)
+- ~~Explore the differences that were detected~~ 
+- Add more options to explore versions of differences
+- Use a graphic/rendered page difference instead of text (see the experimental `selenium-screenshot-diff` branch)
 
-### Notifications
-
-ChangeDetection.io supports a massive amount of notifications (including email, office365, custom APIs, etc) when a web-page has a change detected thanks to the <a href="https://github.com/caronc/apprise">apprise</a> library.
-Simply set one or more notification URL's in the _[edit]_ tab of that watch.
-
-Just some examples
-
-    discord://webhook_id/webhook_token
-    flock://app_token/g:channel_id
-    gitter://token/room
-    gchat://workspace/key/token
-    msteams://TokenA/TokenB/TokenC/
-    o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
-    rocket://user:password@hostname/#Channel
-    mailto://user:pass@example.com?to=receivingAddress@example.com
-    json://someserver.com/custom-api
-    syslog://
  
-<a href="https://github.com/caronc/apprise">And everything else in this list!</a>
-
-<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
-
-
-### Notes
-
-- Does not yet support Javascript
-- Wont work with Cloudfare type "Please turn on javascript" protected pages
-
+Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/

backend/__init__.py

@@ -15,15 +15,13 @@
 import time
 import os
 import timeago
-import flask_login
-from flask_login import login_required
 
 import threading
 from threading import Event
 
 import queue
 
-from flask import Flask, render_template, request, send_from_directory, abort, redirect, url_for
+from flask import Flask, render_template, request, send_file, send_from_directory, abort, redirect, url_for
 
 from feedgen.feed import FeedGenerator
 from flask import make_response
@@ -41,9 +39,7 @@ extra_stylesheets = []
 
 update_q = queue.Queue()
 
-notification_q = queue.Queue()
-
-app = Flask(__name__, static_url_path="/var/www/change-detection/backend/static")
+app = Flask(__name__, static_url_path="/var/www/change-detection/backen/static")
 
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
@@ -52,8 +48,6 @@ app.config.exit = Event()
 
 app.config['NEW_VERSION_AVAILABLE'] = False
 
-app.config['LOGIN_DISABLED'] = False
-
 # Disables caching of the templates
 app.config['TEMPLATES_AUTO_RELOAD'] = True
 
@@ -86,128 +80,27 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
     # return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
 
 
-class User(flask_login.UserMixin):
-    id=None
-
-    def set_password(self, password):
-        return True
-    def get_user(self, email="defaultuser@changedetection.io"):
-        return self
-    def is_authenticated(self):
-
-        return True
-    def is_active(self):
-        return True
-    def is_anonymous(self):
-        return False
-    def get_id(self):
-        return str(self.id)
-
-    def check_password(self, password):
-
-        import hashlib
-        import base64
-
-        # Getting the values back out
-        raw_salt_pass = base64.b64decode(datastore.data['settings']['application']['password'])
-        salt_from_storage = raw_salt_pass[:32]  # 32 is the length of the salt
-
-        # Use the exact same setup you used to generate the key, but this time put in the password to check
-        new_key = hashlib.pbkdf2_hmac(
-            'sha256',
-            password.encode('utf-8'),  # Convert the password to bytes
-            salt_from_storage,
-            100000
-        )
-        new_key =  salt_from_storage + new_key
-
-        return new_key == raw_salt_pass
-
-    pass
-
-def changedetection_app(conig=None, datastore_o=None):
+def changedetection_app(config=None, datastore_o=None):
     global datastore
     datastore = datastore_o
 
     app.config.update(dict(DEBUG=True))
-    #app.config.update(config or {})
-
-    login_manager = flask_login.LoginManager(app)
-    login_manager.login_view = 'login'
-
+    app.config.update(config or {})
 
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
     #    CORS(app)
 
-    @login_manager.user_loader
-    def user_loader(email):
-        user = User()
-        user.get_user(email)
-        return user
-
-    @login_manager.unauthorized_handler
-    def unauthorized_handler():
-        # @todo validate its a URL of this host and use that
-        return redirect(url_for('login', next=url_for('index')))
-
-    @app.route('/logout')
-    def logout():
-        flask_login.logout_user()
-        return redirect(url_for('index'))
-
     # https://github.com/pallets/flask/blob/93dd1709d05a1cf0e886df6223377bdab3b077fb/examples/tutorial/flaskr/__init__.py#L39
     # You can divide up the stuff like this
-    @app.route('/login', methods=['GET', 'POST'])
-    def login():
-
-        global messages
-
-        if request.method == 'GET':
-            output = render_template("login.html", messages=messages)
-            # Show messages but once.
-            messages = []
-            return output
-
-        user = User()
-        user.id = "defaultuser@changedetection.io"
-
-        password = request.form.get('password')
-
-        if (user.check_password(password)):
-            flask_login.login_user(user, remember=True)
-            next = request.args.get('next')
-            #            if not is_safe_url(next):
-            #                return flask.abort(400)
-            return redirect(next or url_for('index'))
-
-        else:
-            messages.append({'class': 'error', 'message': 'Incorrect password'})
-
-        return redirect(url_for('login'))
-
-    @app.before_request
-    def do_something_whenever_a_request_comes_in():
-        # Disable password  loginif there is not one set
-        app.config['LOGIN_DISABLED'] = datastore.data['settings']['application']['password'] == False
 
     @app.route("/", methods=['GET'])
-    @login_required
     def index():
         global messages
+
         limit_tag = request.args.get('tag')
-
-        pause_uuid = request.args.get('pause')
-
-        if pause_uuid:
-            try:
-                datastore.data['watching'][pause_uuid]['paused'] ^= True
-                datastore.needs_write = True
-
-                return redirect(url_for('index', tag = limit_tag))
-            except KeyError:
-                pass
-
+        rss = request.args.get('rss')
+        mode = request.args.get('mode')
 
         # Sort by last_changed and add the uuid which is usually the key..
         sorted_watches = []
@@ -228,7 +121,63 @@ def changedetection_app(conig=None, datastore_o=None):
         sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)
 
         existing_tags = datastore.get_all_tags()
-        rss = request.args.get('rss')
+
+        if mode == 'stream':
+            import difflib
+
+            import pprint
+            streams = []
+
+            extra_stylesheets = ['/static/css/diff.css']
+            for watch in sorted_watches:
+                if not watch['viewed']:
+
+                    # get last two date keys
+                    dates = list(watch['history'].keys())
+                    # Convert to int, sort and back to str again
+                    dates = [int(i) for i in dates]
+                    dates.sort(reverse=True)
+                    dates = [str(i) for i in dates]
+                    print ("OK", watch['uuid'])
+
+                    if len(dates) < 2:
+                        print ("Skipping", watch['url'])
+                        continue
+                    else:
+                        try:
+                            path = datastore.data['watching'][watch['uuid']]['history'][str(dates[1])]
+                            with open(path,
+                                      encoding='utf-8') as file:
+                                txt1=[line.rstrip() for line in file.readlines()]
+
+                            path = datastore.data['watching'][watch['uuid']]['history'][str(dates[0])]
+                            with open(path,
+                                      encoding='utf-8') as file:
+                                txt2 = [line.rstrip() for line in file.readlines()]
+                        except FileNotFoundError:
+                            print ("Skipping", watch['url'])
+                            continue
+
+                        df = list(difflib.unified_diff(txt1, txt2,n=1))
+                        diff_entry=[]
+                        for line in df:
+                            if line[0] == '-' or line[0] == '+':
+                                diff_entry.append(line)
+
+
+                        # pprint(df)
+                        #s = pprint.pformat(df)
+                        streams.append(diff_entry)
+
+
+            print ("###########", len(streams))
+
+            output = render_template("watch-diff-stream.html",
+                                     streams=streams,
+                                     extra_stylesheets=extra_stylesheets
+                                     )
+            return output
+
 
         if rss:
             fg = FeedGenerator()
@@ -252,7 +201,8 @@ def changedetection_app(conig=None, datastore_o=None):
             return response
 
         else:
-            output = render_template("watch-overview.html",
+            #table = render_template('watch-table.html', watches=sorted_watches)
+            output = render_template("watch-table.html",
                                      watches=sorted_watches,
                                      messages=messages,
                                      tags=existing_tags,
@@ -265,7 +215,6 @@ def changedetection_app(conig=None, datastore_o=None):
         return output
 
     @app.route("/scrub", methods=['GET', 'POST'])
-    @login_required
     def scrub_page():
         from pathlib import Path
 
@@ -273,16 +222,19 @@ def changedetection_app(conig=None, datastore_o=None):
 
         if request.method == 'POST':
             confirmtext = request.form.get('confirmtext')
-            limit_timestamp = int(request.form.get('limit_date'))
 
             if confirmtext == 'scrub':
 
-                for uuid, watch in datastore.data['watching'].items():
-                    if len(str(limit_timestamp)) == 10:
-                        datastore.scrub_watch(uuid, limit_timestamp = limit_timestamp)
-                    else:
-                        datastore.scrub_watch(uuid)
+                for txt_file_path in Path(app.config['datastore_path']).rglob('*.txt'):
+                    os.unlink(txt_file_path)
 
+                for uuid, watch in datastore.data['watching'].items():
+                    watch['last_checked'] = 0
+                    watch['last_changed'] = 0
+                    watch['previous_md5'] = None
+                    watch['history'] = {}
+
+                datastore.needs_write = True
                 messages.append({'class': 'ok', 'message': 'Cleaned all version history.'})
             else:
                 messages.append({'class': 'error', 'message': 'Wrong confirm text.'})
@@ -320,7 +272,6 @@ def changedetection_app(conig=None, datastore_o=None):
         return datastore.data['watching'][uuid]['previous_md5']
 
     @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
-    @login_required
     def edit_page(uuid):
         global messages
         import validators
@@ -349,22 +300,11 @@ def changedetection_app(conig=None, datastore_o=None):
                           'headers': extra_headers
                           }
 
-            # Notification URLs
-            form_notification_text = request.form.get('notification_urls')
-            notification_urls = []
-            if form_notification_text:
-                for text in form_notification_text.strip().split("\n"):
-                    text = text.strip()
-                    if len(text):
-                        notification_urls.append(text)
-
-            datastore.data['watching'][uuid]['notification_urls'] = notification_urls
-
             # Ignore text
-            form_ignore_text = request.form.get('ignore-text')
+            form_ignore_text = request.form.get('ignore-text').strip()
             ignore_text = []
-            if form_ignore_text:
-                for text in form_ignore_text.strip().split("\n"):
+            if len(form_ignore_text):
+                for text in form_ignore_text.split("\n"):
                     text = text.strip()
                     if len(text):
                         ignore_text.append(text)
@@ -381,14 +321,6 @@ def changedetection_app(conig=None, datastore_o=None):
 
             messages.append({'class': 'ok', 'message': 'Updated watch.'})
 
-            trigger_n = request.form.get('trigger-test-notification')
-            if trigger_n:
-                n_object = {'watch_url': url,
-                            'notification_urls': datastore.data['settings']['application']['notification_urls']}
-                notification_q.put(n_object)
-
-                messages.append({'class': 'ok', 'message': 'Notifications queued.'})
-
             return redirect(url_for('index'))
 
         else:
@@ -397,62 +329,9 @@ def changedetection_app(conig=None, datastore_o=None):
         return output
 
     @app.route("/settings", methods=['GET', "POST"])
-    @login_required
     def settings_page():
         global messages
-
-        if request.method == 'GET':
-            if request.values.get('notification-test'):
-                url_count = len(datastore.data['settings']['application']['notification_urls'])
-                if url_count:
-                    import apprise
-                    apobj = apprise.Apprise()
-                    apobj.debug = True
-
-                    # Add each notification
-                    for n in datastore.data['settings']['application']['notification_urls']:
-                        apobj.add(n)
-                    outcome = apobj.notify(
-                        body='Hello from the worlds best and simplest web page change detection and monitoring service!',
-                        title='Changedetection.io Notification Test',
-                    )
-
-                    if outcome:
-                        messages.append(
-                            {'class': 'notice', 'message': "{} Notification URLs reached.".format(url_count)})
-                    else:
-                        messages.append(
-                            {'class': 'error', 'message': "One or more Notification URLs failed"})
-
-                return redirect(url_for('settings_page'))
-
-            if request.values.get('removepassword'):
-                from pathlib import Path
-
-                datastore.data['settings']['application']['password'] = False
-                messages.append({'class': 'notice', 'message': "Password protection removed."})
-                flask_login.logout_user()
-
-                return redirect(url_for('settings_page'))
-
         if request.method == 'POST':
-
-            password = request.values.get('password')
-            if password:
-                import hashlib
-                import base64
-                import secrets
-
-                # Make a new salt on every new password and store it with the password
-                salt = secrets.token_bytes(32)
-
-                key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
-                store = base64.b64encode(salt + key).decode('ascii')
-                datastore.data['settings']['application']['password'] = store
-                messages.append({'class': 'notice', 'message': "Password protection enabled."})
-                flask_login.logout_user()
-                return redirect(url_for('index'))
-
             try:
                 minutes = int(request.values.get('minutes').strip())
             except ValueError:
@@ -462,36 +341,19 @@ def changedetection_app(conig=None, datastore_o=None):
                 if minutes >= 5:
                     datastore.data['settings']['requests']['minutes_between_check'] = minutes
                     datastore.needs_write = True
+
+                    messages.append({'class': 'ok', 'message': "Updated"})
                 else:
                     messages.append(
                         {'class': 'error', 'message': "Must be atleast 5 minutes."})
 
-            # 'validators' package doesnt work because its often a non-stanadard protocol. :(
-            datastore.data['settings']['application']['notification_urls'] = []
-            trigger_n = request.form.get('trigger-test-notification')
-
-            for n in request.values.get('notification_urls').strip().split("\n"):
-                url = n.strip()
-                datastore.data['settings']['application']['notification_urls'].append(url)
-                datastore.needs_write = True
-
-            if trigger_n:
-                n_object = {'watch_url': "Test from changedetection.io!",
-                            'notification_urls': datastore.data['settings']['application']['notification_urls']}
-                notification_q.put(n_object)
-
-                messages.append({'class': 'ok', 'message': 'Notifications queued.'})
-
         output = render_template("settings.html", messages=messages,
-                                 minutes=datastore.data['settings']['requests']['minutes_between_check'],
-                                 notification_urls="\r\n".join(
-                                     datastore.data['settings']['application']['notification_urls']))
+                                 minutes=datastore.data['settings']['requests']['minutes_between_check'])
         messages = []
 
         return output
 
     @app.route("/import", methods=['GET', "POST"])
-    @login_required
     def import_page():
         import validators
         global messages
@@ -529,7 +391,6 @@ def changedetection_app(conig=None, datastore_o=None):
 
     # Clear all statuses, so we do not see the 'unviewed' class
     @app.route("/api/mark-all-viewed", methods=['GET'])
-    @login_required
     def mark_all_viewed():
 
         # Save the current newest history as the most recently viewed
@@ -540,7 +401,6 @@ def changedetection_app(conig=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/diff/<string:uuid>", methods=['GET'])
-    @login_required
     def diff_history_page(uuid):
         global messages
 
@@ -596,40 +456,13 @@ def changedetection_app(conig=None, datastore_o=None):
 
         return output
 
-    @app.route("/preview/<string:uuid>", methods=['GET'])
-    @login_required
-    def preview_page(uuid):
-        global messages
-
-        # More for testing, possible to return the first/only
-        if uuid == 'first':
-            uuid = list(datastore.data['watching'].keys()).pop()
-
-        extra_stylesheets = ['/static/css/diff.css']
-
-        try:
-            watch = datastore.data['watching'][uuid]
-        except KeyError:
-            messages.append({'class': 'error', 'message': "No history found for the specified link, bad link?"})
-            return redirect(url_for('index'))
-
-        print(watch)
-        with open(list(watch['history'].values())[-1], 'r') as f:
-            content = f.readlines()
-
-        output = render_template("preview.html", content=content, extra_stylesheets=extra_stylesheets)
-        return output
-
-
     @app.route("/favicon.ico", methods=['GET'])
     def favicon():
         return send_from_directory("/app/static/images", filename="favicon.ico")
 
     # We're good but backups are even better!
     @app.route("/backup", methods=['GET'])
-    @login_required
     def get_backup():
-
         import zipfile
         from pathlib import Path
 
@@ -638,31 +471,26 @@ def changedetection_app(conig=None, datastore_o=None):
 
         # We only care about UUIDS from the current index file
         uuids = list(datastore.data['watching'].keys())
-        backup_filepath = os.path.join(app.config['datastore_path'], backupname)
 
-        with zipfile.ZipFile(backup_filepath, "w",
+        with zipfile.ZipFile(os.path.join(app.config['datastore_path'], backupname), 'w',
                              compression=zipfile.ZIP_DEFLATED,
-                             compresslevel=8) as zipObj:
+                             compresslevel=6) as zipObj:
 
             # Be sure we're written fresh
             datastore.sync_to_json()
 
             # Add the index
-            zipObj.write(os.path.join(app.config['datastore_path'], "url-watches.json"), arcname="url-watches.json")
-
-            # Add the flask app secret
-            zipObj.write(os.path.join(app.config['datastore_path'], "secret.txt"), arcname="secret.txt")
-
-            # Add any snapshot data we find, use the full path to access the file, but make the file 'relative' in the Zip.
+            zipObj.write(os.path.join(app.config['datastore_path'], "url-watches.json"))
+            # Add any snapshot data we find
             for txt_file_path in Path(app.config['datastore_path']).rglob('*.txt'):
                 parent_p = txt_file_path.parent
                 if parent_p.name in uuids:
-                    zipObj.write(txt_file_path,
-                                 arcname=str(txt_file_path).replace(app.config['datastore_path'], ''),
-                                 compress_type=zipfile.ZIP_DEFLATED,
-                                 compresslevel=8)
+                    zipObj.write(txt_file_path)
 
-        return send_from_directory(app.config['datastore_path'], backupname)
+        return send_file(os.path.join(app.config['datastore_path'], backupname),
+                         as_attachment=True,
+                         mimetype="application/zip",
+                         attachment_filename=backupname)
 
     @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
     def static_content(group, filename):
@@ -676,17 +504,11 @@ def changedetection_app(conig=None, datastore_o=None):
             abort(404)
 
     @app.route("/api/add", methods=['POST'])
-    @login_required
     def api_watch_add():
         global messages
 
-        url = request.form.get('url').strip()
-        if datastore.url_exists(url):
-            messages.append({'class': 'error', 'message': 'The URL {} already exists'.format(url)})
-            return redirect(url_for('index'))
-
         # @todo add_watch should throw a custom Exception for validation etc
-        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
+        new_uuid = datastore.add_watch(url=request.form.get('url').strip(), tag=request.form.get('tag').strip())
         # Straight into the queue.
         update_q.put(new_uuid)
 
@@ -694,7 +516,6 @@ def changedetection_app(conig=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/api/delete", methods=['GET'])
-    @login_required
     def api_delete():
         global messages
         uuid = request.args.get('uuid')
@@ -704,7 +525,6 @@ def changedetection_app(conig=None, datastore_o=None):
         return redirect(url_for('index'))
 
     @app.route("/api/checknow", methods=['GET'])
-    @login_required
     def api_watch_checknow():
 
         global messages
@@ -728,17 +548,15 @@ def changedetection_app(conig=None, datastore_o=None):
             # Items that have this current tag
             for watch_uuid, watch in datastore.data['watching'].items():
                 if (tag != None and tag in watch['tag']):
-                    if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
+                    i += 1
+                    if watch_uuid not in running_uuids:
                         update_q.put(watch_uuid)
-                        i += 1
-
         else:
             # No tag, no uuid, add everything.
             for watch_uuid, watch in datastore.data['watching'].items():
-
-                if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
+                i += 1
+                if watch_uuid not in running_uuids:
                     update_q.put(watch_uuid)
-                    i += 1
 
         messages.append({'class': 'ok', 'message': "{} watches are rechecking.".format(i)})
         return redirect(url_for('index', tag=tag))
@@ -746,8 +564,6 @@ def changedetection_app(conig=None, datastore_o=None):
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
 
-    threading.Thread(target=notification_runner).start()
-
     # Check for new release version
     threading.Thread(target=check_for_new_version).start()
     return app
@@ -779,51 +595,63 @@ def check_for_new_version():
         # Check daily
         app.config.exit.wait(86400)
 
-def notification_runner():
 
-    while not app.config.exit.is_set():
-        try:
-            # At the moment only one thread runs (single runner)
-            n_object = notification_q.get(block=False)
-        except queue.Empty:
-            time.sleep(1)
-            pass
+# Requests for checking on the site use a pool of thread Workers managed by a Queue.
+class Worker(threading.Thread):
+    current_uuid = None
 
-        else:
-            import apprise
+    def __init__(self, q, *args, **kwargs):
+        self.q = q
+        super().__init__(*args, **kwargs)
+
+    def run(self):
+        from backend import fetch_site_status
+
+        update_handler = fetch_site_status.perform_site_check(datastore=datastore)
+
+        while not app.config.exit.is_set():
 
-            # Create an Apprise instance
             try:
-                apobj = apprise.Apprise()
-                for url in n_object['notification_urls']:
-                    apobj.add(url.strip())
+                uuid = self.q.get(block=False)
+            except queue.Empty:
+                pass
 
-                apobj.notify(
-                    body=n_object['watch_url'],
-                    # @todo This should be configurable.
-                    title="ChangeDetection.io Notification - {}".format(n_object['watch_url'])
-                )
+            else:
+                self.current_uuid = uuid
 
-            except Exception as e:
-                print("Watch URL: {}  Error {}".format(n_object['watch_url'],e))
+                if uuid in list(datastore.data['watching'].keys()):
+
+                    try:
+                        changed_detected, result, contents = update_handler.run(uuid)
+
+                    except PermissionError as s:
+                        app.logger.error("File permission error updating", uuid, str(s))
+                    else:
+                        if result:
+
+                            datastore.update_watch(uuid=uuid, update_obj=result)
+                            if changed_detected:
+                                # A change was detected
+                                datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
+
+                self.current_uuid = None  # Done
+                self.q.task_done()
+
+            app.config.exit.wait(1)
 
 
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
-    from backend import update_worker
-
     # Spin up Workers.
     for _ in range(datastore.data['settings']['requests']['workers']):
-        new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
+        new_worker = Worker(update_q)
         running_update_threads.append(new_worker)
         new_worker.start()
 
     while not app.config.exit.is_set():
-
         running_uuids = []
         for t in running_update_threads:
-            if t.current_uuid:
-                running_uuids.append(t.current_uuid)
+            running_uuids.append(t.current_uuid)
 
         # Look at the dataset, find a stale watch to process
 
@@ -832,7 +660,7 @@ def ticker_thread_check_time_launch_checks():
 
         threshold = time.time() - (minutes * 60)
         for uuid, watch in datastore.data['watching'].items():
-            if not watch['paused'] and watch['last_checked'] <= threshold:
+            if watch['last_checked'] <= threshold:
                 if not uuid in running_uuids and uuid not in update_q.queue:
                     update_q.put(uuid)
 

backend/pytest.ini

@@ -1,7 +1,7 @@
 [pytest]
 addopts = --no-start-live-server --live-server-port=5005
 #testpaths = tests pytest_invenio
-#live_server_scope = function
+#live_server_scope = session
 
 filterwarnings =
     ignore::DeprecationWarning:urllib3.*:

backend/run_all_tests.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-
-# live_server will throw errors even with live_server_scope=function if I have the live_server setup in different functions
-# and I like to restart the server for each test (and have the test cleanup after each test)
-# merge request welcome :)
-
-
-# exit when any command fails
-set -e
-
-find tests/test_*py -type f|while read test_name
-do
-  echo "TEST RUNNING $test_name"
-  pytest $test_name
-done

backend/static/css/styles.css

@@ -267,11 +267,7 @@ footer {
     color: #e07171;
 }
 
-.paused-state.state-False img {
-    opacity: 0.2;
+#diff-stream {
+    font-size: 10px;
+    white-space: pre-wrap;
 }
-
-
-.paused-state.state-False:hover img{
-    opacity: 0.8;
-}

backend/static/images/pause.svg

@@ -1,84 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   version="1.1"
-   id="Capa_1"
-   x="0px"
-   y="0px"
-   viewBox="0 0 15 14.998326"
-   xml:space="preserve"
-   width="15"
-   height="14.998326"><metadata
-   id="metadata39"><rdf:RDF><cc:Work
-       rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
-         rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
-   id="defs37" />
-<path
-   id="path2"
-   style="fill:#1b98f8;fill-opacity:1;stroke-width:0.0292893"
-   d="M 7.4975161,6.5052867e-4 C 4.549072,-0.04028702 1.7055675,1.8548221 0.58868606,4.5801341 -0.57739762,7.2574642 0.02596981,10.583326 2.069916,12.671949 4.0364753,14.788409 7.2763651,15.56067 9.989207,14.57284 12.801145,13.617602 14.87442,10.855325 14.985833,7.8845744 15.172496,4.9966544 13.49856,2.1100704 10.911002,0.8209349 9.8598067,0.28073592 8.6791261,-0.00114855 7.4975161,6.5052867e-4 Z M 6.5602569,10.251923 c -0.00509,0.507593 -0.5693885,0.488472 -0.9352002,0.468629 -0.3399386,0.0018 -0.8402048,0.07132 -0.9297965,-0.374189 -0.015842,-1.8973128 -0.015872,-3.7979649 0,-5.6952784 0.1334405,-0.5224315 0.7416869,-0.3424086 1.1377562,-0.374189 0.3969969,-0.084515 0.8245634,0.1963256 0.7272405,0.6382917 0,1.7789118 0,3.5578239 0,5.3367357 z m 3.7490371,0 c -0.0051,0.507593 -0.5693888,0.488472 -0.9352005,0.468629 -0.3399386,0.0018 -0.8402048,0.07132 -0.9297965,-0.374189 -0.015842,-1.8973128 -0.015872,-3.7979649 0,-5.6952784 0.1334405,-0.5224315 0.7416869,-0.3424086 1.1377562,-0.374189 0.3969969,-0.084515 0.8245638,0.1963256 0.7272408,0.6382917 0,1.7789118 0,3.5578239 0,5.3367357 z" />
-<g
-   id="g4"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g6"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g8"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g10"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g12"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g14"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g16"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g18"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g20"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g22"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g24"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g26"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g28"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g30"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-<g
-   id="g32"
-   transform="translate(-0.01903604,0.02221043)">
-</g>
-</svg>

backend/store.py

@@ -37,10 +37,6 @@ class ChangeDetectionStore:
                     'timeout': 15,  # Default 15 seconds
                     'minutes_between_check': 3 * 60,  # Default 3 hours
                     'workers': 10  # Number of threads, lower is better for slow connections
-                },
-                'application': {
-                    'password': False,
-                    'notification_urls': [] # Apprise URL list
                 }
             }
         }
@@ -51,7 +47,6 @@ class ChangeDetectionStore:
             'tag': None,
             'last_checked': 0,
             'last_changed': 0,
-            'paused': False,
             'last_viewed': 0,  # history key value of the last viewed via the [diff] link
             'newest_history_key': "",
             'title': None,
@@ -59,8 +54,7 @@ class ChangeDetectionStore:
             'uuid': str(uuid_builder.uuid4()),
             'headers': {},  # Extra headers to send
             'history': {},  # Dict of timestamp and output stripped filename
-            'ignore_text': [], # List of text to ignore when calculating the comparison checksum
-            'notification_urls': [] # List of URLs to add to the notification Queue (Usually AppRise)
+            'ignore_text': [] # List of text to ignore when calculating the comparison checksum
         }
 
         if path.isfile('/source.txt'):
@@ -89,9 +83,6 @@ class ChangeDetectionStore:
                     if 'requests' in from_disk['settings']:
                         self.__data['settings']['requests'].update(from_disk['settings']['requests'])
 
-                    if 'application' in from_disk['settings']:
-                        self.__data['settings']['application'].update(from_disk['settings']['application'])
-
                 # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
                 # @todo pretty sure theres a python we todo this with an abstracted(?) object!
                 for uuid, watch in self.__data['watching'].items():
@@ -111,15 +102,11 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
                 self.add_watch(url='https://changedetection.io', tag='Tech news')
 
-        self.__data['version_tag'] = "0.30"
+
+        self.__data['version_tag'] = "0.27"
 
         if not 'app_guid' in self.__data:
-            import sys
-            import os
-            if "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ:
-                self.__data['app_guid'] = "test-" + str(uuid_builder.uuid4())
-            else:
-                self.__data['app_guid'] = str(uuid_builder.uuid4())
+            self.__data['app_guid'] = str(uuid_builder.uuid4())
 
         self.needs_write = True
 
@@ -147,10 +134,6 @@ class ChangeDetectionStore:
 
     def update_watch(self, uuid, update_obj):
 
-        # Skip if 'paused' state
-        if self.__data['watching'][uuid]['paused']:
-            return
-
         with self.lock:
 
             # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
@@ -169,6 +152,7 @@ class ChangeDetectionStore:
     def data(self):
 
         has_unviewed = False
+
         for uuid, v in self.__data['watching'].items():
             self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
             if int(v['newest_history_key']) <= int(v['last_viewed']):
@@ -195,35 +179,19 @@ class ChangeDetectionStore:
         tags.sort()
         return tags
 
-    def unlink_history_file(self, path):
-        try:
-            os.unlink(path)
-        except (FileNotFoundError, IOError):
-            pass
-
-    # Delete a single watch by UUID
     def delete(self, uuid):
         with self.lock:
             if uuid == 'all':
                 self.__data['watching'] = {}
-
-                # GitHub #30 also delete history records
-                for uuid in self.data['watching']:
-                    for path in self.data['watching'][uuid]['history'].values():
-                        self.unlink_history_file(path)
-
             else:
-                for path in self.data['watching'][uuid]['history'].values():
-                    self.unlink_history_file(path)
-
-                del self.data['watching'][uuid]
+                del (self.__data['watching'][uuid])
 
             self.needs_write = True
 
     def url_exists(self, url):
 
         # Probably their should be dict...
-        for watch in self.data['watching'].values():
+        for watch in self.data['watching']:
             if watch['url'] == url:
                 return True
 
@@ -233,47 +201,6 @@ class ChangeDetectionStore:
         # Probably their should be dict...
         return self.data['watching'][uuid].get(val)
 
-    # Remove a watchs data but keep the entry (URL etc)
-    def scrub_watch(self, uuid, limit_timestamp = False):
-
-        import hashlib
-        del_timestamps = []
-
-        for timestamp, path in self.data['watching'][uuid]['history'].items():
-            if not limit_timestamp or (limit_timestamp is not False and int(timestamp) > limit_timestamp):
-                self.unlink_history_file(path)
-                del_timestamps.append(timestamp)
-
-
-
-                if not limit_timestamp:
-                    self.data['watching'][uuid]['last_checked'] = 0
-                    self.data['watching'][uuid]['last_changed'] = 0
-                    self.data['watching'][uuid]['previous_md5'] = 0
-
-        for timestamp in del_timestamps:
-            del self.data['watching'][uuid]['history'][str(timestamp)]
-
-            # If there was a limitstamp, we need to reset some meta data about the entry
-            # This has to happen after we remove the others from the list
-            if limit_timestamp:
-                newest_key = self.get_newest_history_key(uuid)
-                if newest_key:
-                    self.data['watching'][uuid]['last_checked'] = int(newest_key)
-                    # @todo should be the original value if it was less than newest key
-                    self.data['watching'][uuid]['last_changed'] = int(newest_key)
-                    try:
-                        with open(self.data['watching'][uuid]['history'][str(newest_key)], "rb") as fp:
-                            content = fp.read()
-                        self.data['watching'][uuid]['previous_md5'] = hashlib.md5(content).hexdigest()
-                    except (FileNotFoundError, IOError):
-                        self.data['watching'][uuid]['previous_md5'] = False
-                        pass
-
-
-        self.needs_write = True
-
-
     def add_watch(self, url, tag):
         with self.lock:
             # @todo use a common generic version of this
@@ -301,6 +228,12 @@ class ChangeDetectionStore:
     # result_obj from fetch_site_status.run()
     def save_history_text(self, uuid, result_obj, contents):
 
+        output_path = "{}/{}".format(self.datastore_path, uuid)
+        try:
+            os.mkdir(output_path)
+        except FileExistsError:
+            pass
+
         output_path = "{}/{}".format(self.datastore_path, uuid)
         fname = "{}/{}-{}.stripped.txt".format(output_path, result_obj['previous_md5'], str(time.time()))
         with open(fname, 'w') as f:
@@ -315,21 +248,11 @@ class ChangeDetectionStore:
 
     def sync_to_json(self):
         print("Saving..")
-        data ={}
+        with open(self.json_store_path, 'w') as json_file:
+            json.dump(self.__data, json_file, indent=4)
+            logging.info("Re-saved index")
 
-        try:
-            data = deepcopy(self.__data)
-        except RuntimeError:
-            time.sleep(0.5)
-            print ("! Data changed when writing to JSON, trying again..")
-            self.sync_to_json()
-            return
-        else:
-            with open(self.json_store_path, 'w') as json_file:
-                json.dump(data, json_file, indent=4)
-                logging.info("Re-saved index")
-
-            self.needs_write = False
+        self.needs_write = False
 
     # Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON
     # by just running periodically in one thread, according to python, dict updates are threadsafe.
@@ -339,8 +262,8 @@ class ChangeDetectionStore:
             if self.stop_thread:
                 print("Shutting down datastore thread")
                 return
-            
             if self.needs_write:
                 self.sync_to_json()
-            time.sleep(3)
+            time.sleep(1)
 
+# body of the constructor

backend/templates/base.html

@@ -17,11 +17,7 @@
 
 <div class="header">
     <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed">
-        {% if has_password and not current_user.is_authenticated %}
-            <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener"><strong>Change</strong>Detection.io</a>
-        {% else %}
-            <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
-        {% endif %}
+        <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
         {% if current_diff_url %}
         <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</span></a>
         {% else %}
@@ -31,7 +27,7 @@
         {% endif %}
 
         <ul class="pure-menu-list">
-        {% if current_user.is_authenticated or not has_password %}
+
             <li class="pure-menu-item">
                 <a href="/backup" class="pure-menu-link">BACKUP</a>
             </li>
@@ -41,15 +37,6 @@
             <li class="pure-menu-item">
                 <a href="/settings" class="pure-menu-link">SETTINGS</a>
             </li>
-        {% else %}
-            <li class="pure-menu-item">
-                <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
-            </li>
-        {% endif %}
-
-        {% if current_user.is_authenticated %}
-            <li class="pure-menu-item"><a href="/logout" class="pure-menu-link">LOG OUT</a></li>
-        {% endif %}
             <li class="pure-menu-item"><a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
                 <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16"
                      version="1.1"
@@ -58,6 +45,10 @@
                           d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
                 </svg>
             </a></li>
+            <!--
+            <li class="pure-menu-item"><a href="#" class="pure-menu-link">Tour</a></li>
+            <li class="pure-menu-item"><a href="#" class="pure-menu-link">Sign Up</a></li>
+            -->
         </ul>
     </div>
 </div>

backend/templates/diff.html

@@ -8,7 +8,7 @@
         <fieldset>
 
             <label for="diffWords" class="pure-checkbox">
-                <input type="radio" name="diff_type" id="diffWords" value="diffWords"/> Words</label>
+                <input type="radio" name="diff_type" id="diffWords" value="diffWords" /> Words</label>
             <label for="diffLines" class="pure-checkbox">
                 <input type="radio" name="diff_type" id="diffLines" value="diffLines" checked=""/> Lines</label>
 
@@ -19,9 +19,9 @@
             <label for="diff-version">Compare newest (<span id="current-v-date"></span>) with</label>
             <select id="diff-version" name="previous_version">
                 {% for version in versions %}
-                <option value="{{version}}" {% if version== current_previous_version %} selected="" {% endif %}>
-                    {{version}}
-                </option>
+                    <option value="{{version}}" {% if version== current_previous_version %} selected="" {% endif %}>
+                        {{version}}
+                    </option>
                 {% endfor %}
             </select>
             <button type="submit" class="pure-button pure-button-primary">Go</button>
@@ -90,10 +90,6 @@ function changed() {
 
 	result.textContent = '';
 	result.appendChild(fragment);
-
-	// Jump at start
-	inputs.current=0;
-    next_diff();
 }
 
 window.onload = function() {
@@ -116,7 +112,6 @@ window.onload = function() {
 
 	onDiffTypeChange(document.querySelector('#settings [name="diff_type"]:checked'));
 	changed();
-
 };
 
 a.onpaste = a.onchange =
@@ -145,7 +140,6 @@ for (var i = 0; i < radio.length; i++) {
 var inputs = document.getElementsByClassName('change');
 inputs.current=0;
 
-
 function next_diff() {
 
     var element = inputs[inputs.current];
@@ -165,7 +159,6 @@ function next_diff() {
 }
 
 
-
 </script>
 
 

backend/templates/edit.html

@@ -49,24 +49,8 @@ User-Agent: wonderbra 1.0"
                 <br/>
 
             </fieldset>
-            <div class="pure-control-group">
-                <label for="tag">Notification URLs</label>
-                <textarea id="notification_urls" name="notification_urls" class="pure-input-1-2" placeholder=""
-                          style="width: 100%;
-                            font-family:monospace;
-                            white-space: pre;
-                            overflow-wrap: normal;
-                            overflow-x: scroll;" rows="5">{% for value in watch.notification_urls %}{{ value }}
-{% endfor %}</textarea>
-                <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</a> </span>
-                <br/>
-                <div class="pure-controls">
-                    <span class="pure-form-message-inline"><label for="trigger-test-notification" class="pure-checkbox">
-                        <input type="checkbox" id="trigger-test-notification" name="trigger-test-notification"> Send test notification on save.</label></span>
 
-                </div>
-            </div>
-            <br/>
+
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Save</button>
             </div>

backend/templates/login.html

@@ -1,20 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-<div class="edit-form">
-    <form class="pure-form pure-form-stacked" action="/login" method="POST">
-        <fieldset>
-            <div class="pure-control-group">
-                <label for="password">Password</label>
-                <input type="password" id="password" required="" name="password" value=""
-                       size="15"/>
-                <input type="hidden" id="email" name="email" value="defaultuser@changedetection.io" />
-            </div>
-            <div class="pure-control-group">
-                <button type="submit" class="pure-button pure-button-primary">Login</button>
-            </div>
-        </fieldset>
-    </form>
-</div>
-
-{% endblock %}

backend/templates/preview.html

@@ -1,26 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-
-<div id="settings">
-    <h1>Current</h1>
-</div>
-
-
-<div id="diff-ui">
-
-    <table>
-        <tbody>
-        <tr>
-            <!-- just proof of concept copied straight from github.com/kpdecker/jsdiff -->
-
-            <td id="diff-col">
-                <span id="result">{% for row in content %}<pre>{{row}}</pre>{% endfor %}</span>
-            </td>
-        </tr>
-        </tbody>
-    </table>
-</div>
-
-
-{% endblock %}

backend/templates/scrub.html

@@ -17,19 +17,14 @@
 
             <div class="pure-control-group">
                 <br/>
-                <label for="confirmtext">Confirm text</label><br/>
+                <label for="confirmtext">Confirm</label><br/>
                 <input type="text" id="confirmtext" required="" name="confirmtext" value="" size="10"/>
-            </div>
-
-
-            <div class="pure-control-group">
-                <br/>
-                <label for="confirmtext">Limit delete history including and after date</label><br/>
-                <input type="text" id="limit_date" required="" name="limit_date" value="" size="10"/>
                 <br/>
 
 
             </div>
+
+
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Scrub!</button>
             </div>

backend/templates/settings.html

@@ -12,36 +12,7 @@
                        size="5"/>
                 <span class="pure-form-message-inline">This is a required field.</span>
             </div>
-            <br/>
-            <hr>
-            <div class="pure-control-group">
-                <label for="minutes">Password protection</label>
-                <input type="password" id="password" name="password" size="15"/>
-                {% if current_user.is_authenticated %}
-                    <a href="/settings?removepassword=true" class="pure-button pure-button-primary">Remove password</a>
-                {% endif %}
-            </div>
 
-            <br/>
-            <hr>
-
-            <div class="pure-control-group">
-                <label for="minutes">Global notification settings</label><br/>
-                Notification URLs <a href="https://github.com/caronc/apprise"> see Apprise examples</a>.
-                <textarea style="overflow-wrap: normal;  overflow-x: scroll;" id="notification_urls" name="notification_urls" cols="80"
-                          rows="6" wrap=off placeholder="Example:
-
-Gitter - gitter://token/room
-Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
-AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
-SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
-">{{notification_urls}}</textarea>
-            </div>
-                <div class="pure-controls">
-                    <span class="pure-form-message-inline"><label for="trigger-test-notification" class="pure-checkbox">
-                        <input type="checkbox" id="trigger-test-notification" name="trigger-test-notification"> Send test notification on save.</label></span>
-
-                </div>
 
             <br/>
             <div class="pure-control-group">
@@ -51,7 +22,7 @@ SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
 
             <div class="pure-control-group">
                 <a href="/" class="pure-button button-small button-cancel">Back</a>
-                <a href="/scrub" class="pure-button button-small button-cancel">Delete history version data</a>
+                <a href="/scrub" class="pure-button button-small button-cancel">Reset all version data</a>
             </div>
 
 

backend/templates/watch-diff-stream.html

@@ -0,0 +1,12 @@
+{% extends 'watch-overview.html' %}
+{% block innercontent %}
+Entries: {{ streams|length }}
+
+<div id="diff-stream" class="edit-form">
+    {% for item in streams %}
+        {{ loop.index }}
+{% for diff in item %}{% if diff[0] =='+' %}<ins>{{ diff }}</ins>{% endif %}{% if diff[0] =='-' %}<del>{{ diff }}</del>{% endif %}
+        {% endfor %}
+    {% endfor %}
+</div>
+{% endblock %}

backend/templates/watch-overview.html

@@ -24,74 +24,9 @@
     </div>
 
     <div id="watch-table-wrapper">
-        <table class="pure-table pure-table-striped watch-table">
-            <thead>
-            <tr>
-                <th>#</th>
-                <th></th>
-                <th></th>
-                <th>Last Checked</th>
-                <th>Last Changed</th>
-                <th></th>
-            </tr>
-            </thead>
-            <tbody>
+    {% block innercontent %}
 
-
-            {% for watch in watches %}
-            <tr id="{{ watch.uuid }}"
-                class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
-                {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
-                {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
-                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
-                <td>{{ loop.index }}</td>
-                <td class="paused-state state-{{watch.paused}}"><a href="/?pause={{ watch.uuid}}{% if active_tag %}&tag={{active_tag}}{% endif %}"><img src="/static/images/pause.svg" alt="Pause"/></a></td>
-                <td class="title-col">{{watch.title if watch.title is not none else watch.url}}
-                    <a class="external" target="_blank" rel="noopener" href="{{ watch.url }}"></a>
-                    {% if watch.last_error is defined and watch.last_error != False %}
-                    <div class="fetch-error">{{ watch.last_error }}</div>
-                    {% endif %}
-                    {% if not active_tag %}
-                    <span class="watch-tag-list">{{ watch.tag}}</span>
-                    {% endif %}
-                </td>
-                <td>{{watch|format_last_checked_time}}</td>
-                <td>{% if watch.history|length >= 2 and watch.last_changed %}
-                    {{watch.last_changed|format_timestamp_timeago}}
-                    {% else %}
-                    Not yet
-                    {% endif %}
-                </td>
-                <td>
-                    <a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
-                       class="pure-button button-small pure-button-primary">Recheck</a>
-                    <a href="/edit/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
-                    {% if watch.history|length >= 2 %}
-                    <a href="/diff/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
-                    {% else %}
-                        {% if watch.history|length == 1 %}
-                            <a href="/preview/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
-                        {% endif %}
-                    {% endif %}
-                </td>
-            </tr>
-            {% endfor %}
-            </tbody>
-        </table>
-        <ul id="post-list-buttons">
-            {% if has_unviewed %}
-            <li>
-                <a href="/api/mark-all-viewed" class="pure-button button-tag ">Mark all viewed</a>
-            </li>
-            {% endif %}
-            <li>
-               <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
-                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
-            </li>
-            <li>
-                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="/static/images/Generic_Feed-icon.svg" height="15px"></a>
-            </li>
-        </ul>
+    {% endblock %}
     </div>
 </div>
 {% endblock %}

backend/templates/watch-table.html

@@ -0,0 +1,64 @@
+{% extends 'watch-overview.html' %}
+{% block innercontent %}
+        <table class="pure-table pure-table-striped watch-table">
+            <thead>
+            <tr>
+                <th>#</th>
+                <th></th>
+                <th>Last Checked</th>
+                <th>Last Changed</th>
+                <th></th>
+            </tr>
+            </thead>
+            <tbody>
+
+
+            {% for watch in watches %}
+            <tr id="{{ watch.uuid }}"
+                class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
+                {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
+                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
+                <td>{{ loop.index }}</td>
+                <td class="title-col">{{watch.title if watch.title is not none else watch.url}}
+                    <a class="external" target=_blank href="{{ watch.url }}"></a>
+                    {% if watch.last_error is defined and watch.last_error != False %}
+                    <div class="fetch-error">{{ watch.last_error }}</div>
+                    {% endif %}
+                    {% if not active_tag %}
+                    <span class="watch-tag-list">{{ watch.tag}}</span>
+                    {% endif %}
+                </td>
+                <td>{{watch|format_last_checked_time}}</td>
+                <td>{% if watch.history|length >= 2 and watch.last_changed %}
+                    {{watch.last_changed|format_timestamp_timeago}}
+                    {% else %}
+                    Not yet
+                    {% endif %}
+                </td>
+                <td>
+                    <a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
+                       class="pure-button button-small pure-button-primary">Recheck</a>
+                    <a href="/edit/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
+                    {% if watch.history|length >= 2 %}
+                    <a href="/diff/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
+                    {% endif %}
+                </td>
+            </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+        <ul id="post-list-buttons">
+            {% if has_unviewed %}
+            <li>
+                <a href="/api/mark-all-viewed" class="pure-button button-tag ">Mark all viewed</a>
+            </li>
+            {% endif %}
+            <li>
+               <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
+                all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
+            </li>
+            <li>
+                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="/static/images/Generic_Feed-icon.svg" height="15px"></a>
+            </li>
+        </ul>
+{% endblock %}

backend/tests/conftest.py

@@ -35,12 +35,13 @@ def app(request):
     def teardown():
         datastore.stop_thread = True
         app.config.exit.set()
-        for fname in ["url-watches.json", "count.txt", "output.txt"]:
-            try:
-                os.unlink("{}/{}".format(datastore_path, fname))
-            except FileNotFoundError:
-                # This is fine in the case of a failure.
-                pass
+        try:
+            os.unlink("{}/url-watches.json".format(datastore_path))
+        except FileNotFoundError:
+            # This is fine in the case of a failure.
+            pass
+
+        assert 1 == 1
 
     request.addfinalizer(teardown)
     yield app

backend/tests/test_access_control.py

@@ -1,58 +0,0 @@
-from flask import url_for
-
-def test_check_access_control(app, client):
-    # Still doesnt work, but this is closer.
-    return
-    with app.test_client() as c:
-
-        # Check we dont have any password protection enabled yet.
-        res = c.get(url_for("settings_page"))
-        assert b"Remove password" not in res.data
-
-        # Enable password check.
-        res = c.post(
-            url_for("settings_page"),
-            data={"password": "foobar"},
-            follow_redirects=True
-        )
-        assert b"Password protection enabled." in res.data
-        assert b"LOG OUT" not in res.data
-        print ("SESSION:", res.session)
-        # Check we hit the login
-
-        res = c.get(url_for("settings_page"), follow_redirects=True)
-        res = c.get(url_for("login"), follow_redirects=True)
-
-        assert b"Login" in res.data
-
-        print ("DEBUG >>>>>",res.data)
-        # Menu should not be available yet
-        assert b"SETTINGS" not in res.data
-        assert b"BACKUP" not in res.data
-        assert b"IMPORT" not in res.data
-
-
-
-        #defaultuser@changedetection.io is actually hardcoded for now, we only use a single password
-        res = c.post(
-            url_for("login"),
-            data={"password": "foobar", "email": "defaultuser@changedetection.io"},
-            follow_redirects=True
-        )
-
-        assert b"LOG OUT" in res.data
-        res = c.get(url_for("settings_page"))
-
-        # Menu should be available now
-        assert b"SETTINGS" in res.data
-        assert b"BACKUP" in res.data
-        assert b"IMPORT" in res.data
-
-        assert b"LOG OUT" in res.data
-
-        # Now remove the password so other tests function, @todo this should happen before each test automatically
-
-        c.get(url_for("settings_page", removepassword="true"))
-        c.get(url_for("import_page"))
-        assert b"LOG OUT" not in res.data
-

backend/tests/test_backend.py

@@ -3,16 +3,55 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
-from . util import set_original_response, set_modified_response, live_server_setup
+import pytest
 
 sleep_time_for_fetch_thread = 3
 
 
+def test_setup_liveserver(live_server):
+    @live_server.app.route('/test-endpoint')
+    def test_endpoint():
+        # Tried using a global var here but didn't seem to work, so reading from a file instead.
+        with open("test-datastore/output.txt", "r") as f:
+            return f.read()
+
+    live_server.start()
+
+    assert 1 == 1
+
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>which has this one new line</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
 
 
 def test_check_basic_change_detection_functionality(client, live_server):
     set_original_response()
-    live_server_setup(live_server)
 
     # Add our URL to the import page
     res = client.post(
@@ -36,12 +75,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
         assert b'unviewed' not in res.data
         assert b'test-endpoint' in res.data
 
-        # Default no password set, this stuff should be always available.
-
-        assert b"SETTINGS" in res.data
-        assert b"BACKUP" in res.data
-        assert b"IMPORT" in res.data
-
     #####################
 
     # Make a change
@@ -88,4 +121,3 @@ def test_check_basic_change_detection_functionality(client, live_server):
     # Cleanup everything
     res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
-

backend/tests/test_ignore_text.py

@@ -2,10 +2,9 @@
 
 import time
 from flask import url_for
-from . util import live_server_setup
+from urllib.request import urlopen
+import pytest
 
-def test_setup(live_server):
-    live_server_setup(live_server)
 
 # Unit test of the stripper
 # Always we are dealing in utf-8

backend/tests/test_notification.py

@@ -1,66 +0,0 @@
-
-import time
-from flask import url_for
-from . util import set_original_response, set_modified_response, live_server_setup
-
-# Hard to just add more live server URLs when one test is already running (I think)
-# So we add our test here (was in a different file)
-def test_check_notification(client, live_server):
-
-    live_server_setup(live_server)
-    set_original_response()
-
-    # Give the endpoint time to spin up
-    time.sleep(3)
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    # Give the thread time to pick it up
-    time.sleep(3)
-
-    # Goto the edit page, add our ignore text
-    # Add our URL to the import page
-    url = url_for('test_notification_endpoint', _external=True)
-    notification_url = url.replace('http', 'json')
-
-    print (">>>> Notification URL: "+notification_url)
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"notification_urls": notification_url, "url": test_url, "tag": "", "headers": ""},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
-    # Hit the edit page, be sure that we saved it
-    res = client.get(
-        url_for("edit_page", uuid="first"))
-    assert bytes(notification_url.encode('utf-8')) in res.data
-
-    set_modified_response()
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(3)
-
-    # Did the front end see it?
-    res = client.get(
-        url_for("index"))
-    assert bytes("just now".encode('utf-8')) in res.data
-
-
-    # Check it triggered
-    res = client.get(
-        url_for("test_notification_counter"),
-    )
-    print (res.data)
-
-    assert bytes("we hit it".encode('utf-8')) in res.data

backend/tests/util.py

@@ -1,60 +0,0 @@
-#!/usr/bin/python3
-
-
-def set_original_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>Which is across multiple lines</p>
-     </br>
-     So let's see what happens.  </br>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/output.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-def set_modified_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>which has this one new line</p>
-     </br>
-     So let's see what happens.  </br>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/output.txt", "w") as f:
-        f.write(test_return_data)
-
-    return None
-
-
-def live_server_setup(live_server):
-
-    @live_server.app.route('/test-endpoint')
-    def test_endpoint():
-        # Tried using a global var here but didn't seem to work, so reading from a file instead.
-        with open("test-datastore/output.txt", "r") as f:
-            return f.read()
-
-    @live_server.app.route('/test_notification_endpoint', methods=['POST'])
-    def test_notification_endpoint():
-        with open("test-datastore/count.txt", "w") as f:
-            f.write("we hit it")
-        print("\n>> Test notification endpoint was hit.\n")
-        return "Text was set"
-
-    # And this should return not zero.
-    @live_server.app.route('/test_notification_counter')
-    def test_notification_counter():
-        try:
-            with open("test-datastore/count.txt", "r") as f:
-                return f.read()
-        except FileNotFoundError:
-            return "nope :("
-
-    live_server.start()

backend/update_worker.py

@@ -1,67 +0,0 @@
-import threading
-import queue
-
-# Requests for checking on the site use a pool of thread Workers managed by a Queue.
-class update_worker(threading.Thread):
-    current_uuid = None
-
-    def __init__(self, q, notification_q, app, datastore, *args, **kwargs):
-        self.q = q
-        self.app = app
-        self.notification_q = notification_q
-        self.datastore = datastore
-        super().__init__(*args, **kwargs)
-
-    def run(self):
-        from backend import fetch_site_status
-
-        update_handler = fetch_site_status.perform_site_check(datastore=self.datastore)
-
-        while not self.app.config.exit.is_set():
-
-            try:
-                uuid = self.q.get(block=False)
-            except queue.Empty:
-                pass
-
-            else:
-                self.current_uuid = uuid
-
-                if uuid in list(self.datastore.data['watching'].keys()):
-                    try:
-                        changed_detected, result, contents = update_handler.run(uuid)
-
-                    except PermissionError as s:
-                        self.app.logger.error("File permission error updating", uuid, str(s))
-                    else:
-                        if result:
-                            try:
-                                self.datastore.update_watch(uuid=uuid, update_obj=result)
-                                if changed_detected:
-                                    # A change was detected
-                                    self.datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
-
-                                    watch = self.datastore.data['watching'][uuid]
-
-                                    # Did it have any notification alerts to hit?
-                                    if len(watch['notification_urls']):
-                                        print("Processing notifications for UUID: {}".format(uuid))
-                                        n_object = {'watch_url': self.datastore.data['watching'][uuid]['url'],
-                                                    'notification_urls': watch['notification_urls']}
-                                        self.notification_q.put(n_object)
-
-
-                                    # No? maybe theres a global setting, queue them all
-                                    elif len(self.datastore.data['settings']['application']['notification_urls']):
-                                        print("Processing GLOBAL notifications for UUID: {}".format(uuid))
-                                        n_object = {'watch_url': self.datastore.data['watching'][uuid]['url'],
-                                                    'notification_urls': self.datastore.data['settings']['application'][
-                                                        'notification_urls']}
-                                        self.notification_q.put(n_object)
-                            except Exception as e:
-                                print("!!!! Exception in update_worker !!!\n", e)
-
-                self.current_uuid = None  # Done
-                self.q.task_done()
-
-            self.app.config.exit.wait(1)

changedetection.py

@@ -3,7 +3,6 @@
 # Launch as a eventlet.wsgi server instance.
 
 import getopt
-import os
 import sys
 
 import eventlet
@@ -13,31 +12,10 @@ import backend
 from backend import store
 
 
-def init_app_secret(datastore_path):
-    secret = ""
-
-    path = "{}/secret.txt".format(datastore_path)
-
-    try:
-        with open(path, "r") as f:
-            secret = f.read()
-
-    except FileNotFoundError:
-
-        import secrets
-        with open(path, "w") as f:
-            secret = secrets.token_hex(32)
-            f.write(secret)
-
-    return secret
-
-
 def main(argv):
     ssl_mode = False
     port = 5000
-
-    # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
-    datastore_path = os.path.join(os.getcwd(), "datastore")
+    datastore_path = "./datastore"
 
     try:
         opts, args = getopt.getopt(argv, "sd:p:", "purge")
@@ -60,21 +38,25 @@ def main(argv):
         if opt == '-d':
             datastore_path = arg
 
+
+
+    # threads can read from disk every x seconds right?
+    # front end can just save
+    # We just need to know which threads are looking at which UUIDs
+
     # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
     app_config = {'datastore_path': datastore_path}
 
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'])
     app = backend.changedetection_app(app_config, datastore)
 
-    app.config['datastore_path'] = datastore_path
-    app.secret_key = init_app_secret(app_config['datastore_path'])
-
     @app.context_processor
     def inject_version():
-        return dict(version=datastore.data['version_tag'],
-                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
-                    has_password=datastore.data['settings']['application']['password'] != False
-                    )
+        return dict(version=datastore.data['version_tag'])
+
+    @app.context_processor
+    def inject_new_version_available():
+        return dict(new_version_available=app.config['NEW_VERSION_AVAILABLE'])
 
     if ssl_mode:
         # @todo finalise SSL config, but this should get you in the right direction if you need it.

docker-compose-development.yml

@@ -0,0 +1,23 @@
+version: "2"
+services:
+
+# I have a feeling we can get rid of this, and just use one docker-compose.yml, and just set a ENV var if
+# we want dev mode (just gives a docker shell) or not.
+
+  backend:
+    build: ./backend/dev-docker
+    image: dgtlmoon/changedetection.io:dev
+    container_name: changedetection.io-dev
+    volumes:
+      - .:/app
+      - ./requirements.txt:/requirements.txt # Normally COPY'ed in the Dockerfile
+      - ./datastore:/datastore
+
+    ports:
+      - "127.0.0.1:5001:5000"
+
+    networks:
+      - changenet
+
+networks:
+  changenet:

requirements.txt

@@ -1,14 +1,13 @@
 chardet==2.3.0
+yarl
 flask~= 1.0
 pytest ~=6.2
-pytest-flask ~=1.2
-eventlet>=0.31.0
-requests ~= 2.15
+pytest-flask ~=1.1
+eventlet ~= 0.30
+requests
 validators
 timeago ~=1.0
 inscriptis ~= 1.1
 feedgen ~= 0.9
-flask-login ~= 0.5
 pytz
-urllib3
-apprise ~= 0.9
+urllib3