v0.39.1

* Re #185 - [feature] Custom text templates for the notification per monitored entry as override.
Bonus points: Adding validation for apprise URLs

.github/workflows/image-javascript.yml

@@ -0,0 +1,88 @@
+name: Javascript/Webdriver support - Test, build and push to Docker Hub :javascript tag
+
+on:
+  push:
+    branches: [  javascript-browser ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Create release metadata
+        run: |
+          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > changedetectionio/source.txt
+          echo ${{ github.ref }} > changedetectionio/tag.txt
+
+      - name: Test with pytest
+        run: |
+          # Each test is totally isolated and performs its own cleanup/reset
+          cd changedetectionio; ./run_all_tests.sh
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: all
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+          version: latest
+          driver-opts: image=moby/buildkit:master
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:javascript-dev
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+
+      - name: Image digest
+        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
+
+# failed: Cache service responded with 503
+#      - name: Cache Docker layers
+#        uses: actions/cache@v2
+#        with:
+#          path: /tmp/.buildx-cache
+#          key: ${{ runner.os }}-buildx-${{ github.sha }}
+#          restore-keys: |
+#            ${{ runner.os }}-buildx-
+
+

.github/workflows/image-tag.yml

@@ -0,0 +1,92 @@
+name: Test, build and push tagged release to Docker Hub
+
+on:
+  push:
+    tags:
+      - '*.*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - uses: olegtarasov/get-tag@v2.1
+        id: tagName
+
+#        with:
+#          tagRegex: "foobar-(.*)"  # Optional. Returns specified group text as tag name. Full tag string is returned if regex is not defined.
+#          tagRegexGroup: 1 # Optional. Default is 1.
+
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Create release metadata
+        run: |
+          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > changedetectionio/source.txt
+          echo ${{ github.ref }} > changedetectionio/tag.txt
+
+      - name: Test with pytest
+        run: |
+          # Each test is totally isolated and performs its own cleanup/reset
+          cd changedetectionio; ./run_all_tests.sh
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: all
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+          version: latest
+          driver-opts: image=moby/buildkit:master
+
+      - name: tag
+        run : echo ${{ github.event.release.tag_name }}
+
+      - name: Build and push tagged version
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ steps.tagName.outputs.tag }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+        env:
+            SOURCE_NAME: ${{ steps.branch_name.outputs.SOURCE_NAME }}
+            SOURCE_BRANCH: ${{ steps.branch_name.outputs.SOURCE_BRANCH }}
+            SOURCE_TAG: ${{ steps.branch_name.outputs.SOURCE_TAG }}
+
+      - name: Image digest
+        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}

.github/workflows/image.yml

@@ -20,6 +20,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
 
       - name: Lint with flake8
         run: |
@@ -30,20 +31,21 @@ jobs:
 
       - name: Create release metadata
         run: |
-          # COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py
-          echo ${{ github.sha }} > backend/source.txt
-          echo ${{ github.ref }} > backend/tag.txt
+          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > changedetectionio/source.txt
+          echo ${{ github.ref }} > changedetectionio/tag.txt
 
       - name: Test with pytest
         run: |
           # Each test is totally isolated and performs its own cleanup/reset
-          cd backend; ./run_all_tests.sh
+          cd changedetectionio; ./run_all_tests.sh
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
         with:
           image: tonistiigi/binfmt:latest
           platforms: all
+
       - name: Login to Docker Hub
         uses: docker/login-action@v1
         with:
@@ -66,10 +68,8 @@ jobs:
           file: ./Dockerfile
           push: true
           tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
-          #                ${{ secrets.DOCKER_HUB_USERNAME }}:/changedetection.io:${{ env.RELEASE_VERSION }}
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
           platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
-#          platforms: linux/amd64
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
 

.github/workflows/test-only.yml

@@ -19,15 +19,28 @@ jobs:
           python -m pip install --upgrade pip
           pip install flake8 pytest
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or undefined names
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
       - name: Test with pytest
         run: |
           # Each test is totally isolated and performs its own cleanup/reset
-          cd backend; ./run_all_tests.sh
+          cd changedetectionio; ./run_all_tests.sh
+
+      - name: Test that pip builds without error
+        run: |
+          pip3 --version
+          python3 -m pip install wheel
+          python3 setup.py bdist_wheel
+          python3 -m pip install dist/changedetection.io-*-none-any.whl --force
+          changedetection.io -d /tmp -p 10000 &
+          sleep 3
+          curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+          killall -9 changedetection.io
+
 

.gitignore

@@ -5,3 +5,6 @@ datastore/url-watches.json
 datastore/*
 __pycache__
 .pytest_cache
+build
+dist
+.vscode/settings.json

Dockerfile

@@ -12,7 +12,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libxslt-dev \
     zlib1g-dev \
     g++
-    
+
 RUN mkdir /install
 WORKDIR /install
 
@@ -47,7 +47,7 @@ COPY --from=builder /dependencies /usr/local
 ENV PYTHONPATH=/usr/local
 
 # The actual flask app
-COPY backend /app/backend
+COPY changedetectionio /app/changedetectionio
 # The eventlet server wrapper
 COPY changedetection.py /app/changedetection.py
 

MANIFEST.in

@@ -0,0 +1,4 @@
+recursive-include changedetectionio/templates *
+recursive-include changedetectionio/static *
+include changedetection.py
+global-exclude *.pyc

README-pip.md

@@ -0,0 +1,71 @@
+#  changedetection.io
+![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)
+<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
+  <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
+</a>
+<a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
+  <img src="https://img.shields.io/github/v/release/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
+</a>
+
+## Self-hosted open source change monitoring of web pages.
+
+_Know when web pages change! Stay ontop of new information!_ 
+
+Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
+
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
+
+#### Example use cases
+
+Know when ...
+
+- Government department updates (changes are often only on their websites)
+- Local government news (changes are often only on their websites)
+- New software releases, security advisories when you're not on their mailing list.
+- Festivals with changes
+- Realestate listing changes
+- COVID related news from government websites
+- Detect and monitor changes in JSON API responses 
+- API monitoring and alerting
+
+**Get monitoring now!**
+
+```bash
+$ pip3 install changedetection.io   
+```
+
+Specify a target for the *datastore path* with `-d` (required) and a *listening port* with `-p` (defaults to `5000`)
+
+```bash
+$ changedetection.io -d /path/to/empty/data/dir -p 5000
+```
+
+
+Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
+
+### Features
+- Website monitoring
+- Change detection of content and analyses
+- Filters on change (Select by CSS or JSON)
+- Triggers (Wait for text, wait for regex)
+- Notification support
+- JSON API Monitoring
+- Parse JSON embedded in HTML
+- (Reverse) Proxy support
+- Javascript support via WebDriver
+- RaspberriPi (arm v6/v7/64 support)
+
+See https://github.com/dgtlmoon/changedetection.io for more information.
+
+
+
+### Support us
+
+Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
+
+Please support us, even small amounts help a LOT.
+
+BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
+
+<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />

README.md

@@ -7,12 +7,14 @@
   <img src="https://img.shields.io/github/v/release/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
 </a>
 
-## Self-hosted change monitoring of web pages.
+## Self-hosted open source change monitoring of web pages.
 
 _Know when web pages change! Stay ontop of new information!_ 
 
 Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
 
+Open source web page monitoring, notification and change detection.
+
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />
 
@@ -26,32 +28,48 @@ Know when ...
 - Festivals with changes
 - Realestate listing changes
 - COVID related news from government websites
+- Detect and monitor changes in JSON API responses 
+- API monitoring and alerting
 
+_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver!</a>_
 
 **Get monitoring now! super simple, one command!**
 
+Run the python code on your own machine by cloning this repository, or with <a href="https://docs.docker.com/get-docker/">docker</a> and/or <a href="https://www.digitalocean.com/community/tutorial_collections/how-to-install-docker-compose">docker-compose</a>
+
+## Installation
+
+### Docker
+
+Check out our Docker hub page https://hub.docker.com/r/dgtlmoon/changedetection.io
+
+With Docker composer, just clone this repository and
 ```bash
-docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
-```  
-
-Now visit http://127.0.0.1:5000 , You should now be able to access the UI.
-
-#### Updating to latest version
-
-Highly recommended :)
-
-```bash
-docker pull dgtlmoon/changedetection.io
-docker kill $(docker ps -a|grep changedetection.io|awk '{print $1}')
-docker rm $(docker ps -a|grep changedetection.io|awk '{print $1}')
-docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
+$ docker-compose up -d
 ```
-  
+Docker standalone
+```bash
+$ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
+```
+
+### Python Pip
+
+Check out our pypi page https://pypi.org/project/changedetection.io/
+
+```bash
+$ pip3 install changedetection.io
+$ changedetection.io -d /path/to/empty/data/dir -p 5000
+```
+
+Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
+
+_Now with per-site configurable support for using a fast built in HTTP fetcher or use a Chrome based fetcher for monitoring of JavaScript websites!_
+
+ 
 ### Screenshots
 
 Examining differences in content.
 
-
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
@@ -78,9 +96,35 @@ Just some examples
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
 
+Now you can also customise your notification content!
+
+### JSON API Monitoring
+
+Detect changes and monitor data in JSON API's by using the built-in JSONPath selectors as a filter / selector.
+
+![image](https://user-images.githubusercontent.com/275001/125165842-0ce01980-e1dc-11eb-9e73-d8137dd162dc.png)
+
+This will re-parse the JSON and apply formatting to the text, making it super easy to monitor and detect changes in JSON API results
+
+![image](https://user-images.githubusercontent.com/275001/125165995-d9ea5580-e1dc-11eb-8030-f0deced2661a.png)
+
+#### Parse JSON embedded in HTML!
+
+When you enable a `json:` filter, you can even automatically extract and parse embedded JSON inside a HTML page! Amazingly handy for sites that build content based on JSON, such as many e-commerce websites. 
+
+```
+<html>
+...
+<script type="application/ld+json">
+  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","price": 23.50 }
+</script>
+```  
+
+`json:$.price` would give `23.50`, or you can extract the whole structure
+
 ### Proxy
 
-A proxy for ChangeDectection.io can be configured by setting environment the 
+A proxy for ChangeDetection.io can be configured by setting environment the 
 `HTTP_PROXY`, `HTTPS_PROXY` variables, examples are also in the `docker-compose.yml`
 
 `NO_PROXY` exclude list can be specified by following `"localhost,192.168.0.0/24"`
@@ -97,16 +141,14 @@ For more information see https://docs.python-requests.org/en/master/user/advance
 
 This proxy support also extends to the notifications https://github.com/caronc/apprise/issues/387#issuecomment-841718867
 
-### Notes
-
-- Does not yet support Javascript
-- Wont work with Cloudfare type "Please turn on javascript" protected pages
-- You can use the 'headers' section to monitor password protected web page changes
 
 ### RaspberriPi support?
 
-RaspberriPi and linux/arm/v6 linux/arm/v7 devices are supported! 
+RaspberriPi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! 
 
+### Windows native support?
+
+Sorry not yet :( https://github.com/dgtlmoon/changedetection.io/labels/windows
 
 ### Support us
 

backend/fetch_site_status.py

@@ -1,145 +0,0 @@
-import time
-import requests
-import hashlib
-from inscriptis import get_text
-import urllib3
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-# Some common stuff here that can be moved to a base class
-class perform_site_check():
-
-    def __init__(self, *args, datastore, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.datastore = datastore
-
-    def strip_ignore_text(self, content, list_ignore_text):
-        import re
-        ignore = []
-        ignore_regex = []
-        for k in list_ignore_text:
-
-            # Is it a regex?
-            if k[0] == '/':
-                ignore_regex.append(k.strip(" /"))
-            else:
-                ignore.append(k)
-
-        output = []
-        for line in content.splitlines():
-
-            # Always ignore blank lines in this mode. (when this function gets called)
-            if len(line.strip()):
-                regex_matches = False
-
-                # if any of these match, skip
-                for regex in ignore_regex:
-                    try:
-                        if re.search(regex, line, re.IGNORECASE):
-                            regex_matches = True
-                    except Exception as e:
-                        continue
-
-                if not regex_matches and not any(skip_text in line for skip_text in ignore):
-                    output.append(line.encode('utf8'))
-
-        return "\n".encode('utf8').join(output)
-
-
-
-    def run(self, uuid):
-        timestamp = int(time.time())  # used for storage etc too
-        stripped_text_from_html = False
-        changed_detected = False
-
-        update_obj = {'previous_md5': self.datastore.data['watching'][uuid]['previous_md5'],
-                      'history': {},
-                      "last_checked": timestamp
-                      }
-
-        extra_headers = self.datastore.get_val(uuid, 'headers')
-
-        # Tweak the base config with the per-watch ones
-        request_headers = self.datastore.data['settings']['headers']
-        request_headers.update(extra_headers)
-
-        # https://github.com/psf/requests/issues/4525
-        # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
-        # do this by accident.
-        if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
-            request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
-
-        try:
-            timeout = self.datastore.data['settings']['requests']['timeout']
-        except KeyError:
-            # @todo yeah this should go back to the default value in store.py, but this whole object should abstract off it
-            timeout = 15
-
-        try:
-            url = self.datastore.get_val(uuid, 'url')
-
-            r = requests.get(url,
-                             headers=request_headers,
-                             timeout=timeout,
-                             verify=False)
-
-            # CSS Filter
-            css_filter = self.datastore.data['watching'][uuid]['css_filter']
-            if css_filter and len(css_filter.strip()):
-                from bs4 import BeautifulSoup
-                soup = BeautifulSoup(r.content, "html.parser")
-                stripped_text_from_html = ""
-                for item in soup.select(css_filter):
-                    text = str(item.get_text()).strip() + '\n'
-                    stripped_text_from_html += text
-
-            else:
-                stripped_text_from_html = get_text(r.text)
-
-        # Usually from networkIO/requests level
-        except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
-            update_obj["last_error"] = str(e)
-            print(str(e))
-
-        except requests.exceptions.MissingSchema:
-            print("Skipping {} due to missing schema/bad url".format(uuid))
-
-        # Usually from html2text level
-        except Exception as e:
-            #        except UnicodeDecodeError as e:
-            update_obj["last_error"] = str(e)
-            print(str(e))
-            # figure out how to deal with this cleaner..
-            # 'utf-8' codec can't decode byte 0xe9 in position 480: invalid continuation byte
-
-
-        else:
-            # We rely on the actual text in the html output.. many sites have random script vars etc,
-            # in the future we'll implement other mechanisms.
-
-            update_obj["last_check_status"] = r.status_code
-            update_obj["last_error"] = False
-
-            if not len(r.text):
-                update_obj["last_error"] = "Empty reply"
-
-            # If there's text to skip
-            # @todo we could abstract out the get_text() to handle this cleaner
-            if len(self.datastore.data['watching'][uuid]['ignore_text']):
-                content = self.strip_ignore_text(stripped_text_from_html,
-                                                 self.datastore.data['watching'][uuid]['ignore_text'])
-            else:
-                content = stripped_text_from_html.encode('utf8')
-
-            fetched_md5 = hashlib.md5(content).hexdigest()
-
-            # could be None or False depending on JSON type
-            if self.datastore.data['watching'][uuid]['previous_md5'] != fetched_md5:
-                changed_detected = True
-
-                # Don't confuse people by updating as last-changed, when it actually just changed from None..
-                if self.datastore.get_val(uuid, 'previous_md5'):
-                    update_obj["last_changed"] = timestamp
-
-                update_obj["previous_md5"] = fetched_md5
-
-        return changed_detected, update_obj, stripped_text_from_html

backend/forms.py

@@ -1,131 +0,0 @@
-from wtforms import Form, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \
-    Field
-from wtforms import widgets
-from wtforms.validators import ValidationError
-from wtforms.fields import html5
-
-
-class StringListField(StringField):
-    widget = widgets.TextArea()
-
-    def _value(self):
-        if self.data:
-            return "\r\n".join(self.data)
-        else:
-            return u''
-
-    # incoming
-    def process_formdata(self, valuelist):
-        if valuelist:
-            # Remove empty strings
-            cleaned = list(filter(None, valuelist[0].split("\n")))
-            self.data = [x.strip() for x in cleaned]
-            p = 1
-        else:
-            self.data = []
-
-
-
-class SaltyPasswordField(StringField):
-    widget = widgets.PasswordInput()
-    encrypted_password = ""
-
-    def build_password(self, password):
-        import hashlib
-        import base64
-        import secrets
-
-        # Make a new salt on every new password and store it with the password
-        salt = secrets.token_bytes(32)
-
-        key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
-        store = base64.b64encode(salt + key).decode('ascii')
-
-        return store
-
-    # incoming
-    def process_formdata(self, valuelist):
-        if valuelist:
-            # Be really sure it's non-zero in length
-            if len(valuelist[0].strip()) > 0:
-                self.encrypted_password = self.build_password(valuelist[0])
-                self.data = ""
-        else:
-            self.data = False
-
-
-# Separated by  key:value
-class StringDictKeyValue(StringField):
-    widget = widgets.TextArea()
-
-    def _value(self):
-        if self.data:
-            output = u''
-            for k in self.data.keys():
-                output += "{}: {}\r\n".format(k, self.data[k])
-
-            return output
-        else:
-            return u''
-
-    # incoming
-    def process_formdata(self, valuelist):
-        if valuelist:
-            self.data = {}
-            # Remove empty strings
-            cleaned = list(filter(None, valuelist[0].split("\n")))
-            for s in cleaned:
-                parts = s.strip().split(':')
-                if len(parts) == 2:
-                    self.data.update({parts[0].strip(): parts[1].strip()})
-
-        else:
-            self.data = {}
-
-class ListRegex(object):
-    """
-    Validates that anything that looks like a regex passes as a regex
-    """
-    def __init__(self, message=None):
-        self.message = message
-
-    def __call__(self, form, field):
-        import re
-
-        for line in field.data:
-            if line[0] == '/' and line[-1] == '/':
-                # Because internally we dont wrap in /
-                line = line.strip('/')
-                try:
-                    re.compile(line)
-                except re.error:
-                    message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
-                    raise ValidationError(message % (line))
-
-
-class watchForm(Form):
-    # https://wtforms.readthedocs.io/en/2.3.x/fields/#module-wtforms.fields.html5
-    # `require_tld` = False is needed even for the test harness "http://localhost:5005.." to run
-
-    url = html5.URLField('URL', [validators.URL(require_tld=False)])
-    tag = StringField('Tag', [validators.Optional(), validators.Length(max=35)])
-    minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
-                                               [validators.Optional(), validators.NumberRange(min=1)])
-    css_filter = StringField('CSS Filter')
-    title = StringField('Title')
-
-    ignore_text = StringListField('Ignore Text', [ListRegex()])
-    notification_urls = StringListField('Notification URL List')
-    headers = StringDictKeyValue('Request Headers')
-    trigger_check = BooleanField('Send test notification on save')
-
-
-class globalSettingsForm(Form):
-
-    password = SaltyPasswordField()
-
-    minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
-                                               [validators.NumberRange(min=1)])
-
-    notification_urls = StringListField('Notification URL List')
-    trigger_check = BooleanField('Send test notification on save')

backend/static/styles/styles.css

@@ -1,313 +0,0 @@
-/*
- * -- BASE STYLES --
- * Most of these are inherited from Base, but I want to change a few.
- * npm run scss
- */
-body {
-  color: #333;
-  background: #262626; }
-
-.pure-table-even {
-  background: #fff; }
-
-/* Some styles from https://css-tricks.com/ */
-a {
-  text-decoration: none;
-  color: #1b98f8; }
-
-a.github-link {
-  color: #fff; }
-
-.pure-menu-horizontal {
-  background: #fff;
-  padding: 5px;
-  display: flex;
-  justify-content: space-between;
-  border-bottom: 2px solid #ed5900;
-  align-items: center; }
-
-section.content {
-  padding-top: 5em;
-  padding-bottom: 5em;
-  flex-direction: column;
-  display: flex;
-  align-items: center;
-  justify-content: center; }
-
-/* table related */
-.watch-table {
-  width: 100%; }
-  .watch-table tr.unviewed {
-    font-weight: bold; }
-  .watch-table .error {
-    color: #a00; }
-  .watch-table td {
-    font-size: 80%;
-    white-space: nowrap; }
-  .watch-table td.title-col {
-    word-break: break-all;
-    white-space: normal; }
-  .watch-table th {
-    white-space: nowrap; }
-  .watch-table .title-col a[target="_blank"]::after, .watch-table .current-diff-url::after {
-    content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);
-    margin: 0 3px 0 5px; }
-
-.watch-tag-list {
-  color: #e70069;
-  white-space: nowrap; }
-
-.box {
-  max-width: 80%;
-  flex-direction: column;
-  display: flex;
-  justify-content: center; }
-
-#post-list-buttons {
-  text-align: right;
-  padding: 0px;
-  margin: 0px; }
-  #post-list-buttons li {
-    display: inline-block; }
-  #post-list-buttons a {
-    border-top-left-radius: initial;
-    border-top-right-radius: initial;
-    border-bottom-left-radius: 5px;
-    border-bottom-right-radius: 5px; }
-
-body:after {
-  content: "";
-  background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%); }
-
-body:after, body:before {
-  display: block;
-  height: 600px;
-  position: absolute;
-  top: 0;
-  left: 0;
-  width: 100%;
-  z-index: -1; }
-
-body::after {
-  opacity: 0.91; }
-
-body::before {
-  content: "";
-  background-image: url(/static/images/gradient-border.png); }
-
-body:before {
-  background-size: cover; }
-
-body:after, body:before {
-  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
-  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%); }
-
-.button-small {
-  font-size: 85%; }
-
-.fetch-error {
-  padding-top: 1em;
-  font-size: 60%;
-  max-width: 400px;
-  display: block; }
-
-.edit-form {
-  background: #fff;
-  padding: 2em;
-  margin: 1em;
-  border-radius: 5px; }
-
-.button-secondary {
-  color: white;
-  border-radius: 4px;
-  text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2); }
-
-.button-success {
-  background: #1cb841;
-  /* this is a green */ }
-
-.button-tag {
-  background: #636363;
-  color: #fff;
-  font-size: 65%;
-  border-bottom-left-radius: initial;
-  border-bottom-right-radius: initial; }
-  .button-tag.active {
-    background: #9c9c9c;
-    font-weight: bold; }
-
-.button-error {
-  background: #ca3c3c;
-  /* this is a maroon */ }
-
-.button-warning {
-  background: #df7514;
-  /* this is an orange */ }
-
-.button-secondary {
-  background: #42b8dd;
-  /* this is a light blue */ }
-
-.button-cancel {
-  background: #c8c8c8;
-  /* this is a green */ }
-
-.messages li {
-  list-style: none;
-  padding: 1em;
-  border-radius: 10px;
-  color: #fff;
-  font-weight: bold; }
-  .messages li.message {
-    background: rgba(255, 255, 255, 0.2); }
-  .messages li.error {
-    background: rgba(255, 1, 1, 0.5); }
-  .messages li.notice {
-    background: rgba(255, 255, 255, 0.5); }
-
-#new-watch-form {
-  background: rgba(0, 0, 0, 0.05);
-  padding: 1em;
-  border-radius: 10px;
-  margin-bottom: 1em; }
-
-#new-watch-form legend {
-  color: #fff; }
-
-#new-watch-form input {
-  width: auto !important; }
-
-#diff-col {
-  padding-left: 40px; }
-
-#diff-jump {
-  position: fixed;
-  left: 0px;
-  top: 80px;
-  background: #fff;
-  padding: 10px;
-  border-top-right-radius: 5px;
-  border-bottom-right-radius: 5px;
-  box-shadow: 5px 0 5px -2px #888; }
-
-#diff-jump a {
-  color: #1b98f8;
-  cursor: grabbing;
-  -moz-user-select: none;
-  -webkit-user-select: none;
-  -ms-user-select: none;
-  user-select: none;
-  -o-user-select: none; }
-
-footer {
-  padding: 10px;
-  background: #fff;
-  color: #444;
-  text-align: center; }
-
-#feed-icon {
-  vertical-align: middle; }
-
-#version {
-  position: absolute;
-  top: 80px;
-  right: 0px;
-  font-size: 8px;
-  background: #fff;
-  padding: 10px; }
-
-#new-version-text a {
-  color: #e07171; }
-
-.paused-state.state-False img {
-  opacity: 0.2; }
-
-.paused-state.state-False:hover img {
-  opacity: 0.8; }
-
-.monospaced-textarea textarea {
-  width: 100%;
-  font-family: monospace;
-  white-space: pre;
-  overflow-wrap: normal;
-  overflow-x: scroll; }
-
-.pure-form {
-  /* The input fields with errors */
-  /* The list of errors */ }
-  .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
-    padding-bottom: 1em; }
-    .pure-form .pure-control-group dd, .pure-form .pure-group dd, .pure-form .pure-controls dd {
-      margin: 0px; }
-  .pure-form .error input {
-    background-color: #ffebeb; }
-  .pure-form ul.errors {
-    padding: .5em .6em;
-    border: 1px solid #dd0000;
-    border-radius: 4px;
-    vertical-align: middle;
-    -webkit-box-sizing: border-box;
-    box-sizing: border-box; }
-    .pure-form ul.errors li {
-      margin-left: 1em;
-      color: #dd0000; }
-  .pure-form label {
-    font-weight: bold; }
-  .pure-form input[type=url] {
-    width: 100%; }
-  .pure-form textarea {
-    width: 100%;
-    font-size: 14px; }
-
-@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
-  .box {
-    max-width: 95%; }
-  .edit-form {
-    padding: 0.5em;
-    margin: 0.5em; }
-  #nav-menu {
-    overflow-x: scroll; } }
-
-/*
-Max width before this PARTICULAR table gets nasty
-This query will take effect for any screen smaller than 760px
-and also iPads specifically.
-*/
-@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
-  .watch-table {
-    /* Force table to not be like tables anymore */
-    /* Force table to not be like tables anymore */
-    /* Hide table headers (but not display: none;, for accessibility) */ }
-    .watch-table thead, .watch-table tbody, .watch-table th, .watch-table td, .watch-table tr {
-      display: block; }
-    .watch-table .last-checked::before {
-      color: #555;
-      content: "Last Checked "; }
-    .watch-table .last-changed::before {
-      color: #555;
-      content: "Last Changed "; }
-    .watch-table td.inline {
-      display: inline-block; }
-    .watch-table thead tr {
-      position: absolute;
-      top: -9999px;
-      left: -9999px; }
-    .watch-table .pure-table td, .watch-table .pure-table th {
-      border: none; }
-    .watch-table td {
-      /* Behave  like a "row" */
-      border: none;
-      border-bottom: 1px solid #eee; }
-      .watch-table td:before {
-        /* Top/left values mimic padding */
-        top: 6px;
-        left: 6px;
-        width: 45%;
-        padding-right: 10px;
-        white-space: nowrap; }
-    .watch-table.pure-table-striped tr {
-      background-color: #fff; }
-    .watch-table.pure-table-striped tr:nth-child(2n-1) {
-      background-color: #eee; }
-    .watch-table.pure-table-striped tr:nth-child(2n-1) td {
-      background-color: inherit; } }

backend/templates/_helpers.jinja

@@ -1,12 +0,0 @@
-{% macro render_field(field) %}
-  <dt {% if field.errors %} class="error" {% endif %}>{{ field.label }}
-  <dd {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
-  {% if field.errors %}
-    <ul class=errors>
-    {% for error in field.errors %}
-      <li>{{ error }}</li>
-    {% endfor %}
-    </ul>
-  {% endif %}
-  </dd>
-{% endmacro %}

backend/templates/edit.html

@@ -1,71 +0,0 @@
-{% extends 'base.html' %}
-{% block content %}
-{% from '_helpers.jinja' import render_field %}
-<div class="edit-form monospaced-textarea">
-    <form class="pure-form pure-form-stacked" action="/edit/{{uuid}}" method="POST">
-        <fieldset>
-            <div class="pure-control-group">
-                {{ render_field(form.url, placeholder="https://...", size=30, required=true) }}
-            </div>
-            <div class="pure-control-group">
-                {{ render_field(form.title, size=30) }}
-            </div>
-            <div class="pure-control-group">
-                {{ render_field(form.tag, size=10) }}
-            </div>
-            <div class="pure-control-group">
-                {{ render_field(form.minutes_between_check, size=5) }}
-            </div>
-            <div class="pure-control-group">
-                {{ render_field(form.css_filter, size=25, placeholder=".class-name or #some-id, or other CSS selector rule.") }}
-                <span class="pure-form-message-inline">Limit text to this CSS rule, only text matching this CSS rule is included.<br/>
-                    Please be sure that you thoroughly understand how to write CSS selector rules before filing an issue on GitHub!<br/>
-                    Go <a href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>
-                </span>
-            </div>
-            <!-- @todo: move to tabs --->
-            <fieldset class="pure-group">
-                {{ render_field(form.ignore_text, rows=5,  placeholder="Some text to ignore in a line
-/some.regex\d{2}/   for case-INsensitive regex
-") }}
-                <span class="pure-form-message-inline">
-                    Each line processed separately, any line matching will be ignored.<br/>
-                    Regular Expression support, wrap the line in forward slash <b>/regex/</b>.
-                </span>
-
-            </fieldset>
-
-            <fieldset class="pure-group">
-                {{ render_field(form.headers, rows=5, placeholder="Example
-Cookie: foobar
-User-Agent: wonderbra 1.0") }}
-            </fieldset>
-
-            <div class="pure-control-group">
-                {{ render_field(form.notification_urls, rows=5, placeholder="Gitter - gitter://token/room
-Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
-AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
-SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
-") }}
-                <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
-            </div>
-
-            <div class="pure-controls">
-                {{ render_field(form.trigger_check, rows=5) }}
-            </div>
-            <div class="pure-control-group">
-                <button type="submit" class="pure-button pure-button-primary">Save</button>
-            </div>
-            <br/>
-            <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Cancel</a>
-                <a href="/api/delete?uuid={{uuid}}"
-                   class="pure-button button-small button-error ">Delete</a>
-            </div>
-        </fieldset>
-    </form>
-
-
-</div>
-
-{% endblock %}

backend/templates/import.html

@@ -1,26 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-<div class="edit-form">
-
-
-    <form class="pure-form pure-form-aligned" action="/import" method="POST">
-
-        <fieldset class="pure-group">
-            <legend>One URL per line, URLs that do not pass validation will stay in the textarea.</legend>
-
-            <textarea name="urls" class="pure-input-1-2" placeholder="https://"
-                      style="width: 100%;
-                            font-family:monospace;
-                            white-space: pre;
-                            overflow-wrap: normal;
-                            overflow-x: scroll;" rows="25">{{ remaining }}</textarea>
-        </fieldset>
-        <button type="submit" class="pure-button pure-input-1-2 pure-button-primary">Import</button>
-
-    </form>
-
-</div>
-
-{% endblock %}
-

backend/templates/settings.html

@@ -1,51 +0,0 @@
-{% extends 'base.html' %}
-
-{% block content %}
-{% from '_helpers.jinja' import render_field %}
-
-<div class="edit-form">
-    <form class="pure-form pure-form-stacked settings" action="/settings" method="POST">
-        <fieldset>
-            <div class="pure-control-group">
-                {{ render_field(form.minutes_between_check, size=5) }}
-            </div>
-            <div class="pure-control-group">
-                {% if current_user.is_authenticated %}
-                    <a href="/settings?removepassword=true" class="pure-button pure-button-primary">Remove password</a>
-                {% else %}
-                    {{ render_field(form.password, size=10) }}
-                {% endif %}
-            </div>
-            <div class="pure-control-group">
-                {{ render_field(form.notification_urls, rows=5, placeholder="Gitter - gitter://token/room
-Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
-AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
-SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
-") }}
-                <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
-            </div>
-                <div class="pure-controls">
-                    <span class="pure-form-message-inline"><label for="trigger-test-notification" class="pure-checkbox">
-                        <input type="checkbox" id="trigger-test-notification" name="trigger-test-notification"> Send test notification on save.</label></span>
-
-                </div>
-
-            <br/>
-            <div class="pure-control-group">
-                <button type="submit" class="pure-button pure-button-primary">Save</button>
-            </div>
-            <br/>
-
-            <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Back</a>
-                <a href="/scrub" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
-            </div>
-
-
-        </fieldset>
-    </form>
-
-
-</div>
-
-{% endblock %}

backend/tests/test_notification.py

@@ -1,72 +0,0 @@
-
-import time
-from flask import url_for
-from . util import set_original_response, set_modified_response, live_server_setup
-
-# Hard to just add more live server URLs when one test is already running (I think)
-# So we add our test here (was in a different file)
-def test_check_notification(client, live_server):
-
-    live_server_setup(live_server)
-    set_original_response()
-
-    # Give the endpoint time to spin up
-    time.sleep(3)
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    # Give the thread time to pick it up
-    time.sleep(3)
-
-    # Goto the edit page, add our ignore text
-    # Add our URL to the import page
-    url = url_for('test_notification_endpoint', _external=True)
-    notification_url = url.replace('http', 'json')
-
-    print (">>>> Notification URL: "+notification_url)
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"notification_urls": notification_url, "url": test_url, "tag": "", "headers": ""},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
-    # Hit the edit page, be sure that we saved it
-    res = client.get(
-        url_for("edit_page", uuid="first"))
-    assert bytes(notification_url.encode('utf-8')) in res.data
-
-    set_modified_response()
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(3)
-
-    # Did the front end see it?
-    res = client.get(
-        url_for("index"))
-
-    assert bytes("just now".encode('utf-8')) in res.data
-
-
-    # Check it triggered
-    res = client.get(
-        url_for("test_notification_counter"),
-    )
-
-    assert bytes("we hit it".encode('utf-8')) in res.data
-
-    # Did we see the URL that had a change, in the notification?
-    assert bytes("test-endpoint".encode('utf-8')) in res.data
-
-    # Re #65 - did we see our foobar.com BASE_URL ?
-    assert bytes("https://foobar.com".encode('utf-8')) in res.data

backend/update_worker.py

@@ -1,67 +0,0 @@
-import threading
-import queue
-
-# Requests for checking on the site use a pool of thread Workers managed by a Queue.
-class update_worker(threading.Thread):
-    current_uuid = None
-
-    def __init__(self, q, notification_q, app, datastore, *args, **kwargs):
-        self.q = q
-        self.app = app
-        self.notification_q = notification_q
-        self.datastore = datastore
-        super().__init__(*args, **kwargs)
-
-    def run(self):
-        from backend import fetch_site_status
-
-        update_handler = fetch_site_status.perform_site_check(datastore=self.datastore)
-
-        while not self.app.config.exit.is_set():
-
-            try:
-                uuid = self.q.get(block=False)
-            except queue.Empty:
-                pass
-
-            else:
-                self.current_uuid = uuid
-
-                if uuid in list(self.datastore.data['watching'].keys()):
-                    try:
-                        changed_detected, result, contents = update_handler.run(uuid)
-
-                    except PermissionError as s:
-                        self.app.logger.error("File permission error updating", uuid, str(s))
-                    else:
-                        if result:
-                            try:
-                                self.datastore.update_watch(uuid=uuid, update_obj=result)
-                                if changed_detected:
-                                    # A change was detected
-                                    self.datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
-
-                                    watch = self.datastore.data['watching'][uuid]
-
-                                    # Did it have any notification alerts to hit?
-                                    if len(watch['notification_urls']):
-                                        print("Processing notifications for UUID: {}".format(uuid))
-                                        n_object = {'watch_url': self.datastore.data['watching'][uuid]['url'],
-                                                    'notification_urls': watch['notification_urls']}
-                                        self.notification_q.put(n_object)
-
-
-                                    # No? maybe theres a global setting, queue them all
-                                    elif len(self.datastore.data['settings']['application']['notification_urls']):
-                                        print("Processing GLOBAL notifications for UUID: {}".format(uuid))
-                                        n_object = {'watch_url': self.datastore.data['watching'][uuid]['url'],
-                                                    'notification_urls': self.datastore.data['settings']['application'][
-                                                        'notification_urls']}
-                                        self.notification_q.put(n_object)
-                            except Exception as e:
-                                print("!!!! Exception in update_worker !!!\n", e)
-
-                self.current_uuid = None  # Done
-                self.q.task_done()
-
-            self.app.config.exit.wait(1)

changedetection.py

@@ -8,20 +8,20 @@ import sys
 
 import eventlet
 import eventlet.wsgi
-import backend
+import changedetectionio
 
-from backend import store
+from changedetectionio import store
 
-def main(argv):
+def main():
     ssl_mode = False
-    port = 5000
+    port = os.environ.get('PORT') or 5000
     do_cleanup = False
 
     # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
     datastore_path = os.path.join(os.getcwd(), "datastore")
 
     try:
-        opts, args = getopt.getopt(argv, "csd:p:", "port")
+        opts, args = getopt.getopt(sys.argv[1:], "csd:p:", "port")
     except getopt.GetoptError:
         print('backend.py -s SSL enable -p [port] -d [datastore path]')
         sys.exit(2)
@@ -48,8 +48,13 @@ def main(argv):
     # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
     app_config = {'datastore_path': datastore_path}
 
-    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'])
-    app = backend.changedetection_app(app_config, datastore)
+    if not os.path.isdir(app_config['datastore_path']):
+        print ("ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists.\n"
+               "Alternatively, use the -d parameter.".format(app_config['datastore_path']),file=sys.stderr)
+        sys.exit(2)
+
+    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=changedetectionio.__version__)
+    app = changedetectionio.changedetection_app(app_config, datastore)
 
     # Go into cleanup mode
     if do_cleanup:
@@ -60,11 +65,23 @@ def main(argv):
 
     @app.context_processor
     def inject_version():
-        return dict(version=datastore.data['version_tag'],
+        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
                     new_version_available=app.config['NEW_VERSION_AVAILABLE'],
                     has_password=datastore.data['settings']['application']['password'] != False
                     )
 
+    # Proxy sub-directory support
+    # Set environment var USE_X_SETTINGS=1 on this script
+    # And then in your proxy_pass settings
+    #
+    #         proxy_set_header Host "localhost";
+    #         proxy_set_header X-Forwarded-Prefix /app;
+
+    if os.getenv('USE_X_SETTINGS'):
+        print ("USE_X_SETTINGS is ENABLED\n")
+        from werkzeug.middleware.proxy_fix import ProxyFix
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
+
     if ssl_mode:
         # @todo finalise SSL config, but this should get you in the right direction if you need it.
         eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen(('', port)),
@@ -73,8 +90,8 @@ def main(argv):
                                                server_side=True), app)
 
     else:
-        eventlet.wsgi.server(eventlet.listen(('', port)), app)
+        eventlet.wsgi.server(eventlet.listen(('', int(port))), app)
 
 
 if __name__ == '__main__':
-    main(sys.argv[1:])
+    main()

changedetectionio/__init__.py

@@ -29,6 +29,8 @@ from flask import make_response
 import datetime
 import pytz
 
+__version__ = '0.39.1'
+
 datastore = None
 
 # Local
@@ -41,7 +43,11 @@ update_q = queue.Queue()
 
 notification_q = queue.Queue()
 
-app = Flask(__name__, static_url_path="/var/www/change-detection/backend/static")
+# Needs to be set this way because we also build and publish via pip
+base_path = os.path.dirname(os.path.realpath(__file__))
+app = Flask(__name__,
+            static_url_path="{}/static".format(base_path),
+            template_folder="{}/templates".format(base_path))
 
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
@@ -157,7 +163,6 @@ def changedetection_app(config=None, datastore_o=None):
     global datastore
     datastore = datastore_o
 
-    app.config.update(dict(DEBUG=True))
     #app.config.update(config or {})
 
     login_manager = flask_login.LoginManager(app)
@@ -223,7 +228,6 @@ def changedetection_app(config=None, datastore_o=None):
     @login_required
     def index():
         limit_tag = request.args.get('tag')
-
         pause_uuid = request.args.get('pause')
 
         if pause_uuid:
@@ -279,7 +283,11 @@ def changedetection_app(config=None, datastore_o=None):
             return response
 
         else:
+            from changedetectionio import forms
+            form = forms.quickWatchForm(request.form)
+
             output = render_template("watch-overview.html",
+                                     form=form,
                                      watches=sorted_watches,
                                      tags=existing_tags,
                                      active_tag=limit_tag,
@@ -296,25 +304,28 @@ def changedetection_app(config=None, datastore_o=None):
         if request.method == 'POST':
             confirmtext = request.form.get('confirmtext')
             limit_date = request.form.get('limit_date')
+            limit_timestamp = 0
 
-            try:
-                limit_date = limit_date.replace('T', ' ')
-                # I noticed chrome will show '/' but actually submit '-'
-                limit_date = limit_date.replace('-', '/')
-                # In the case that :ss seconds are supplied
-                limit_date = re.sub('(\d\d:\d\d)(:\d\d)', '\\1', limit_date)
+            # Re #149 - allow empty/0 timestamp limit
+            if len(limit_date):
+                try:
+                    limit_date = limit_date.replace('T', ' ')
+                    # I noticed chrome will show '/' but actually submit '-'
+                    limit_date = limit_date.replace('-', '/')
+                    # In the case that :ss seconds are supplied
+                    limit_date = re.sub('(\d\d:\d\d)(:\d\d)', '\\1', limit_date)
 
-                str_to_dt = datetime.datetime.strptime(limit_date, '%Y/%m/%d %H:%M')
-                limit_timestamp = int(str_to_dt.timestamp())
+                    str_to_dt = datetime.datetime.strptime(limit_date, '%Y/%m/%d %H:%M')
+                    limit_timestamp = int(str_to_dt.timestamp())
 
-                if limit_timestamp > time.time():
-                    flash("Timestamp is in the future, cannot continue.", 'error')
+                    if limit_timestamp > time.time():
+                        flash("Timestamp is in the future, cannot continue.", 'error')
+                        return redirect(url_for('scrub_page'))
+
+                except ValueError:
+                    flash('Incorrect date format, cannot continue.', 'error')
                     return redirect(url_for('scrub_page'))
 
-            except ValueError:
-                flash('Incorrect date format, cannot continue.', 'error')
-                return redirect(url_for('scrub_page'))
-
             if confirmtext == 'scrub':
                 changes_removed = 0
                 for uuid, watch in datastore.data['watching'].items():
@@ -338,7 +349,7 @@ def changedetection_app(config=None, datastore_o=None):
     def get_current_checksum_include_ignore_text(uuid):
 
         import hashlib
-        from backend import fetch_site_status
+        from changedetectionio import fetch_site_status
 
         # Get the most recent one
         newest_history_key = datastore.get_val(uuid, 'newest_history_key')
@@ -365,13 +376,14 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
     @login_required
     def edit_page(uuid):
-        from backend import forms
+        from changedetectionio import forms
         form = forms.watchForm(request.form)
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
+
         if request.method == 'GET':
             if not uuid in datastore.data['watching']:
                 flash("No watch with the UUID %s found." % (uuid), "error")
@@ -379,11 +391,27 @@ def changedetection_app(config=None, datastore_o=None):
 
             populate_form_from_watch(form, datastore.data['watching'][uuid])
 
+            if datastore.data['watching'][uuid]['fetch_backend'] is None:
+                form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
+
         if request.method == 'POST' and form.validate():
+
+            # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
+            if form.minutes_between_check.data == datastore.data['settings']['requests']['minutes_between_check']:
+                form.minutes_between_check.data = None
+
+            if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']:
+                form.fetch_backend.data = None
+
             update_obj = {'url': form.url.data.strip(),
+                          'minutes_between_check': form.minutes_between_check.data,
                           'tag': form.tag.data.strip(),
                           'title': form.title.data.strip(),
-                          'headers': form.headers.data
+                          'headers': form.headers.data,
+                          'fetch_backend': form.fetch_backend.data,
+                          'trigger_text': form.trigger_text.data,
+                          'notification_title': form.notification_title.data,
+                          'notification_body': form.notification_body.data
                           }
 
             # Notification URLs
@@ -416,18 +444,36 @@ def changedetection_app(config=None, datastore_o=None):
 
             if form.trigger_check.data:
                 n_object = {'watch_url': form.url.data.strip(),
-                            'notification_urls': form.notification_urls.data}
+                            'notification_urls': form.notification_urls.data,
+                            'notification_title': form.notification_title.data,
+                            'notification_body' :  form.notification_body.data
+                }
                 notification_q.put(n_object)
 
                 flash('Notifications queued.')
 
-            return redirect(url_for('index'))
+            # Diff page [edit] link should go back to diff page
+            if request.args.get("next") and request.args.get("next") == 'diff':
+                return redirect(url_for('diff_history_page', uuid=uuid))
+            else:
+                return redirect(url_for('index'))
 
         else:
             if request.method == 'POST' and not form.validate():
                 flash("An error occurred, please see below.", "error")
 
-            output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], form=form)
+            # Re #110 offer the default minutes
+            using_default_minutes = False
+            if form.minutes_between_check.data == None:
+                form.minutes_between_check.data = datastore.data['settings']['requests']['minutes_between_check']
+                using_default_minutes = True
+
+            output = render_template("edit.html",
+                                     uuid=uuid,
+                                     watch=datastore.data['watching'][uuid],
+                                     form=form,
+                                     using_default_minutes=using_default_minutes
+                                     )
 
         return output
 
@@ -435,50 +481,45 @@ def changedetection_app(config=None, datastore_o=None):
     @login_required
     def settings_page():
 
-        from backend import forms
+        from changedetectionio import forms
+        from changedetectionio import content_fetcher
+
         form = forms.globalSettingsForm(request.form)
 
         if request.method == 'GET':
             form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check'])
             form.notification_urls.data = datastore.data['settings']['application']['notification_urls']
+            form.extract_title_as_title.data = datastore.data['settings']['application']['extract_title_as_title']
+            form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
+            form.notification_title.data = datastore.data['settings']['application']['notification_title']
+            form.notification_body.data = datastore.data['settings']['application']['notification_body']
 
             # Password unset is a GET
-            if request.values.get('removepassword') == 'true':
+            if request.values.get('removepassword') == 'yes':
                 from pathlib import Path
                 datastore.data['settings']['application']['password'] = False
                 flash("Password protection removed.", 'notice')
                 flask_login.logout_user()
+                return redirect(url_for('settings_page'))
 
         if request.method == 'POST' and form.validate():
 
             datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
             datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data
-
-            if len(form.notification_urls.data):
-                import apprise
-                apobj = apprise.Apprise()
-                apobj.debug = True
-
-                # Add each notification
-                for n in datastore.data['settings']['application']['notification_urls']:
-                    apobj.add(n)
-                outcome = apobj.notify(
-                    body='Hello from the worlds best and simplest web page change detection and monitoring service!',
-                    title='Changedetection.io Notification Test',
-                )
-
-                if outcome:
-                    flash("{} Notification URLs reached.".format(len(form.notification_urls.data)), "notice")
-                else:
-                    flash("One or more Notification URLs failed", 'error')
-
+            datastore.data['settings']['application']['extract_title_as_title'] = form.extract_title_as_title.data
+            datastore.data['settings']['application']['fetch_backend'] = form.fetch_backend.data
+            datastore.data['settings']['application']['notification_title'] = form.notification_title.data
+            datastore.data['settings']['application']['notification_body'] = form.notification_body.data
 
             datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
             datastore.needs_write = True
 
-            if form.trigger_check.data:
+            if form.trigger_check.data and len(form.notification_urls.data):
                 n_object = {'watch_url': "Test from changedetection.io!",
-                            'notification_urls': form.notification_urls.data}
+                            'notification_urls': form.notification_urls.data,
+                            'notification_title': form.notification_title.data,
+                            'notification_body': form.notification_body.data
+                            }
                 notification_q.put(n_object)
                 flash('Notifications queued.')
 
@@ -493,7 +534,10 @@ def changedetection_app(config=None, datastore_o=None):
         if request.method == 'POST' and not form.validate():
             flash("An error occurred, please see below.", "error")
 
-        output = render_template("settings.html", form=form)
+        # Same as notification.py
+        base_url = os.getenv('BASE_URL', '').strip('"')
+        output = render_template("settings.html", form=form, base_url=base_url)
+
         return output
 
     @app.route("/import", methods=['GET', "POST"])
@@ -549,7 +593,7 @@ def changedetection_app(config=None, datastore_o=None):
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
-        extra_stylesheets = ['/static/styles/diff.css']
+        extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
         try:
             watch = datastore.data['watching'][uuid]
         except KeyError:
@@ -592,7 +636,9 @@ def changedetection_app(config=None, datastore_o=None):
                                  uuid=uuid,
                                  newest_version_timestamp=dates[0],
                                  current_previous_version=str(previous_version),
-                                 current_diff_url=watch['url'])
+                                 current_diff_url=watch['url'],
+                                 extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
+                                 left_sticky= True )
 
         return output
 
@@ -604,7 +650,7 @@ def changedetection_app(config=None, datastore_o=None):
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
-        extra_stylesheets = ['/static/styles/diff.css']
+        extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
 
         try:
             watch = datastore.data['watching'][uuid]
@@ -687,30 +733,34 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/static/<string:group>/<string:filename>", methods=['GET'])
     def static_content(group, filename):
         # These files should be in our subdirectory
-        full_path = os.path.realpath(__file__)
-        p = os.path.dirname(full_path)
-
         try:
-            return send_from_directory("{}/static/{}".format(p, group), filename=filename)
+            return send_from_directory("static/{}".format(group), filename=filename)
         except FileNotFoundError:
             abort(404)
 
     @app.route("/api/add", methods=['POST'])
     @login_required
     def api_watch_add():
+        from changedetectionio import forms
+        form = forms.quickWatchForm(request.form)
 
-        url = request.form.get('url').strip()
-        if datastore.url_exists(url):
-            flash('The URL {} already exists'.format(url), "error")
+        if form.validate():
+
+            url = request.form.get('url').strip()
+            if datastore.url_exists(url):
+                flash('The URL {} already exists'.format(url), "error")
+                return redirect(url_for('index'))
+
+            # @todo add_watch should throw a custom Exception for validation etc
+            new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
+            # Straight into the queue.
+            update_q.put(new_uuid)
+
+            flash("Watch added.")
+            return redirect(url_for('index'))
+        else:
+            flash("Error")
             return redirect(url_for('index'))
-
-        # @todo add_watch should throw a custom Exception for validation etc
-        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
-        # Straight into the queue.
-        update_q.put(new_uuid)
-
-        flash("Watch added.")
-        return redirect(url_for('index'))
 
     @app.route("/api/delete", methods=['GET'])
     @login_required
@@ -722,6 +772,16 @@ def changedetection_app(config=None, datastore_o=None):
 
         return redirect(url_for('index'))
 
+    @app.route("/api/clone", methods=['GET'])
+    @login_required
+    def api_clone():
+
+        uuid = request.args.get('uuid')
+        datastore.clone(uuid)
+        flash('Cloned.')
+
+        return redirect(url_for('index'))
+
     @app.route("/api/checknow", methods=['GET'])
     @login_required
     def api_watch_checknow():
@@ -779,8 +839,10 @@ def check_for_new_version():
     while not app.config.exit.is_set():
         try:
             r = requests.post("https://changedetection.io/check-ver.php",
-                              data={'version': datastore.data['version_tag'],
-                                    'app_guid': datastore.data['app_guid']},
+                              data={'version': __version__,
+                                    'app_guid': datastore.data['app_guid'],
+                                    'watch_count': len(datastore.data['watching'])
+                                    },
 
                               verify=False)
         except:
@@ -796,44 +858,27 @@ def check_for_new_version():
         app.config.exit.wait(86400)
 
 def notification_runner():
-
     while not app.config.exit.is_set():
         try:
             # At the moment only one thread runs (single runner)
             n_object = notification_q.get(block=False)
         except queue.Empty:
             time.sleep(1)
-            pass
 
         else:
-            import apprise
-
-            # Create an Apprise instance
+            # Process notifications
             try:
-                apobj = apprise.Apprise()
-                for url in n_object['notification_urls']:
-                    apobj.add(url.strip())
-
-                n_body = n_object['watch_url']
-
-                # 65 - Append URL of instance to the notification if it is set.
-                base_url = os.getenv('BASE_URL')
-                if base_url != None:
-                    n_body += "\n" + base_url
-
-                apobj.notify(
-                    body=n_body,
-                    # @todo This should be configurable.
-                    title="ChangeDetection.io Notification - {}".format(n_object['watch_url'])
-                )
+                from changedetectionio import notification
+                notification.process_notification(n_object, datastore)
 
             except Exception as e:
-                print("Watch URL: {}  Error {}".format(n_object['watch_url'],e))
+                print("Watch URL: {}  Error {}".format(n_object['watch_url'], e))
+
 
 
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
-    from backend import update_worker
+    from changedetectionio import update_worker
 
     # Spin up Workers.
     for _ in range(datastore.data['settings']['requests']['workers']):
@@ -853,7 +898,7 @@ def ticker_thread_check_time_launch_checks():
         for uuid, watch in datastore.data['watching'].items():
 
             # If they supplied an individual entry minutes to threshold.
-            if 'minutes_between_check' in watch:
+            if 'minutes_between_check' in watch and watch['minutes_between_check'] is not None:
                 max_time = watch['minutes_between_check'] * 60
             else:
                 # Default system wide.

changedetectionio/content_fetcher.py

@@ -0,0 +1,131 @@
+import os
+import time
+from abc import ABC, abstractmethod
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from selenium.common.exceptions import WebDriverException
+import urllib3.exceptions
+
+
+class EmptyReply(Exception):
+    pass
+
+class Fetcher():
+    error = None
+    status_code = None
+    content = None # Should be bytes?
+
+    fetcher_description ="No description"
+
+    @abstractmethod
+    def get_error(self):
+        return self.error
+
+    @abstractmethod
+    def run(self, url, timeout, request_headers):
+        # Should set self.error, self.status_code and self.content
+        pass
+
+    @abstractmethod
+    def get_last_status_code(self):
+        return self.status_code
+
+    @abstractmethod
+    # Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
+    def is_ready(self):
+        return True
+
+#   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
+#   the current one would return javascript output (as we use JS to generate the diff)
+#
+#   Returns tuple(mime_type, stream)
+#    @abstractmethod
+#    def return_diff(self, stream_a, stream_b):
+#        return
+
+def available_fetchers():
+        import inspect
+        from changedetectionio import content_fetcher
+        p=[]
+        for name, obj in inspect.getmembers(content_fetcher):
+            if inspect.isclass(obj):
+                # @todo html_ is maybe better as fetcher_ or something
+                # In this case, make sure to edit the default one in store.py and fetch_site_status.py
+                if "html_" in name:
+                    t=tuple([name,obj.fetcher_description])
+                    p.append(t)
+
+        return p
+
+class html_webdriver(Fetcher):
+    if os.getenv("WEBDRIVER_URL"):
+        fetcher_description = "WebDriver Chrome/Javascript via '{}'".format(os.getenv("WEBDRIVER_URL"))
+    else:
+        fetcher_description = "WebDriver Chrome/Javascript"
+
+    command_executor = ''
+
+    def __init__(self):
+        self.command_executor = os.getenv("WEBDRIVER_URL", 'http://browser-chrome:4444/wd/hub')
+
+    def run(self, url, timeout, request_headers):
+
+        # check env for WEBDRIVER_URL
+        driver = webdriver.Remote(
+            command_executor=self.command_executor,
+            desired_capabilities=DesiredCapabilities.CHROME)
+
+        try:
+            driver.get(url)
+        except WebDriverException as e:
+            # Be sure we close the session window
+            driver.quit()
+            raise
+
+        # @todo - how to check this? is it possible?
+        self.status_code = 200
+
+        # @todo - dom wait loaded?
+        time.sleep(5)
+        self.content = driver.page_source
+
+        driver.quit()
+
+
+    def is_ready(self):
+        from selenium import webdriver
+        from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+        from selenium.common.exceptions import WebDriverException
+
+        driver = webdriver.Remote(
+            command_executor=self.command_executor,
+            desired_capabilities=DesiredCapabilities.CHROME)
+
+        # driver.quit() seems to cause better exceptions
+        driver.quit()
+
+
+        return True
+
+# "html_requests" is listed as the default fetcher in store.py!
+class html_requests(Fetcher):
+    fetcher_description = "Basic fast Plaintext/HTTP Client"
+
+    def run(self, url, timeout, request_headers):
+        import requests
+
+        r = requests.get(url,
+                         headers=request_headers,
+                         timeout=timeout,
+                         verify=False)
+
+        html = r.text
+
+
+        # @todo test this
+        if not r or not html or not len(html):
+            raise EmptyReply(url)
+
+        self.status_code = r.status_code
+        self.content = html
+

changedetectionio/fetch_site_status.py

@@ -0,0 +1,178 @@
+import time
+from changedetectionio import content_fetcher
+import hashlib
+from inscriptis import get_text
+import urllib3
+from . import html_tools
+import re
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+# Some common stuff here that can be moved to a base class
+class perform_site_check():
+
+    def __init__(self, *args, datastore, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.datastore = datastore
+
+    def strip_ignore_text(self, content, list_ignore_text):
+        import re
+        ignore = []
+        ignore_regex = []
+        for k in list_ignore_text:
+
+            # Is it a regex?
+            if k[0] == '/':
+                ignore_regex.append(k.strip(" /"))
+            else:
+                ignore.append(k)
+
+        output = []
+        for line in content.splitlines():
+
+            # Always ignore blank lines in this mode. (when this function gets called)
+            if len(line.strip()):
+                regex_matches = False
+
+                # if any of these match, skip
+                for regex in ignore_regex:
+                    try:
+                        if re.search(regex, line, re.IGNORECASE):
+                            regex_matches = True
+                    except Exception as e:
+                        continue
+
+                if not regex_matches and not any(skip_text in line for skip_text in ignore):
+                    output.append(line.encode('utf8'))
+
+        return "\n".encode('utf8').join(output)
+
+
+
+    def run(self, uuid):
+        timestamp = int(time.time())  # used for storage etc too
+
+        changed_detected = False
+        stripped_text_from_html = ""
+
+        watch = self.datastore.data['watching'][uuid]
+
+        update_obj = {'previous_md5': self.datastore.data['watching'][uuid]['previous_md5'],
+                      'history': {},
+                      "last_checked": timestamp
+                      }
+
+        extra_headers = self.datastore.get_val(uuid, 'headers')
+
+        # Tweak the base config with the per-watch ones
+        request_headers = self.datastore.data['settings']['headers'].copy()
+        request_headers.update(extra_headers)
+
+        # https://github.com/psf/requests/issues/4525
+        # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot
+        # do this by accident.
+        if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
+            request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
+
+        # @todo check the failures are really handled how we expect
+
+        else:
+            timeout = self.datastore.data['settings']['requests']['timeout']
+            url = self.datastore.get_val(uuid, 'url')
+
+            # Pluggable content fetcher
+            prefer_backend = watch['fetch_backend']
+            if hasattr(content_fetcher, prefer_backend):
+                klass = getattr(content_fetcher, prefer_backend)
+            else:
+                # If the klass doesnt exist, just use a default
+                klass = getattr(content_fetcher, "html_requests")
+
+
+            fetcher = klass()
+            fetcher.run(url, timeout, request_headers)
+            # Fetching complete, now filters
+            # @todo move to class / maybe inside of fetcher abstract base?
+
+            # @note: I feel like the following should be in a more obvious chain system
+            #  - Check filter text
+            #  - Is the checksum different?
+            #  - Do we convert to JSON?
+            # https://stackoverflow.com/questions/41817578/basic-method-chaining ?
+            # return content().textfilter().jsonextract().checksumcompare() ?
+
+            is_html = True
+            css_filter_rule = watch['css_filter']
+            if css_filter_rule and len(css_filter_rule.strip()):
+                if 'json:' in css_filter_rule:
+                    stripped_text_from_html = html_tools.extract_json_as_string(content=fetcher.content, jsonpath_filter=css_filter_rule)
+                    is_html = False
+                else:
+                    # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+                    stripped_text_from_html = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
+
+            if is_html:
+                # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+                html_content = fetcher.content
+                if css_filter_rule and len(css_filter_rule.strip()):
+                    html_content = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
+
+                # get_text() via inscriptis
+                stripped_text_from_html = get_text(html_content)
+
+            # We rely on the actual text in the html output.. many sites have random script vars etc,
+            # in the future we'll implement other mechanisms.
+
+            update_obj["last_check_status"] = fetcher.get_last_status_code()
+            update_obj["last_error"] = False
+
+
+            # If there's text to skip
+            # @todo we could abstract out the get_text() to handle this cleaner
+            if len(watch['ignore_text']):
+                stripped_text_from_html = self.strip_ignore_text(stripped_text_from_html, watch['ignore_text'])
+            else:
+                stripped_text_from_html = stripped_text_from_html.encode('utf8')
+
+
+            fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest()
+
+            blocked_by_not_found_trigger_text = False
+
+            if len(watch['trigger_text']):
+                blocked_by_not_found_trigger_text = True
+                for line in watch['trigger_text']:
+                    # Because JSON wont serialize a re.compile object
+                    if line[0] == '/' and line[-1] == '/':
+                        regex = re.compile(line.strip('/'), re.IGNORECASE)
+                        # Found it? so we don't wait for it anymore
+                        r = re.search(regex, str(stripped_text_from_html))
+                        if r:
+                            blocked_by_not_found_trigger_text = False
+                            break
+
+                    elif line.lower() in str(stripped_text_from_html).lower():
+                        # We found it don't wait for it.
+                        blocked_by_not_found_trigger_text = False
+                        break
+
+
+            # could be None or False depending on JSON type
+            # On the first run of a site, watch['previous_md5'] will be an empty string
+            if not blocked_by_not_found_trigger_text and watch['previous_md5'] != fetched_md5:
+                changed_detected = True
+
+                # Don't confuse people by updating as last-changed, when it actually just changed from None..
+                if self.datastore.get_val(uuid, 'previous_md5'):
+                    update_obj["last_changed"] = timestamp
+
+                update_obj["previous_md5"] = fetched_md5
+
+            # Extract title as title
+            if is_html and self.datastore.data['settings']['application']['extract_title_as_title']:
+                if not watch['title'] or not len(watch['title']):
+                    update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
+
+
+        return changed_detected, update_obj, stripped_text_from_html

changedetectionio/forms.py

@@ -0,0 +1,228 @@
+from wtforms import Form, SelectField, RadioField, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \
+    Field
+from wtforms import widgets
+from wtforms.validators import ValidationError
+from wtforms.fields import html5
+from changedetectionio import content_fetcher
+import re
+
+class StringListField(StringField):
+    widget = widgets.TextArea()
+
+    def _value(self):
+        if self.data:
+            return "\r\n".join(self.data)
+        else:
+            return u''
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            # Remove empty strings
+            cleaned = list(filter(None, valuelist[0].split("\n")))
+            self.data = [x.strip() for x in cleaned]
+            p = 1
+        else:
+            self.data = []
+
+
+
+class SaltyPasswordField(StringField):
+    widget = widgets.PasswordInput()
+    encrypted_password = ""
+
+    def build_password(self, password):
+        import hashlib
+        import base64
+        import secrets
+
+        # Make a new salt on every new password and store it with the password
+        salt = secrets.token_bytes(32)
+
+        key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
+        store = base64.b64encode(salt + key).decode('ascii')
+
+        return store
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            # Be really sure it's non-zero in length
+            if len(valuelist[0].strip()) > 0:
+                self.encrypted_password = self.build_password(valuelist[0])
+                self.data = ""
+        else:
+            self.data = False
+
+
+# Separated by  key:value
+class StringDictKeyValue(StringField):
+    widget = widgets.TextArea()
+
+    def _value(self):
+        if self.data:
+            output = u''
+            for k in self.data.keys():
+                output += "{}: {}\r\n".format(k, self.data[k])
+
+            return output
+        else:
+            return u''
+
+    # incoming
+    def process_formdata(self, valuelist):
+        if valuelist:
+            self.data = {}
+            # Remove empty strings
+            cleaned = list(filter(None, valuelist[0].split("\n")))
+            for s in cleaned:
+                parts = s.strip().split(':', 1)
+                if len(parts) == 2:
+                    self.data.update({parts[0].strip(): parts[1].strip()})
+
+        else:
+            self.data = {}
+
+class ValidateContentFetcherIsReady(object):
+    """
+    Validates that anything that looks like a regex passes as a regex
+    """
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        from changedetectionio import content_fetcher
+        import urllib3.exceptions
+
+        # Better would be a radiohandler that keeps a reference to each class
+        if field.data is not None:
+            klass = getattr(content_fetcher, field.data)
+            some_object = klass()
+            try:
+                ready = some_object.is_ready()
+
+            except urllib3.exceptions.MaxRetryError as e:
+                driver_url = some_object.command_executor
+                message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data))
+                message += '<br/>'+field.gettext('Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.')
+                message += '<br/>' + field.gettext('Did you follow the instructions in the wiki?')
+                message += '<br/><br/>' + field.gettext('WebDriver Host: %s' % (driver_url))
+                message += '<br/><a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">Go here for more information</a>'
+
+                raise ValidationError(message)
+
+            except Exception as e:
+                message = field.gettext('Content fetcher \'%s\' did not respond properly, unable to use it.\n %s')
+                raise ValidationError(message % (field.data, e))
+
+
+class ValidateAppRiseServers(object):
+    """
+       Validates that each URL given is compatible with AppRise
+       """
+
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        import apprise
+        apobj = apprise.Apprise()
+
+        for server_url in field.data:
+            if not apobj.add(server_url):
+                message = field.gettext('\'%s\' is not a valid AppRise URL.' % (server_url))
+                raise ValidationError(message)
+
+class ValidateTokensList(object):
+    """
+    Validates that a {token} is from a valid set
+    """
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        from changedetectionio import notification
+        regex = re.compile('{.*?}')
+        for p in re.findall(regex, field.data):
+            if not p.strip('{}') in notification.valid_tokens:
+                message = field.gettext('Token \'%s\' is not a valid token.')
+                raise ValidationError(message % (p))
+
+class ValidateListRegex(object):
+    """
+    Validates that anything that looks like a regex passes as a regex
+    """
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+
+        for line in field.data:
+            if line[0] == '/' and line[-1] == '/':
+                # Because internally we dont wrap in /
+                line = line.strip('/')
+                try:
+                    re.compile(line)
+                except re.error:
+                    message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
+                    raise ValidationError(message % (line))
+
+class ValidateCSSJSONInput(object):
+    """
+    Filter validation
+    @todo CSS validator ;)
+    """
+
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        if 'json:' in field.data:
+            from jsonpath_ng.exceptions import JsonPathParserError
+            from jsonpath_ng import jsonpath, parse
+
+            input = field.data.replace('json:', '')
+
+            try:
+                parse(input)
+            except JsonPathParserError as e:
+                message = field.gettext('\'%s\' is not a valid JSONPath expression. (%s)')
+                raise ValidationError(message % (input, str(e)))
+
+
+class quickWatchForm(Form):
+    # https://wtforms.readthedocs.io/en/2.3.x/fields/#module-wtforms.fields.html5
+    # `require_tld` = False is needed even for the test harness "http://localhost:5005.." to run
+    url = html5.URLField('URL', [validators.URL(require_tld=False)])
+    tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)])
+
+class commonSettingsForm(Form):
+
+    notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
+    notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {watch_url}', validators=[validators.Optional(), ValidateTokensList()])
+    notification_body = TextAreaField('Notification Body', default='{watch_url} had a change.', validators=[validators.Optional(), ValidateTokensList()])
+    trigger_check = BooleanField('Send test notification on save')
+    fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+
+class watchForm(commonSettingsForm):
+
+    url = html5.URLField('URL', [validators.URL(require_tld=False)])
+    tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)])
+
+    minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
+                                               [validators.Optional(), validators.NumberRange(min=1)])
+    css_filter = StringField('CSS/JSON Filter', [ValidateCSSJSONInput()])
+    title = StringField('Title')
+
+    ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
+    headers = StringDictKeyValue('Request Headers')
+    trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
+
+
+class globalSettingsForm(commonSettingsForm):
+
+    password = SaltyPasswordField()
+
+    minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
+                                               [validators.NumberRange(min=1)])
+    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title')

changedetectionio/html_tools.py

@@ -0,0 +1,90 @@
+import json
+from bs4 import BeautifulSoup
+from jsonpath_ng import parse
+
+
+class JSONNotFound(ValueError):
+    def __init__(self, msg):
+        ValueError.__init__(self, msg)
+
+# Given a CSS Rule, and a blob of HTML, return the blob of HTML that matches
+def css_filter(css_filter, html_content):
+    soup = BeautifulSoup(html_content, "html.parser")
+    html_block = ""
+    for item in soup.select(css_filter, separator=""):
+        html_block += str(item)
+
+    return html_block + "\n"
+
+
+# Extract/find element
+def extract_element(find='title', html_content=''):
+
+    #Re #106, be sure to handle when its not found
+    element_text = None
+
+    soup = BeautifulSoup(html_content, 'html.parser')
+    result = soup.find(find)
+    if result and result.string:
+        element_text = result.string.strip()
+
+    return element_text
+
+#
+def _parse_json(json_data, jsonpath_filter):
+    s=[]
+    jsonpath_expression = parse(jsonpath_filter.replace('json:', ''))
+    match = jsonpath_expression.find(json_data)
+
+    # More than one result, we will return it as a JSON list.
+    if len(match) > 1:
+        for i in match:
+            s.append(i.value)
+
+    # Single value, use just the value, as it could be later used in a token in notifications.
+    if len(match) == 1:
+        s = match[0].value
+
+    if not s:
+        raise JSONNotFound("No Matching JSON could be found for the rule {}".format(jsonpath_filter.replace('json:', '')))
+
+    stripped_text_from_html = json.dumps(s, indent=4)
+
+    return stripped_text_from_html
+
+def extract_json_as_string(content, jsonpath_filter):
+
+    stripped_text_from_html = False
+
+    # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded <script type=ldjson>
+    try:
+        stripped_text_from_html = _parse_json(json.loads(content), jsonpath_filter)
+    except json.JSONDecodeError:
+
+        # Foreach <script json></script> blob.. just return the first that matches jsonpath_filter
+        s = []
+        soup = BeautifulSoup(content, 'html.parser')
+        bs_result = soup.findAll('script')
+
+        if not bs_result:
+            raise JSONNotFound("No parsable JSON found in this document")
+
+        for result in bs_result:
+            # Skip empty tags, and things that dont even look like JSON
+            if not result.string or not '{' in result.string:
+                continue
+                
+            try:
+                json_data = json.loads(result.string)
+            except json.JSONDecodeError:
+                # Just skip it
+                continue
+            else:
+                stripped_text_from_html = _parse_json(json_data, jsonpath_filter)
+                if stripped_text_from_html:
+                    break
+
+    if not stripped_text_from_html:
+        raise JSONNotFound("No JSON matching the rule '%s' found" % jsonpath_filter.replace('json:',''))
+
+    return stripped_text_from_html

changedetectionio/notification.py

@@ -0,0 +1,87 @@
+import os
+import apprise
+
+valid_tokens = {
+    'base_url': '',
+    'watch_url': '',
+    'watch_uuid': '',
+    'watch_title': '',
+    'watch_tag': '',
+    'diff_url': '',
+    'preview_url': '',
+    'current_snapshot': ''
+}
+
+
+def process_notification(n_object, datastore):
+    import logging
+    log = logging.getLogger('apprise')
+    log.setLevel('TRACE')
+    apobj = apprise.Apprise(debug=True)
+
+    for url in n_object['notification_urls']:
+        url = url.strip()
+        print (">> Process Notification: AppRise notifying {}".format(url))
+        apobj.add(url)
+
+    # Get the notification body from datastore
+    n_body = n_object['notification_body']
+    n_title = n_object['notification_title']
+
+    # Insert variables into the notification content
+    notification_parameters = create_notification_parameters(n_object, datastore)
+
+    for n_k in notification_parameters:
+        token = '{' + n_k + '}'
+        val = notification_parameters[n_k]
+        n_title = n_title.replace(token, val)
+        n_body = n_body.replace(token, val)
+
+    apobj.notify(
+        body=n_body,
+        title=n_title
+    )
+
+# Notification title + body content parameters get created here.
+def create_notification_parameters(n_object, datastore):
+    from copy import deepcopy
+
+    # in the case we send a test notification from the main settings, there is no UUID.
+    uuid = n_object['uuid'] if 'uuid' in n_object else ''
+
+    if uuid != '':
+        watch_title = datastore.data['watching'][uuid]['title']
+        watch_tag = datastore.data['watching'][uuid]['tag']
+    else:
+        watch_title = 'Change Detection'
+        watch_tag = ''
+
+    # Create URLs to customise the notification with
+    base_url = os.getenv('BASE_URL', '').strip('"')
+    watch_url = n_object['watch_url']
+
+    # Re #148 - Some people have just {base_url} in the body or title, but this may break some notification services
+    #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.
+    if base_url == '':
+        base_url = "<base-url-env-var-not-set>"
+
+    diff_url = "{}/diff/{}".format(base_url, uuid)
+    preview_url = "{}/preview/{}".format(base_url, uuid)
+
+    # Not sure deepcopy is needed here, but why not
+    tokens = deepcopy(valid_tokens)
+
+    # Valid_tokens also used as a field validator
+    tokens.update(
+    {
+        'base_url': base_url if base_url is not None else '',
+        'watch_url': watch_url,
+        'watch_uuid': uuid,
+        'watch_title': watch_title if watch_title is not None else '',
+        'watch_tag': watch_tag if watch_tag is not None else '',
+        'diff_url': diff_url,
+        'preview_url': preview_url,
+        'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
+    })
+
+    return tokens

changedetectionio/static/js/settings.js

@@ -0,0 +1,13 @@
+window.addEventListener("load", (event) => {
+  // just an example for now
+  function toggleVisible(elem) {
+    // theres better ways todo this
+    var x = document.getElementById(elem);
+    if (x.style.display === "block") {
+      x.style.display = "none";
+    } else {
+      x.style.display = "block";
+    }
+  }
+});
+

changedetectionio/static/js/tabs.js

@@ -0,0 +1,51 @@
+// Rewrite this is a plugin.. is all this JS really 'worth it?'
+
+
+window.addEventListener('hashchange', function() {
+  var tabs = document.getElementsByClassName('active');
+  while (tabs[0]) {
+    tabs[0].classList.remove('active')
+  }
+  set_active_tab();
+}, false);
+
+var has_errors=document.querySelectorAll(".messages .error");
+if (!has_errors.length) {
+    if (document.location.hash == "" ) {
+        document.location.hash = "#general";
+        document.getElementById("default-tab").className = "active";
+    } else {
+        set_active_tab();
+    }
+} else {
+  focus_error_tab();
+}
+
+
+function set_active_tab() {
+  var tab=document.querySelectorAll("a[href='"+location.hash+"']");
+  if (tab.length) {
+    tab[0].parentElement.className="active";
+  }
+    // hash could move the page down
+    window.scrollTo(0, 0);
+}
+
+function focus_error_tab() {
+  // time to use jquery or vuejs really,
+  // activate the tab with the error
+    var tabs = document.querySelectorAll('.tabs li a'),i;
+    for (i = 0; i < tabs.length; ++i) {
+      var tab_name=tabs[i].hash.replace('#','');
+      var pane_errors=document.querySelectorAll('#'+tab_name+' .error')
+      if (pane_errors.length) {
+        document.location.hash = '#'+tab_name;
+        return true;
+      }
+    }
+    return false;
+}
+
+
+
+

changedetectionio/static/styles/diff.css

@@ -12,7 +12,8 @@
     border: 1px solid transparent;
     vertical-align: top;
     font: 1em monospace;
-    text-align: left;
+    text-align: left; }
+  #diff-ui pre {
     white-space: pre-wrap; }
 
 h1 {

changedetectionio/static/styles/diff.scss

@@ -16,9 +16,10 @@
         vertical-align: top;
         font: 1em monospace;
         text-align: left;
-        white-space: pre-wrap;
     }
-
+    pre {
+            white-space: pre-wrap;
+    }
 }
 h1 {
 	display: inline;
@@ -64,4 +65,4 @@ ins {
 	body {
 		height: 99%; /* Hide scroll bar in Firefox */
 	}
-}
+}

changedetectionio/static/styles/package.json

@@ -10,6 +10,8 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "node-sass": "^6.0.0"
+    "node-sass": "^6.0.1",
+    "tar": "^6.1.9",
+    "trim-newlines": "^3.0.1"
   }
 }

changedetectionio/static/styles/styles.scss

@@ -7,7 +7,6 @@ body {
   color: #333;
   background: #262626;
 }
-
 .pure-table-even {
   background: #fff;
 }
@@ -72,7 +71,6 @@ section.content {
   }
 }
 
-
 .watch-tag-list {
   color: #e70069;
   white-space: nowrap;
@@ -137,12 +135,33 @@ body:after, body:before {
   clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
 }
 
+.arrow {
+  border: solid black;
+  border-width: 0 3px 3px 0;
+  display: inline-block;
+  padding: 3px;
+    &.right {
+      transform: rotate(-45deg);
+      -webkit-transform: rotate(-45deg);
+    }
+    &.left {
+      transform: rotate(135deg);
+      -webkit-transform: rotate(135deg);
+    }
+    &.up {
+      transform: rotate(-135deg);
+      -webkit-transform: rotate(-135deg);
+    }
+    &.down {
+      transform: rotate(45deg);
+      -webkit-transform: rotate(45deg);
+    }
+}
 
 .button-small {
   font-size: 85%;
 }
 
-
 .fetch-error {
   padding-top: 1em;
   font-size: 60%;
@@ -150,12 +169,6 @@ body:after, body:before {
   display: block;
 }
 
-.edit-form {
-  background: #fff;
-  padding: 2em;
-  margin: 1em;
-  border-radius: 5px;
-}
 
 .button-secondary {
   color: white;
@@ -220,7 +233,19 @@ body:after, body:before {
             background: rgba(255, 255, 255, .5);
         }
     }
+}
 
+#notification-customisation {
+    border: 1px solid #ccc;
+    padding: 1rem;
+    border-radius: 5px;
+}
+
+
+#token-table {
+    &.pure-table td, &.pure-table th {
+        font-size: 80%;
+    }
 }
 
 #new-watch-form {
@@ -228,15 +253,20 @@ body:after, body:before {
   padding: 1em;
   border-radius: 10px;
   margin-bottom: 1em;
+  input {
+    width: auto !important;
+    display: inline-block;
+  }
+  .label {
+    display: none;
+  }
+  legend {
+    color: #fff;
+  }
 }
 
-#new-watch-form legend {
-  color: #fff;
-}
 
-#new-watch-form input {
-  width: auto !important;
-}
+
 
 #diff-col {
   padding-left: 40px;
@@ -245,22 +275,21 @@ body:after, body:before {
 #diff-jump {
   position: fixed;
   left: 0px;
-  top: 80px;
+  top: 120px;
   background: #fff;
   padding: 10px;
   border-top-right-radius: 5px;
   border-bottom-right-radius: 5px;
   box-shadow: 5px 0 5px -2px #888;
-}
-
-#diff-jump a {
-  color: #1b98f8;
-  cursor: grabbing;
-  -moz-user-select: none;
-  -webkit-user-select: none;
-  -ms-user-select: none;
-  user-select: none;
-  -o-user-select: none;
+     a {
+      color: #1b98f8;
+      cursor: grabbing;
+      -moz-user-select: none;
+      -webkit-user-select: none;
+      -ms-user-select: none;
+      user-select: none;
+      -o-user-select: none;
+    }
 }
 
 footer {
@@ -274,13 +303,31 @@ footer {
   vertical-align: middle;
 }
 
-#version {
+#top-right-menu {
+// Just let flex overflow the x axis for now
+/*
+    position: absolute;
+    right: 0px;
+    background: linear-gradient(to right, #fff0, #fff 10%);
+    padding-left: 20px;
+    padding-right: 10px;
+    */
+}
+
+
+
+.sticky-tab {
   position: absolute;
   top: 80px;
-  right: 0px;
   font-size: 8px;
   background: #fff;
   padding: 10px;
+  &#left-sticky {
+    left: 0px;
+  }
+  &#right-sticky {
+    right: 0px;
+  }
 }
 
 #new-version-text a {
@@ -311,7 +358,7 @@ footer {
 .pure-form {
     .pure-control-group, .pure-group, .pure-controls {
         padding-bottom: 1em;
-        dd {
+        div {
             margin: 0px;
         }
     }
@@ -321,7 +368,8 @@ footer {
         background-color: #ffebeb;
     }
   }
-   /* The list of errors */
+
+  /* The list of errors */
   ul.errors {
     padding: .5em .6em;
     border: 1px solid #dd0000;
@@ -339,13 +387,17 @@ footer {
     font-weight: bold;
   }
 
-  input[type=url] {
-    width: 100%;
-  }
-
   textarea {
     width: 100%;
-    font-size: 14px;
+  }
+  ul#fetch_backend {
+    margin: 0px;
+    list-style: none;
+    > li {
+        > * {
+            display: inline-block;
+        }
+    }
   }
 }
 
@@ -355,12 +407,11 @@ footer {
   }
   .edit-form {
     padding: 0.5em;
-    margin: 0.5em;
+    margin: 0;
   }
   #nav-menu {
     overflow-x: scroll;
   }
-
 }
 
 /*
@@ -368,9 +419,13 @@ Max width before this PARTICULAR table gets nasty
 This query will take effect for any screen smaller than 760px
 and also iPads specifically.
 */
-@media only screen and (max-width: 760px),
-(min-device-width: 768px) and (max-device-width: 1024px) {
 
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+
+  input[type='text'] {
+    width: 100%;
+  }
+    
   .watch-table {
     /* Force table to not be like tables anymore */
     thead, tbody, th, td, tr {
@@ -436,4 +491,76 @@ and also iPads specifically.
 }
 
 
+/** Desktop vs mobile input field strategy
+- We dont use 'size' with <input> because `size` is too unreliable to override, and will often push-out
+- Rely always on width in CSS
+*/
+@media only screen and (min-width: 761px) {
+/* m-d is medium-desktop */
+    .m-d {
+        min-width: 80%;
+    }
+}
+
+
+
+.tabs {
+  ul {
+    margin: 0px;
+    padding: 0px;
+    display:block;
+    li {
+      margin-right: 3px;
+      display: inline-block;
+      color: #fff;
+      border-top-left-radius: 5px;
+      border-top-right-radius: 5px;
+      background-color: rgba(255, 255, 255, 0.2);
+
+      &.active,:target {
+        background-color: #fff;
+        a {
+          color: #222;
+          font-weight: bold;
+        }
+      }
+      a {
+        display: block;
+        padding: 0.8em;
+        color: #fff;
+      }
+    }
+  }
+}
+
+$form-edge-padding: 20px;
+.pure-form-stacked {
+  >div:first-child {
+    display: block;
+  }
+}
+.edit-form {
+  min-width: 70%;
+  .tab-pane-inner {
+    &:not(:target) {
+        display: none;
+    }
+    &:target {
+      display: block;
+    }
+    // doesnt need padding because theres another row of buttons/activity
+    padding: 0px;
+  }
+  .box-wrap {
+    position: relative;
+  }
+  .inner {
+    background: #fff;;
+    padding: $form-edge-padding;
+  }
+  #actions {
+    display: block;
+    background: #fff;
+  }
+}
 

changedetectionio/store.py

@@ -15,7 +15,7 @@ import threading
 class ChangeDetectionStore:
     lock = Lock()
 
-    def __init__(self, datastore_path="/datastore", include_default_watches=True):
+    def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"):
         self.needs_write = False
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
@@ -38,7 +38,12 @@ class ChangeDetectionStore:
                 },
                 'application': {
                     'password': False,
-                    'notification_urls': [] # Apprise URL list
+                    'extract_title_as_title': False,
+                    'fetch_backend': 'html_requests',
+                    'notification_urls': [], # Apprise URL list
+                    # Custom notification content
+                    'notification_title': None,
+                    'notification_body': None,
                 }
             }
         }
@@ -53,18 +58,25 @@ class ChangeDetectionStore:
             'last_viewed': 0,  # history key value of the last viewed via the [diff] link
             'newest_history_key': "",
             'title': None,
-            'minutes_between_check': 3 * 60,  # Default 3 hours
+            # Re #110, so then if this is set to None, we know to use the default value instead
+            # Requires setting to None on submit if it's the same as the default
+            'minutes_between_check': None,
             'previous_md5': "",
             'uuid': str(uuid_builder.uuid4()),
             'headers': {},  # Extra headers to send
             'history': {},  # Dict of timestamp and output stripped filename
             'ignore_text': [], # List of text to ignore when calculating the comparison checksum
+            # Custom notification content
             'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise)
+            'notification_title': None,
+            'notification_body': None,
             'css_filter': "",
+            'trigger_text': [],  # List of text or regex to wait for until a change is detected
+            'fetch_backend': None,
         }
 
-        if path.isfile('backend/source.txt'):
-            with open('backend/source.txt') as f:
+        if path.isfile('changedetectionio/source.txt'):
+            with open('changedetectionio/source.txt') as f:
                 # Should be set in Dockerfile to look for /source.txt , this will give us the git commit #
                 # So when someone gives us a backup file to examine, we know exactly what code they were running.
                 self.__data['build_sha'] = f.read()
@@ -111,7 +123,7 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
                 self.add_watch(url='https://changedetection.io', tag='Tech news')
 
-        self.__data['version_tag'] = "0.35"
+        self.__data['version_tag'] = version_tag
 
         # Helper to remove password protection
         password_reset_lockfile = "{}/removepassword.lock".format(self.datastore_path)
@@ -173,7 +185,6 @@ class ChangeDetectionStore:
 
     @property
     def data(self):
-
         has_unviewed = False
         for uuid, v in self.__data['watching'].items():
             self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
@@ -184,6 +195,14 @@ class ChangeDetectionStore:
                 self.__data['watching'][uuid]['viewed'] = False
                 has_unviewed = True
 
+            # #106 - Be sure this is None on empty string, False, None, etc
+            if not self.__data['watching'][uuid]['title']:
+                self.__data['watching'][uuid]['title'] = None
+
+            # Default var for fetch_backend
+            if not self.__data['watching'][uuid]['fetch_backend']:
+                self.__data['watching'][uuid]['fetch_backend'] = self.__data['settings']['application']['fetch_backend']
+
         self.__data['has_unviewed'] = has_unviewed
 
         return self.__data
@@ -226,6 +245,27 @@ class ChangeDetectionStore:
 
             self.needs_write = True
 
+    # Clone a watch by UUID
+    def clone(self, uuid):
+        with self.lock:
+            new_uuid = str(uuid_builder.uuid4())
+            _clone = deepcopy(self.data['watching'][uuid])
+            _clone.update({'uuid': new_uuid})
+
+            attributes_to_reset = [
+                'last_checked',
+                'last_changed',
+                'last_viewed',
+                'newest_history_key',
+                'previous_md5',
+                'history'
+            ]
+            for attribute in attributes_to_reset:
+                _clone.update({attribute: self.generic_definition[attribute]})
+
+            self.data['watching'][new_uuid] = _clone
+            self.needs_write = True
+
     def url_exists(self, url):
 
         # Probably their should be dict...
@@ -306,18 +346,15 @@ class ChangeDetectionStore:
 
     # Save some text file to the appropriate path and bump the history
     # result_obj from fetch_site_status.run()
-    def save_history_text(self, uuid, result_obj, contents):
+    def save_history_text(self, watch_uuid, contents):
+        import uuid
 
-        output_path = "{}/{}".format(self.datastore_path, uuid)
-        fname = "{}/{}-{}.stripped.txt".format(output_path, result_obj['previous_md5'], str(time.time()))
-        with open(fname, 'w') as f:
+        output_path = "{}/{}".format(self.datastore_path, watch_uuid)
+        fname = "{}/{}.stripped.txt".format(output_path, uuid.uuid4())
+        with open(fname, 'wb') as f:
             f.write(contents)
             f.close()
 
-        # Update history with the stripped text for future reference, this will also mean we save the first
-        # Should always be keyed by string(timestamp)
-        self.update_watch(uuid, {"history": {str(result_obj["last_checked"]): fname}})
-
         return fname
 
     def sync_to_json(self):
@@ -346,7 +383,7 @@ class ChangeDetectionStore:
             if self.stop_thread:
                 print("Shutting down datastore thread")
                 return
-            
+
             if self.needs_write:
                 self.sync_to_json()
             time.sleep(3)

changedetectionio/templates/_common_fields.jinja

@@ -0,0 +1,90 @@
+
+{% from '_helpers.jinja' import render_field %}
+
+{% macro render_notifications_field(form) %}
+
+
+<fieldset>
+                    <div class="field-group">
+                        <div class="pure-control-group">
+                            {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
+    Gitter - gitter://token/room
+    Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+    AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
+    SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com")
+                            }}
+                            <div class="pure-form-message-inline">Use <a target=_new
+                                                                         href="https://github.com/caronc/apprise">AppRise
+                                URLs</a> for notification to just about any service!
+                            </div>
+                        </div>
+                        <div id="notification-customisation">
+                            <div class="pure-control-group">
+                                {{ render_field(form.notification_title, class="m-d") }}
+                                <span class="pure-form-message-inline">Title for all notifications</span>
+                            </div>
+                            <div class="pure-control-group">
+                                {{ render_field(form.notification_body , rows=5) }}
+                                <span class="pure-form-message-inline">Body for all notifications</span>
+                            </div>
+                            <div class="pure-controls">
+                            <span class="pure-form-message-inline">
+                                These tokens can be used in the notification body and title to
+                                customise the notification text.
+                            </span>
+                                <table class="pure-table" id="token-table">
+                                    <thead>
+                                    <tr>
+                                        <th>Token</th>
+                                        <th>Description</th>
+                                    </tr>
+                                    </thead>
+                                    <tbody>
+                                    <tr>
+                                        <td><code>{base_url}</code></td>
+                                        <td>The URL of the changedetection.io instance you are running.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{watch_url}</code></td>
+                                        <td>The URL being watched.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{watch_uuid}</code></td>
+                                        <td>The UUID of the watch.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{watch_title}</code></td>
+                                        <td>The title of the watch.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{watch_tag}</code></td>
+                                        <td>The tag of the watch.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{preview_url}</code></td>
+                                        <td>The URL of the preview page generated by changedetection.io.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{diff_url}</code></td>
+                                        <td>The URL of the diff page generated by changedetection.io.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><code>{current_snapshot}</code></td>
+                                        <td>The current snapshot value, useful when combined with JSON or CSS filters
+                                        </td>
+                                    </tr>
+                                    </tbody>
+                                </table>
+                                <span class="pure-form-message-inline">
+                                URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
+                                Your <code>BASE_URL</code> var is currently "{{base_url}}"
+                            </span>
+                            </div>
+                        </div>
+                        <div class="pure-control-group">
+                            {{ render_field(form.trigger_check) }}
+                        </div>
+                    </div>
+
+                </fieldset>
+{% endmacro %}

changedetectionio/templates/_helpers.jinja

@@ -0,0 +1,27 @@
+{% macro render_field(field) %}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
+  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </div>
+{% endmacro %}
+
+{% macro render_simple_field(field) %}
+  <span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>
+  <span {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </span>
+{% endmacro %}
+
+

changedetectionio/templates/base.html

@@ -4,9 +4,9 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta name="description" content="Self hosted website change detection.">
-    <title>Change Detection</title>
-    <link rel="stylesheet" href="/static/styles/pure-min.css">
-    <link rel="stylesheet" href="/static/styles/styles.css?ver=1000">
+    <title>Change Detection{{extra_title}}</title>
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}">
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}">
     {% if extra_stylesheets %}
         {% for m in extra_stylesheets %}
         <link rel="stylesheet" href="{{ m }}?ver=1000">
@@ -21,31 +21,31 @@
         {% if has_password and not current_user.is_authenticated %}
             <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener"><strong>Change</strong>Detection.io</a>
         {% else %}
-            <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
+            <a class="pure-menu-heading" href="{{url_for('index')}}"><strong>Change</strong>Detection.io</a>
         {% endif %}
         {% if current_diff_url %}
         <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</span></a>
         {% else %}
-        {% if new_version_available %}
+        {% if new_version_available and not (has_password and not current_user.is_authenticated) %}
         <span id="new-version-text" class="pure-menu-heading"><a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a></span>
         {% endif %}
         {% endif %}
 
-        <ul class="pure-menu-list">
+        <ul class="pure-menu-list"  id="top-right-menu">
         {% if current_user.is_authenticated or not has_password %}
             {% if not current_diff_url %}
             <li class="pure-menu-item">
-                <a href="/backup" class="pure-menu-link">BACKUP</a>
+                <a href="{{ url_for('get_backup')}}" class="pure-menu-link">BACKUP</a>
             </li>
             <li class="pure-menu-item">
-                <a href="/import" class="pure-menu-link">IMPORT</a>
+                <a href="{{ url_for('import_page')}}" class="pure-menu-link">IMPORT</a>
             </li>
             <li class="pure-menu-item">
-                <a href="/settings" class="pure-menu-link">SETTINGS</a>
+                <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
             </li>
             {% else %}
             <li class="pure-menu-item">
-                <a href="{{ url_for('edit_page', uuid=uuid) }}" class="pure-menu-link">EDIT</a>
+                <a href="{{ url_for('edit_page', uuid=uuid, next='diff') }}" class="pure-menu-link">EDIT</a>
             </li>
             {% endif %}
         {% else %}
@@ -55,7 +55,7 @@
         {% endif %}
 
         {% if current_user.is_authenticated %}
-            <li class="pure-menu-item"><a href="/logout" class="pure-menu-link">LOG OUT</a></li>
+            <li class="pure-menu-item"><a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a></li>
         {% endif %}
             <li class="pure-menu-item"><a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
                 <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16"
@@ -68,7 +68,9 @@
         </ul>
     </div>
 </div>
-<div id="version">v{{ version }}</div>
+
+{% if left_sticky %}<div class="sticky-tab" id="left-sticky"><a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a></div> {% endif %}
+{% if right_sticky %}<div class="sticky-tab" id="right-sticky">{{ right_sticky }}</div> {% endif %}
 <section class="content">
     <header>
         {% block header %}{% endblock %}

changedetectionio/templates/diff.html

@@ -30,7 +30,6 @@
     </form>
     <del>Removed text</del>
     <ins>Inserted Text</ins>
-    <a href="{{ url_for('preview_page', uuid=uuid) }}">Show current snapshot</a>
 </div>
 
 <div id="diff-jump">
@@ -53,7 +52,9 @@
 
 </div>
 
-<script src="/static/js/diff.js"></script>
+
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff.js')}}"></script>
+
 <script defer="">
 
 
@@ -130,7 +131,8 @@ if ('oninput' in a) {
 
 function onDiffTypeChange(radio) {
 	window.diffType = radio.value;
-	document.title = "Diff " + radio.value.slice(4);
+// Not necessary 
+//	document.title = "Diff " + radio.value.slice(4);
 }
 
 var radio = document.getElementsByName('diff_type');

changedetectionio/templates/edit.html

@@ -0,0 +1,123 @@
+{% extends 'base.html' %}
+{% block content %}
+{% from '_helpers.jinja' import render_field %}
+{% from '_common_fields.jinja' import render_notifications_field %}
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+
+<div class="edit-form monospaced-textarea">
+
+    <div class="tabs">
+        <ul>
+            <li class="tab" id="default-tab"><a href="#general">General</a></li>
+            <li class="tab"><a href="#notifications">Notifications</a></li>
+            <li class="tab"><a href="#filters">Filters</a></li>
+            <li class="tab"><a href="#triggers">Triggers</a></li>
+        </ul>
+    </div>
+
+    <div class="box-wrap inner">
+        <form class="pure-form pure-form-stacked"
+              action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next') ) }}" method="POST">
+
+            <div class="tab-pane-inner" id="general">
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.title, class="m-d") }}
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.tag) }}
+                        <span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.minutes_between_check) }}
+                        {% if using_default_minutes %}
+                        <span class="pure-form-message-inline">Currently using the <a
+                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
+                        {% else %}
+                        <span class="pure-form-message-inline">Set to blank to use the <a
+                                href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
+                        {% endif %}
+                    </div>
+                    <fieldset class="pure-group">
+                        {{ render_field(form.headers, rows=5, placeholder="Example
+Cookie: foobar
+User-Agent: wonderbra 1.0") }}
+                        <span class="pure-form-message-inline">
+                            Note: ONLY used by Basic fast Plaintext/HTTP Client
+                          </span>
+                    </fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.fetch_backend) }}
+                        <span class="pure-form-message-inline">
+                            <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
+                            <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server. </p>
+                        </span>
+                    </div>
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="notifications">
+                <strong>Note: <i>These settings override the global settings.</i></strong>
+                {{ render_notifications_field(form) }}
+            </div>
+
+            <div class="tab-pane-inner" id="filters">
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.",
+                        class="m-d") }}
+                        <span class="pure-form-message-inline">
+                    <ul>
+                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
+                        <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <b>"json:"</b>, <a
+                                href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
+                    </ul>
+                    Please be sure that you thoroughly understand how to write CSS or JSONPath selector rules before filing an issue on GitHub! <a
+                                href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
+                </span>
+                    </div>
+
+                </fieldset>
+                <fieldset class="pure-group">
+                    {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
+/some.regex\d{2}/ for case-INsensitive regex
+                    ") }}
+                    <span class="pure-form-message-inline">
+                    Each line processed separately, any line matching will be ignored.<br/>
+                    Regular Expression support, wrap the line in forward slash <b>/regex/</b>.
+                </span>
+
+            </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="triggers">
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line
+/some.regex\d{2}/ for case-INsensitive regex
+                    ") }}</br>
+                        <span class="pure-form-message-inline">Text to wait for before triggering a change/notification, all text and regex are tested <i>case-insensitive</i>.</span><br/>
+                        <span class="pure-form-message-inline">Trigger text is processed from the result-text that comes out of any <a href="#filters">CSS/JSON Filters</a> for this watch</span>.<br/>
+                        <span class="pure-form-message-inline">Each line is process separately (think of each line as "OR")</span><br/>
+                        <span class="pure-form-message-inline">Note: Wrap in forward slash / to use regex  example: <span style="font-family: monospace; background: #eee">/foo\d/</span> </span>
+                    </div>
+                </fieldset>
+            </div>
+            <div id="actions">
+                <div class="pure-control-group">
+
+                    <button type="submit" class="pure-button pure-button-primary">Save</button>
+                    <a href="{{url_for('api_delete', uuid=uuid)}}"
+                       class="pure-button button-small button-error ">Delete</a>
+                    <a href="{{url_for('api_clone', uuid=uuid)}}"
+                       class="pure-button button-small ">Create Copy</a>
+                </div>
+            </div>
+        </form>
+    </div>
+</div>
+
+{% endblock %}

changedetectionio/templates/import.html

@@ -0,0 +1,23 @@
+{% extends 'base.html' %}
+
+{% block content %}
+<div class="edit-form">
+     <div class="inner">
+        <form class="pure-form pure-form-aligned" action="{{url_for('import_page')}}" method="POST">
+            <fieldset class="pure-group">
+                <legend>One URL per line, URLs that do not pass validation will stay in the textarea.</legend>
+
+                <textarea name="urls" class="pure-input-1-2" placeholder="https://"
+                          style="width: 100%;
+                                font-family:monospace;
+                                white-space: pre;
+                                overflow-wrap: normal;
+                                overflow-x: scroll;" rows="25">{{ remaining }}</textarea>
+            </fieldset>
+            <button type="submit" class="pure-button pure-input-1-2 pure-button-primary">Import</button>
+        </form>
+     </div>
+</div>
+
+{% endblock %}
+

changedetectionio/templates/login.html

@@ -2,7 +2,9 @@
 
 {% block content %}
 <div class="edit-form">
-    <form class="pure-form pure-form-stacked" action="/login" method="POST">
+
+ <div class="inner">
+    <form class="pure-form pure-form-stacked" action="{{url_for('login')}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
                 <label for="password">Password</label>
@@ -15,6 +17,7 @@
             </div>
         </fieldset>
     </form>
-</div>
+  </div>
+ </div>
 
 {% endblock %}

changedetectionio/templates/scrub.html

@@ -2,7 +2,7 @@
 
 {% block content %}
 <div class="edit-form">
-    <form class="pure-form pure-form-stacked" action="/scrub" method="POST">
+    <form class="pure-form pure-form-stacked" action="{{url_for('scrub_page')}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
                 This will remove all version snapshots/data, but keep your list of URLs. <br/>
@@ -26,7 +26,7 @@
             </div>
             <br/>
             <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Cancel</a>
+                <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Cancel</a>
             </div>
         </fieldset>
     </form>

changedetectionio/templates/settings.html

@@ -0,0 +1,68 @@
+{% extends 'base.html' %}
+
+{% block content %}
+{% from '_helpers.jinja' import render_field %}
+{% from '_common_fields.jinja' import render_notifications_field %}
+
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='settings.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+
+<div class="edit-form">
+    <div class="tabs">
+        <ul>
+            <li class="tab" id="default-tab"><a href="#general">General</a></li>
+            <li class="tab"><a href="#notifications">Notifications</a></li>
+            <li class="tab"><a href="#fetching">Fetching</a></li>
+        </ul>
+    </div>
+    <div class="box-wrap inner">
+        <form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
+            <div class="tab-pane-inner" id="general">
+                <fieldset>
+                    <div class="pure-control-group">
+                        {{ render_field(form.minutes_between_check) }}
+                        <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
+                    </div>
+                    <div class="pure-control-group">
+                        {% if current_user.is_authenticated %}
+                        <a href="{{url_for('settings_page', removepassword='yes')}}"
+                           class="pure-button pure-button-primary">Remove password</a>
+                        {% else %}
+                        {{ render_field(form.password) }}
+                        <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
+                        {% endif %}
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.extract_title_as_title) }}
+                        <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
+                    </div>
+                </fieldset>
+            </div>
+
+            <div class="tab-pane-inner" id="notifications">
+                {{ render_notifications_field(form) }}
+            </div>
+
+            <div class="tab-pane-inner" id="fetching">
+                <div class="pure-control-group">
+                    {{ render_field(form.fetch_backend) }}
+                    <span class="pure-form-message-inline">
+                        <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
+                        <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server. </p>
+                    </span>
+                </div>
+            </div>
+            <div id="actions">
+                <div class="pure-control-group">
+                    <button type="submit" class="pure-button pure-button-primary">Save</button>
+                                           <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
+                        <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete
+                            History
+                            Snapshot Data</a>
+                </div>
+            </div>
+        </form>
+    </div>
+</div>
+
+{% endblock %}

changedetectionio/templates/watch-overview.html

@@ -1,24 +1,24 @@
 {% extends 'base.html' %}
-
 {% block content %}
+{% from '_helpers.jinja' import render_simple_field %}
 
 <div class="box">
 
-    <form class="pure-form" action="/api/add" method="POST" id="new-watch-form">
+    <form class="pure-form" action="{{ url_for('api_watch_add') }}" method="POST" id="new-watch-form">
         <fieldset>
             <legend>Add a new change detection watch</legend>
-            <input type="url" placeholder="https://..." name="url"/>
-            <input type="text" placeholder="tag" size="10" name="tag" value="{{active_tag if active_tag}}"/>
+                {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
+                {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="tag") }}
             <button type="submit" class="pure-button pure-button-primary">Watch</button>
         </fieldset>
         <!-- add extra stuff, like do a http POST and send headers -->
         <!-- user/pass r = requests.get('https://api.github.com/user', auth=('user', 'pass')) -->
     </form>
     <div>
-        <a href="/" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
+        <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
         {% for tag in tags %}
             {% if tag != "" %}
-                <a href="/?tag={{ tag}}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
+                <a href="{{url_for('index', tag=tag) }}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
             {% endif %}
         {% endfor %}
     </div>
@@ -45,9 +45,12 @@
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
                 {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
                 <td class="inline">{{ loop.index }}</td>
-                <td class="inline paused-state state-{{watch.paused}}"><a href="/?pause={{ watch.uuid}}{% if active_tag %}&tag={{active_tag}}{% endif %}"><img src="/static/images/pause.svg" alt="Pause"/></a></td>
-                <td class="title-col inline">{{watch.title if watch.title is not none else watch.url}}
+                <td class="inline paused-state state-{{watch.paused}}"><a href="{{url_for('index', pause=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause" title="Pause"/></a></td>
+
+                <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.url }}"></a>
+                    {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="/static/images/Google-Chrome-icon.png" />{% endif %}
+
                     {% if watch.last_error is defined and watch.last_error != False %}
                     <div class="fetch-error">{{ watch.last_error }}</div>
                     {% endif %}
@@ -63,14 +66,14 @@
                     {% endif %}
                 </td>
                 <td>
-                    <a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
+                    <a href="{{ url_for('api_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="pure-button button-small pure-button-primary">Recheck</a>
-                    <a href="/edit/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
                     {% if watch.history|length >= 2 %}
-                    <a href="/diff/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
+                    <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
                     {% else %}
                         {% if watch.history|length == 1 %}
-                            <a href="/preview/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
+                            <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
                         {% endif %}
                     {% endif %}
                 </td>
@@ -81,15 +84,15 @@
         <ul id="post-list-buttons">
             {% if has_unviewed %}
             <li>
-                <a href="/api/mark-all-viewed" class="pure-button button-tag ">Mark all viewed</a>
+                <a href="{{url_for('mark_all_viewed', tag=request.args.get('tag')) }}" class="pure-button button-tag ">Mark all viewed</a>
             </li>
             {% endif %}
             <li>
-               <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('api_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
                 all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
             </li>
             <li>
-                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="/static/images/Generic_Feed-icon.svg" height="15px"></a>
+                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15px"></a>
             </li>
         </ul>
     </div>

changedetectionio/tests/conftest.py

@@ -1,17 +1,31 @@
 #!/usr/bin/python3
 
 import pytest
-from backend import changedetection_app
-from backend import store
+from changedetectionio import changedetection_app
+from changedetectionio import store
 import os
 
-
 # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
 # Much better boilerplate than the docs
 # https://www.python-boilerplate.com/py3+flask+pytest/
 
 global app
 
+
+def cleanup(datastore_path):
+    # Unlink test output files
+    files = ['output.txt',
+             'url-watches.json',
+             'notification.txt',
+             'count.txt',
+             'endpoint-content.txt']
+    for file in files:
+        try:
+            os.unlink("{}/{}".format(datastore_path, file))
+            x = 1
+        except FileNotFoundError:
+            pass
+
 @pytest.fixture(scope='session')
 def app(request):
     """Create application for the tests."""
@@ -22,12 +36,13 @@ def app(request):
     except FileExistsError:
         pass
 
-    try:
-        os.unlink("{}/url-watches.json".format(datastore_path))
-    except FileNotFoundError:
-        pass
+    # Enable a BASE_URL for notifications to work (so we can look for diff/ etc URLs)
+    os.environ["BASE_URL"] = "http://mysite.com/"
+    cleanup(datastore_path)
+
 
     app_config = {'datastore_path': datastore_path}
+    cleanup(app_config['datastore_path'])
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
     app = changedetection_app(app_config, datastore)
     app.config['STOP_THREADS'] = True
@@ -35,13 +50,8 @@ def app(request):
     def teardown():
         datastore.stop_thread = True
         app.config.exit.set()
-        for fname in ["url-watches.json", "count.txt", "output.txt"]:
-            try:
-                os.unlink("{}/{}".format(datastore_path, fname))
-            except FileNotFoundError:
-                # This is fine in the case of a failure.
-                pass
+        cleanup(app_config['datastore_path'])
 
+       
     request.addfinalizer(teardown)
     yield app
-

changedetectionio/tests/test_access_control.py

@@ -12,7 +12,9 @@ def test_check_access_control(app, client):
         # Enable password check.
         res = c.post(
             url_for("settings_page"),
-            data={"password": "foobar", "minutes_between_check": 180},
+            data={"password": "foobar",
+                  "minutes_between_check": 180,
+                  'fetch_backend': "html_requests"},
             follow_redirects=True
         )
 
@@ -46,7 +48,7 @@ def test_check_access_control(app, client):
         assert b"LOG OUT" in res.data
 
         # Now remove the password so other tests function, @todo this should happen before each test automatically
-        res = c.get(url_for("settings_page", removepassword="true"),
+        res = c.get(url_for("settings_page", removepassword="yes"),
               follow_redirects=True)
         assert b"Password protection removed." in res.data
 
@@ -66,8 +68,11 @@ def test_check_access_control_no_blank_password(app, client):
         # Enable password check.
         res = c.post(
             url_for("settings_page"),
-            data={"password": "", "minutes_between_check": 180},
-            follow_redirects=True
+            data={"password": "",
+                  "minutes_between_check": 180,
+                  'fetch_backend': "html_requests"},
+
+        follow_redirects=True
         )
 
         assert b"Password protection enabled." not in res.data
@@ -86,14 +91,15 @@ def test_check_access_no_remote_access_to_remove_password(app, client):
         # Enable password check.
         res = c.post(
             url_for("settings_page"),
-            data={"password": "password", "minutes_between_check": 180},
+            data={"password": "password", "minutes_between_check": 180,
+                  'fetch_backend': "html_requests"},
             follow_redirects=True
         )
 
         assert b"Password protection enabled." in res.data
         assert b"Login" in res.data
 
-        res = c.get(url_for("settings_page", removepassword="true"),
+        res = c.get(url_for("settings_page", removepassword="yes"),
               follow_redirects=True)
         assert b"Password protection removed." not in res.data
 

changedetectionio/tests/test_backend.py

@@ -8,8 +8,6 @@ from . util import set_original_response, set_modified_response, live_server_set
 sleep_time_for_fetch_thread = 3
 
 
-
-
 def test_check_basic_change_detection_functionality(client, live_server):
     set_original_response()
     live_server_setup(live_server)
@@ -82,15 +80,27 @@ def test_check_basic_change_detection_functionality(client, live_server):
         # It should report nothing found (no new 'unviewed' class)
         res = client.get(url_for("index"))
         assert b'unviewed' not in res.data
+        assert b'head title' not in res.data # Should not be present because this is off by default
         assert b'test-endpoint' in res.data
 
     set_original_response()
 
+    # Enable auto pickup of <title> in settings
+    res = client.post(
+        url_for("settings_page"),
+        data={"extract_title_as_title": "1", "minutes_between_check": 180, 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
+
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
+    # It should have picked up the <title>
+    assert b'head title' in res.data
 
+    #
     # Cleanup everything
     res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

changedetectionio/tests/test_css_selector.py

@@ -4,6 +4,8 @@ import time
 from flask import url_for
 from . util import live_server_setup
 
+from ..html_tools import *
+
 def test_setup(live_server):
     live_server_setup(live_server)
 
@@ -20,7 +22,7 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/output.txt", "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
@@ -37,12 +39,36 @@ def set_modified_response():
      </html>
     """
 
-    with open("test-datastore/output.txt", "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
     return None
 
 
+# Test that the CSS extraction works how we expect, important here is the right placing of new lines \n's
+def test_css_filter_output():
+    from changedetectionio import fetch_site_status
+    from inscriptis import get_text
+
+    # Check text with sub-parts renders correctly
+    content = """<html> <body><div id="thingthing" >  Some really <b>bold</b> text  </div> </body> </html>"""
+    html_blob = css_filter(css_filter="#thingthing", html_content=content)
+    text = get_text(html_blob)
+    assert text == "  Some really bold text"
+
+    content = """<html> <body>
+    <p>foo bar blah</p>
+    <div class="parts">Block A</div> <div class="parts">Block B</div></body> 
+    </html>
+"""
+    html_blob = css_filter(css_filter=".parts", html_content=content)
+    text = get_text(html_blob)
+
+    # Divs are converted to 4 whitespaces by inscriptis
+    assert text == "    Block A\n    Block B"
+
+
+# Tests the whole stack works with the CSS Filter
 def test_check_markup_css_filter_restriction(client, live_server):
     sleep_time_for_fetch_thread = 3
 
@@ -72,7 +98,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": ""},
+        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_headers.py

@@ -0,0 +1,80 @@
+import json
+import time
+from flask import url_for
+from . util import set_original_response, set_modified_response, live_server_setup
+
+# Hard to just add more live server URLs when one test is already running (I think)
+# So we add our test here (was in a different file)
+def test_headers_in_request(client, live_server):
+    live_server_setup(live_server)
+
+    # Add our URL to the import page
+    test_url = url_for('test_headers', _external=True)
+
+    # Add the test URL twice, we will check
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;'
+
+
+    # Add some headers to a request
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": test_url,
+              "tag": "",
+              "fetch_backend": "html_requests",
+              "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+
+    # Give the thread time to pick up the first version
+    time.sleep(5)
+
+    # The service should echo back the request headers
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Flask will convert the header key to uppercase
+    assert b"Xxx:ooo" in res.data
+    assert b"Cool:yeah" in res.data
+
+    # The test call service will return the headers as the body
+    from html import escape
+    assert escape(cookie_header).encode('utf-8') in res.data
+
+    time.sleep(5)
+
+    # Re #137 -  Examine the JSON index file, it should have only one set of headers entered
+    watches_with_headers = 0
+    with open('test-datastore/url-watches.json') as f:
+        app_struct = json.load(f)
+        for uuid in app_struct['watching']:
+            if (len(app_struct['watching'][uuid]['headers'])):
+                watches_with_headers += 1
+
+    # Should be only one with headers set
+    assert watches_with_headers==1
+
+
+
+
+
+
+

changedetectionio/tests/test_ignore_regex_text.py

@@ -10,7 +10,7 @@ def test_setup(live_server):
 # Unit test of the stripper
 # Always we are dealing in utf-8
 def test_strip_regex_text_func():
-    from backend import fetch_site_status
+    from changedetectionio import fetch_site_status
 
     test_content = """
     but sometimes we want to remove the lines.

changedetectionio/tests/test_ignore_text.py

@@ -10,7 +10,7 @@ def test_setup(live_server):
 # Unit test of the stripper
 # Always we are dealing in utf-8
 def test_strip_text_func():
-    from backend import fetch_site_status
+    from changedetectionio import fetch_site_status
 
     test_content = """
     Some content
@@ -41,7 +41,7 @@ def set_original_ignore_response():
 
     """
 
-    with open("test-datastore/output.txt", "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
@@ -57,7 +57,7 @@ def set_modified_original_ignore_response():
 
     """
 
-    with open("test-datastore/output.txt", "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
@@ -75,7 +75,7 @@ def set_modified_ignore_response():
 
     """
 
-    with open("test-datastore/output.txt", "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
 
@@ -107,7 +107,7 @@ def test_check_ignore_text_functionality(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"ignore_text": ignore_text, "url": test_url},
+        data={"ignore_text": ignore_text, "url": test_url, 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_jsonpath_selector.py

@@ -0,0 +1,163 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+import pytest
+
+def test_unittest_inline_html_extract():
+    # So lets pretend that the JSON we want is inside some HTML
+    content="""
+    <html>
+    
+    food and stuff and more
+    <script>
+    alert('nothing really good here');
+    </script>
+    
+    <script type="application/ld+json">
+  xx {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","description":"During the first year of life, nutrition is critical for your baby. NAN OPTIPRO 1 is tailored to ensure your formula fed infant receives balanced, high quality nutrition.<br />Starter infant formula. The age optimised protein source (whey dominant) is from cow’s milk.<br />Backed by more than 150 years of Nestlé expertise.<br />For hygiene and convenience, it is available in an innovative packaging format with a separate storage area for the scoop, and a semi-transparent window which allows you to see how much powder is left in the can without having to open it.","image":"https://cdn0.woolworths.media/content/wowproductimages/large/155536.jpg","brand":{"@context":"http://schema.org","@type":"Organization","name":"Nan"},"gtin13":"7613287517388","offers":{"@context":"http://schema.org","@type":"Offer","potentialAction":{"@context":"http://schema.org","@type":"BuyAction"},"availability":"http://schema.org/InStock","itemCondition":"http://schema.org/NewCondition","price":23.5,"priceCurrency":"AUD"},"review":[],"sku":"155536"}
+</script>
+<body>
+and it can also be repeated
+<script type="application/ld+json">
+  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","description":"During the first year of life, nutrition is critical for your baby. NAN OPTIPRO 1 is tailored to ensure your formula fed infant receives balanced, high quality nutrition.<br />Starter infant formula. The age optimised protein source (whey dominant) is from cow’s milk.<br />Backed by more than 150 years of Nestlé expertise.<br />For hygiene and convenience, it is available in an innovative packaging format with a separate storage area for the scoop, and a semi-transparent window which allows you to see how much powder is left in the can without having to open it.","image":"https://cdn0.woolworths.media/content/wowproductimages/large/155536.jpg","brand":{"@context":"http://schema.org","@type":"Organization","name":"Nan"},"gtin13":"7613287517388","offers":{"@context":"http://schema.org","@type":"Offer","potentialAction":{"@context":"http://schema.org","@type":"BuyAction"},"availability":"http://schema.org/InStock","itemCondition":"http://schema.org/NewCondition","price":23.5,"priceCurrency":"AUD"},"review":[],"sku":"155536"}
+</script>
+<h4>ok</h4>
+</body>
+</html>
+
+    """
+    from .. import html_tools
+
+    # See that we can find the second <script> one, which is not broken, and matches our filter
+    text = html_tools.extract_json_as_string(content, "$.offers.price")
+    assert text == "23.5"
+
+    text = html_tools.extract_json_as_string('{"id":5}', "$.id")
+    assert text == "5"
+
+    # When nothing at all is found, it should throw JSONNOTFound
+    # Which is caught and shown to the user in the watch-overview table
+    with pytest.raises(html_tools.JSONNotFound) as e_info:
+        html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "$.id")
+
+
+def set_original_response():
+    test_return_data = """
+    {
+      "employees": [
+        {
+          "id": 1,
+          "name": "Pankaj",
+          "salary": "10000"
+        },
+        {
+          "name": "David",
+          "salary": "5000",
+          "id": 2
+        }
+      ],
+      "boss": {
+        "name": "Fat guy"
+      }
+    }
+    """
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """
+    {
+      "employees": [
+        {
+          "id": 1,
+          "name": "Pankaj",
+          "salary": "10000"
+        },
+        {
+          "name": "David",
+          "salary": "5000",
+          "id": 2
+        }
+      ],
+      "boss": {
+        "name": "Foobar"
+      }
+    }
+        """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+
+def test_check_json_filter(client, live_server):
+    live_server_setup(live_server)
+
+    json_filter = 'json:boss.name'
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": json_filter,
+              "url": test_url,
+              "tag": "",
+              "headers": "",
+              "fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(json_filter.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+    #  Make a change
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(4)
+
+    # It should have 'unviewed' still
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    # Should not see this, because its not in the JSONPath we entered
+    res = client.get(url_for("diff_history_page", uuid="first"))
+    # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
+    assert b'Foobar' in res.data

changedetectionio/tests/test_notification.py

@@ -0,0 +1,173 @@
+import os
+import time
+import re
+from flask import url_for
+from . util import set_original_response, set_modified_response, live_server_setup
+
+# Hard to just add more live server URLs when one test is already running (I think)
+# So we add our test here (was in a different file)
+def test_check_notification(client, live_server):
+
+    live_server_setup(live_server)
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(3)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Give the thread time to pick up the first version
+    time.sleep(3)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    url = url_for('test_notification_endpoint', _external=True)
+    notification_url = url.replace('http', 'json')
+
+    print (">>>> Notification URL: "+notification_url)
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"notification_urls": notification_url,
+              "notification_title": "New ChangeDetection.io Notification - {watch_url}",
+              "notification_body": "BASE URL: {base_url}\n"
+                                   "Watch URL: {watch_url}\n"
+                                   "Watch UUID: {watch_uuid}\n"
+                                   "Watch title: {watch_title}\n"
+                                   "Watch tag: {watch_tag}\n"
+                                   "Preview: {preview_url}\n"
+                                   "Diff URL: {diff_url}\n"
+                                   "Snapshot: {current_snapshot}\n"
+                                   ":-)",
+              "url": test_url,
+              "tag": "my tag",
+              "title": "my title",
+              "headers": "",
+              "fetch_backend": "html_requests",
+              "trigger_check": "y"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+    assert b"Notifications queued" in res.data
+
+    # Hit the edit page, be sure that we saved it
+    res = client.get(
+        url_for("edit_page", uuid="first"))
+    assert bytes(notification_url.encode('utf-8')) in res.data
+
+
+    # Because we hit 'send test notification on save'
+    time.sleep(3)
+
+    notification_submission = None
+
+    # Verify what was sent as a notification, this file should exist
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+        # Did we see the URL that had a change, in the notification?
+
+    assert test_url in notification_submission
+
+    os.unlink("test-datastore/notification.txt")
+
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Did the front end see it?
+    res = client.get(
+        url_for("index"))
+
+    assert bytes("just now".encode('utf-8')) in res.data
+
+    notification_submission=None
+    # Verify what was sent as a notification
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+        # Did we see the URL that had a change, in the notification?
+
+    assert test_url in notification_submission
+
+        # Re #65 - did we see our foobar.com BASE_URL ?
+        #assert bytes("https://foobar.com".encode('utf-8')) in notification_submission
+
+
+    ##  Now configure something clever, we go into custom config (non-default) mode, this is returned by the endpoint
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(";jasdhflkjadshf kjhsdfkjl ahslkjf haslkjd hfaklsj hf\njl;asdhfkasj stuff we will detect\n")
+
+    res = client.post(
+        url_for("settings_page"),
+        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
+              "notification_urls": "json://foobar.com", #Re #143 should not see that it sent without [test checkbox]
+              "minutes_between_check": 180,
+              "fetch_backend": "html_requests",
+              },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+    # Re #143 - should not see this if we didnt hit the test box
+    assert b"Notifications queued" not in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Did the front end see it?
+    res = client.get(
+        url_for("index"))
+
+    assert bytes("just now".encode('utf-8')) in res.data
+
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+
+        assert re.search('Watch UUID: [0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', notification_submission, re.IGNORECASE)
+        assert "Watch title: my title" in notification_submission
+        assert "Watch tag: my tag" in notification_submission
+        assert "diff/" in notification_submission
+        assert "preview/" in notification_submission
+        assert ":-)" in notification_submission
+        assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
+        # This should insert the {current_snapshot}
+        assert "stuff we will detect" in notification_submission
+
+    # Prove that "content constantly being marked as Changed with no Updating causes notification" is not a thing
+    # https://github.com/dgtlmoon/changedetection.io/discussions/192
+    os.unlink("test-datastore/notification.txt")
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(3)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(3)
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(3)
+    assert os.path.exists("test-datastore/notification.txt") == False
+
+
+    # Now adding a wrong token should give us an error
+    res = client.post(
+        url_for("settings_page"),
+        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
+              "notification_body": "Rubbish: {rubbish}\n",
+              "notification_urls": "json://foobar.com",
+              "minutes_between_check": 180,
+              "fetch_backend": "html_requests"
+              },
+        follow_redirects=True
+    )
+
+    assert bytes("is not a valid token".encode('utf-8')) in res.data

changedetectionio/tests/test_trigger.py

@@ -0,0 +1,131 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_modified_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some NEW nice initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def set_modified_with_trigger_text_response():
+    test_return_data = """<html>
+       <body>
+     Some NEW nice initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     foobar123
+     <br/>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_trigger_functionality(client, live_server):
+
+    live_server_setup(live_server)
+
+    sleep_time_for_fetch_thread = 3
+    trigger_text = "foobar123"
+    set_original_ignore_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"trigger_text": trigger_text,
+              "url": test_url,
+              "fetch_backend": "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(trigger_text.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-endpoint' in res.data
+
+    #  Make a change
+    set_modified_original_ignore_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+    # Just to be sure.. set a regular modified change..
+    time.sleep(sleep_time_for_fetch_thread)
+    set_modified_with_trigger_text_response()
+
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data

changedetectionio/tests/test_trigger_regex.py

@@ -0,0 +1,81 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+
+def test_trigger_regex_functionality(client, live_server):
+
+    live_server_setup(live_server)
+
+    sleep_time_for_fetch_thread = 3
+
+    set_original_ignore_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (just a new one shouldnt have anything)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+    ### test regex
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"trigger_text": '/something \d{3}/',
+              "url": test_url,
+              "fetch_backend": "html_requests"},
+        follow_redirects=True
+    )
+
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("some new noise")
+
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (nothing should match the regex)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("regex test123<br/>\nsomething 123")
+
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data

changedetectionio/tests/test_trigger_regex_with_filter.py

@@ -0,0 +1,84 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+
+def test_trigger_regex_functionality(client, live_server):
+
+    live_server_setup(live_server)
+
+    sleep_time_for_fetch_thread = 3
+
+    set_original_ignore_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (just a new one shouldnt have anything)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+    ### test regex with filter
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"trigger_text": "/cool.stuff\d/",
+              "url": test_url,
+              "css_filter": '#in-here',
+              "fetch_backend": "html_requests"},
+        follow_redirects=True
+    )
+
+    # Check that we have the expected text.. but it's not in the css filter we want
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("<html>some new noise with cool stuff2 ok</html>")
+
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (nothing should match the regex and filter)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write("<html>some new noise with <span id=in-here>cool stuff6</span> ok</html>")
+
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+

changedetectionio/tests/test_watch_fields_storage.py

@@ -0,0 +1,150 @@
+import time
+from flask import url_for
+from urllib.request import urlopen
+from . util import set_original_response, set_modified_response, live_server_setup
+
+
+def test_check_watch_field_storage(client, live_server):
+    set_original_response()
+    live_server_setup(live_server)
+
+    test_url = "http://somerandomsitewewatch.com"
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={ "notification_urls": "json://myapi.com",
+               "minutes_between_check": 126,
+               "css_filter" : ".fooclass",
+               "title" : "My title",
+               "ignore_text" : "ignore this",
+               "url": test_url,
+               "tag": "woohoo",
+               "headers": "curl:foo",
+               'fetch_backend': "html_requests"
+               },
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"json://myapi.com" in res.data
+    assert b"126" in res.data
+    assert b".fooclass" in res.data
+    assert b"My title" in res.data
+    assert b"ignore this" in res.data
+    assert b"http://somerandomsitewewatch.com" in res.data
+    assert b"woohoo" in res.data
+    assert b"curl: foo" in res.data
+
+
+
+# Re https://github.com/dgtlmoon/changedetection.io/issues/110
+def test_check_recheck_global_setting(client, live_server):
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 1566,
+               'fetch_backend': "html_requests"
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Now add a record
+
+    test_url = "http://somerandomsitewewatch.com"
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Now visit the edit page, it should have the default minutes
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should show the default minutes
+    assert b"change to another value if you want to be specific" in res.data
+    assert b"1566" in res.data
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 222,
+                'fetch_backend': "html_requests"
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should show the default minutes
+    assert b"change to another value if you want to be specific" in res.data
+    assert b"222" in res.data
+
+    # Now change it specifically, it should show the new minutes
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url,
+              "minutes_between_check": 55,
+              'fetch_backend': "html_requests"
+              },
+        follow_redirects=True
+    )
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"55" in res.data
+
+    # Now submit an empty field, it should give back the default global minutes
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 666,
+                'fetch_backend': "html_requests"
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url,
+              "minutes_between_check": "",
+              'fetch_backend': "html_requests"
+              },
+        follow_redirects=True
+    )
+
+    assert b"Updated watch." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"666" in res.data
+

changedetectionio/tests/util.py

@@ -3,7 +3,8 @@
 
 def set_original_response():
     test_return_data = """<html>
-       <body>
+    <head><title>head title</title></head>
+    <body>
      Some initial text</br>
      <p>Which is across multiple lines</p>
      </br>
@@ -12,13 +13,14 @@ def set_original_response():
      </html>
     """
 
-    with open("test-datastore/output.txt", "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
     return None
 
 def set_modified_response():
     test_return_data = """<html>
-       <body>
+    <head><title>modified head title</title></head>
+    <body>
      Some initial text</br>
      <p>which has this one new line</p>
      </br>
@@ -27,7 +29,7 @@ def set_modified_response():
      </html>
     """
 
-    with open("test-datastore/output.txt", "w") as f:
+    with open("test-datastore/endpoint-content.txt", "w") as f:
         f.write(test_return_data)
 
     return None
@@ -35,33 +37,38 @@ def set_modified_response():
 
 def live_server_setup(live_server):
 
+
     @live_server.app.route('/test-endpoint')
     def test_endpoint():
         # Tried using a global var here but didn't seem to work, so reading from a file instead.
-        with open("test-datastore/output.txt", "r") as f:
+        with open("test-datastore/endpoint-content.txt", "r") as f:
             return f.read()
 
-    @live_server.app.route('/test_notification_endpoint', methods=['POST'])
+    # Just return the headers in the request
+    @live_server.app.route('/test-headers')
+    def test_headers():
+
+        from flask import request
+        output= []
+
+        for header in request.headers:
+             output.append("{}:{}".format(str(header[0]),str(header[1])   ))
+
+        return "\n".join(output)
+
+
+    # Where we POST to as a notification
+    @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET'])
     def test_notification_endpoint():
         from flask import request
 
-        with open("test-datastore/count.txt", "w") as f:
-            f.write("we hit it\n")
+        with open("test-datastore/notification.txt", "wb") as f:
             # Debug method, dump all POST to file also, used to prove #65
             data = request.stream.read()
             if data != None:
-                f.write(str(data))
+                f.write(data)
 
         print("\n>> Test notification endpoint was hit.\n")
         return "Text was set"
 
-    # And this should return not zero.
-    @live_server.app.route('/test_notification_counter')
-    def test_notification_counter():
-        try:
-            with open("test-datastore/count.txt", "r") as f:
-                return f.read()
-        except FileNotFoundError:
-            return "nope :("
-
-    live_server.start()
+    live_server.start()

changedetectionio/update_worker.py

@@ -0,0 +1,108 @@
+import threading
+import queue
+import time
+
+# Requests for checking on the site use a pool of thread Workers managed by a Queue.
+class update_worker(threading.Thread):
+    current_uuid = None
+
+    def __init__(self, q, notification_q, app, datastore, *args, **kwargs):
+        self.q = q
+        self.app = app
+        self.notification_q = notification_q
+        self.datastore = datastore
+        super().__init__(*args, **kwargs)
+
+    def run(self):
+        from changedetectionio import fetch_site_status
+
+        update_handler = fetch_site_status.perform_site_check(datastore=self.datastore)
+
+        while not self.app.config.exit.is_set():
+
+            try:
+                uuid = self.q.get(block=False)
+            except queue.Empty:
+                pass
+
+            else:
+                self.current_uuid = uuid
+                from changedetectionio import content_fetcher
+
+                if uuid in list(self.datastore.data['watching'].keys()):
+
+                    changed_detected = False
+                    contents = ""
+                    update_obj= {}
+
+                    try:
+                        now = time.time()
+                        changed_detected, update_obj, contents = update_handler.run(uuid)
+
+                        # Always record that we atleast tried
+                        self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3)})
+
+                    except PermissionError as e:
+                        self.app.logger.error("File permission error updating", uuid, str(e))
+                    except content_fetcher.EmptyReply as e:
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error':str(e)})
+
+                    except Exception as e:
+                        self.app.logger.error("Exception reached processing watch UUID:%s - %s", uuid, str(e))
+                        self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
+
+                    else:
+                        if update_obj:
+                            try:
+                                self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
+                                if changed_detected:
+
+                                    # A change was detected
+                                    newest_version_file_contents = ""
+                                    fname = self.datastore.save_history_text(watch_uuid=uuid, contents=contents)
+
+                                    # Update history with the stripped text for future reference, this will also mean we save the first
+                                    # Should always be keyed by string(timestamp)
+                                    self.datastore.update_watch(uuid, {"history": {str(update_obj["last_checked"]): fname}})
+
+                                    watch = self.datastore.data['watching'][uuid]
+
+                                    print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))
+
+                                    # Get the newest snapshot data to be possibily used in a notification
+                                    newest_key = self.datastore.get_newest_history_key(uuid)
+                                    if newest_key:
+                                        with open(watch['history'][newest_key], 'r') as f:
+                                            newest_version_file_contents = f.read().strip()
+
+                                    n_object = {
+                                        'watch_url': watch['url'],
+                                        'uuid': uuid,
+                                        'current_snapshot': newest_version_file_contents
+                                    }
+
+                                    # Did it have any notification alerts to hit?
+                                    if len(watch['notification_urls']):
+                                        print(">>> Notifications queued for UUID from watch {}".format(uuid))
+                                        n_object['notification_urls'] = watch['notification_urls']
+                                        n_object['notification_title'] = watch['notification_title']
+                                        n_object['notification_body'] = watch['notification_body']
+                                        self.notification_q.put(n_object)
+
+                                    # No? maybe theres a global setting, queue them all
+                                    elif len(self.datastore.data['settings']['application']['notification_urls']):
+                                        print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(uuid))
+                                        n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
+                                        n_object['notification_title'] = self.datastore.data['settings']['application']['notification_title']
+                                        n_object['notification_body'] = self.datastore.data['settings']['application']['notification_body']
+                                        self.notification_q.put(n_object)
+                                    else:
+                                        print(">>> NO notifications queued, watch and global notification URLs were empty.")
+
+                            except Exception as e:
+                                print("!!!! Exception in update_worker !!!\n", e)
+
+                self.current_uuid = None  # Done
+                self.q.task_done()
+
+            self.app.config.exit.wait(1)

docker-compose.yml

@@ -6,20 +6,45 @@ services:
       hostname: changedetection.io
       volumes:
         - changedetection-data:/datastore
+
   #    environment:
+  #        Default listening port, can also be changed with the -p option
+  #      - PORT=5000
+
   #      - PUID=1000
   #      - PGID=1000
+  #       # Alternative WebDriver/selenium URL, do not use "'s or 's!
+  #      - WEBDRIVER_URL=http://browser-chrome:4444/wd/hub
   #        Proxy support example.
-  #      - HTTP_PROXY="socks5h://10.10.1.10:1080"
-  #      - HTTPS_PROXY="socks5h://10.10.1.10:1080"
+  #      - HTTP_PROXY=socks5h://10.10.1.10:1080
+  #      - HTTPS_PROXY=socks5h://10.10.1.10:1080
   #        An exclude list (useful for notification URLs above) can be specified by with
   #      - NO_PROXY="localhost,192.168.0.0/24"
-  #        Base URL of your changedetection.io install (Added to notification alert
-  #      - BASE_URL="https://mysite.com"
+  #        Base URL of your changedetection.io install (Added to the notification alert)
+  #      - BASE_URL=https://mysite.com
 
+  #        Respect proxy_pass type settings, `proxy_set_header Host "localhost";` and `proxy_set_header X-Forwarded-Prefix /app;`
+  #        More here https://github.com/dgtlmoon/changedetection.io/wiki/Running-changedetection.io-behind-a-reverse-proxy-sub-directory
+  #      - USE_X_SETTINGS=1
+
+      # Comment out ports: when using behind a reverse proxy , enable networks: etc.
       ports:
         - 5000:5000
-      restart: always
+      restart: unless-stopped
+
+     # Used for fetching pages via WebDriver+Chrome where you need Javascript support.
+     # Does not work on rPi, https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver
+
+#    browser-chrome:
+#        hostname: browser-chrome
+#        image: selenium/standalone-chrome-debug:3.141.59
+#        environment:
+#            - VNC_NO_PASSWORD=1
+#        volumes:
+#            # Workaround to avoid the browser crashing inside a docker container
+#            # See https://github.com/SeleniumHQ/docker-selenium#quick-start
+#            - /dev/shm:/dev/shm
+#        restart: unless-stopped
 
 volumes:
   changedetection-data:

heroku.yml

@@ -0,0 +1,5 @@
+build:
+  docker:
+    changedetection: Dockerfile
+run:
+  changedetection: python ./changedetection.py -d /datastore -p $PORT

requirements-dev.txt

@@ -0,0 +1,2 @@
+pytest ~=6.2
+pytest-flask ~=1.2

requirements.txt

@@ -1,23 +1,22 @@
 chardet==2.3.0
 flask~= 1.0
-pytest ~=6.2
-pytest-flask ~=1.2
+
 eventlet>=0.31.0
 requests[socks] ~= 2.15
 validators
 timeago ~=1.0
-inscriptis ~= 1.1
+inscriptis ~= 1.2
 feedgen ~= 0.9
 flask-login ~= 0.5
 pytz
 urllib3
 wtforms ~= 2.3.3
-
-
+jsonpath-ng ~= 1.5.3
 
 # Notification library
 apprise ~= 0.9
 
-# Used for CSS filtering
+# Used for CSS filtering, replace with soupsieve and lxml for xpath
 bs4
 
+selenium ~= 3.141

setup.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+import codecs
+import os.path
+import re
+import sys
+
+from setuptools import setup, find_packages
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+
+def read(*parts):
+    return codecs.open(os.path.join(here, *parts), 'r').read()
+
+
+def find_version(*file_paths):
+    version_file = read(*file_paths)
+    version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
+                              version_file, re.M)
+    if version_match:
+        return version_match.group(1)
+    raise RuntimeError("Unable to find version string.")
+
+
+install_requires = open('requirements.txt').readlines()
+
+setup(
+    name='changedetection.io',
+    version=find_version("changedetectionio", "__init__.py"),
+    description='Website change detection and monitoring service',
+    long_description=open('README-pip.md').read(),
+    long_description_content_type='text/markdown',
+    keywords='website change monitor for changes notification change detection '
+             'alerts tracking website tracker change alert website and monitoring',
+    zip_safe=False,
+    entry_points={"console_scripts": ["changedetection.io=changedetection:main"]},
+    author='dgtlmoon',
+    url='https://changedetection.io',
+    scripts=['changedetection.py'],
+    packages=['changedetectionio'],
+    include_package_data=True,
+    install_requires=install_requires,
+    license="Apache License 2.0",
+    python_requires=">= 3.6",
+    classifiers=['Intended Audience :: Customer Service',
+                 'Intended Audience :: Developers',
+                 'Intended Audience :: Education',
+                 'Intended Audience :: End Users/Desktop',
+                 'Intended Audience :: Financial and Insurance Industry',
+                 'Intended Audience :: Healthcare Industry',
+                 'Intended Audience :: Information Technology',
+                 'Intended Audience :: Legal Industry',
+                 'Intended Audience :: Manufacturing',
+                 'Intended Audience :: Other Audience',
+                 'Intended Audience :: Religion',
+                 'Intended Audience :: Science/Research',
+                 'Intended Audience :: System Administrators',
+                 'Intended Audience :: Telecommunications Industry',
+                 'Topic :: Education',
+                 'Topic :: Internet',
+                 'Topic :: Internet :: WWW/HTTP :: Indexing/Search',
+                 'Topic :: Internet :: WWW/HTTP :: Site Management',
+                 'Topic :: Internet :: WWW/HTTP :: Site Management :: Link Checking',
+                 'Topic :: Internet :: WWW/HTTP :: Browsers',
+                 'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
+                 'Topic :: Office/Business',
+                 'Topic :: Other/Nonlisted Topic',
+                 'Topic :: Scientific/Engineering :: Information Analysis',
+                 'Topic :: Text Processing :: Markup :: HTML',
+                 'Topic :: Utilities'
+                 ],
+)