tag 0.38.2

Re #156 - You can specify the port here too, to be sure

.github/workflows/image-tag.yml

@@ -0,0 +1,91 @@
+name: Test, build and push tagged release to Docker Hub
+
+on:
+  push:
+    tags:
+      - '*.*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - uses: olegtarasov/get-tag@v2.1
+        id: tagName
+
+#        with:
+#          tagRegex: "foobar-(.*)"  # Optional. Returns specified group text as tag name. Full tag string is returned if regex is not defined.
+#          tagRegexGroup: 1 # Optional. Default is 1.
+
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Create release metadata
+        run: |
+          # COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > backend/source.txt
+          echo ${{ github.ref }} > backend/tag.txt
+
+      - name: Test with pytest
+        run: |
+          # Each test is totally isolated and performs its own cleanup/reset
+          cd backend; ./run_all_tests.sh
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: all
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+          version: latest
+          driver-opts: image=moby/buildkit:master
+
+      - name: tag
+        run : echo ${{ github.event.release.tag_name }}
+
+      - name: Build and push tagged version
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ steps.tagName.outputs.tag }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+        env:
+            SOURCE_NAME: ${{ steps.branch_name.outputs.SOURCE_NAME }}
+            SOURCE_BRANCH: ${{ steps.branch_name.outputs.SOURCE_BRANCH }}
+            SOURCE_TAG: ${{ steps.branch_name.outputs.SOURCE_TAG }}
+
+      - name: Image digest
+        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}

Dockerfile

@@ -12,7 +12,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libxslt-dev \
     zlib1g-dev \
     g++
-    
+
 RUN mkdir /install
 WORKDIR /install
 

README.md

@@ -26,9 +26,22 @@ Know when ...
 - Festivals with changes
 - Realestate listing changes
 - COVID related news from government websites
+- Detect and monitor changes in JSON API responses 
+- API monitoring and alerting
 
+_Need an actual Chrome runner with Javascript support? see the experimental <a href="https://github.com/dgtlmoon/changedetection.io/tree/javascript-browser">Javascript/Chrome support changedetection.io branch!</a>_
 
 **Get monitoring now! super simple, one command!**
+Run the python code on your own machine by cloning this repository, or with <a href="https://docs.docker.com/get-docker/">docker</a> and/or <a href="https://www.digitalocean.com/community/tutorial_collections/how-to-install-docker-compose">docker-compose</a>
+
+With one docker-compose command
+
+```bash
+docker-compose up -d
+```
+
+or
+
 
 ```bash
 docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
@@ -51,7 +64,6 @@ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/dat
 
 Examining differences in content.
 
-
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
 
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
@@ -78,9 +90,35 @@ Just some examples
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications"  title="Self-hosted web page change monitoring notifications"  />
 
+Now you can also customise your notification content!
+
+### JSON API Monitoring
+
+Detect changes and monitor data in JSON API's by using the built-in JSONPath selectors as a filter / selector.
+
+![image](https://user-images.githubusercontent.com/275001/125165842-0ce01980-e1dc-11eb-9e73-d8137dd162dc.png)
+
+This will re-parse the JSON and apply formatting to the text, making it super easy to monitor and detect changes in JSON API results
+
+![image](https://user-images.githubusercontent.com/275001/125165995-d9ea5580-e1dc-11eb-8030-f0deced2661a.png)
+
+#### Parse JSON embedded in HTML!
+
+When you enable a `json:` filter, you can even automatically extract and parse embedded JSON inside a HTML page! Amazingly handy for sites that build content based on JSON, such as many e-commerce websites. 
+
+```
+<html>
+...
+<script type="application/ld+json">
+  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","price": 23.50 }
+</script>
+```  
+
+`json:$.price` would give `23.50`, or you can extract the whole structure
+
 ### Proxy
 
-A proxy for ChangeDectection.io can be configured by setting environment the 
+A proxy for ChangeDetection.io can be configured by setting environment the 
 `HTTP_PROXY`, `HTTPS_PROXY` variables, examples are also in the `docker-compose.yml`
 
 `NO_PROXY` exclude list can be specified by following `"localhost,192.168.0.0/24"`
@@ -99,10 +137,12 @@ This proxy support also extends to the notifications https://github.com/caronc/a
 
 ### Notes
 
-- Does not yet support Javascript
-- Wont work with Cloudfare type "Please turn on javascript" protected pages
+- ~~Does not yet support Javascript~~
+- ~~Wont work with Cloudfare type "Please turn on javascript" protected pages~~
 - You can use the 'headers' section to monitor password protected web page changes
 
+See the experimental <a href="https://github.com/dgtlmoon/changedetection.io/tree/javascript-browser">Javascript/Chrome browser support!</a>
+
 ### RaspberriPi support?
 
 RaspberriPi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! 

backend/__init__.py

@@ -223,7 +223,6 @@ def changedetection_app(config=None, datastore_o=None):
     @login_required
     def index():
         limit_tag = request.args.get('tag')
-
         pause_uuid = request.args.get('pause')
 
         if pause_uuid:
@@ -279,7 +278,11 @@ def changedetection_app(config=None, datastore_o=None):
             return response
 
         else:
+            from backend import forms
+            form = forms.quickWatchForm(request.form)
+
             output = render_template("watch-overview.html",
+                                     form=form,
                                      watches=sorted_watches,
                                      tags=existing_tags,
                                      active_tag=limit_tag,
@@ -296,25 +299,28 @@ def changedetection_app(config=None, datastore_o=None):
         if request.method == 'POST':
             confirmtext = request.form.get('confirmtext')
             limit_date = request.form.get('limit_date')
+            limit_timestamp = 0
 
-            try:
-                limit_date = limit_date.replace('T', ' ')
-                # I noticed chrome will show '/' but actually submit '-'
-                limit_date = limit_date.replace('-', '/')
-                # In the case that :ss seconds are supplied
-                limit_date = re.sub('(\d\d:\d\d)(:\d\d)', '\\1', limit_date)
+            # Re #149 - allow empty/0 timestamp limit
+            if len(limit_date):
+                try:
+                    limit_date = limit_date.replace('T', ' ')
+                    # I noticed chrome will show '/' but actually submit '-'
+                    limit_date = limit_date.replace('-', '/')
+                    # In the case that :ss seconds are supplied
+                    limit_date = re.sub('(\d\d:\d\d)(:\d\d)', '\\1', limit_date)
 
-                str_to_dt = datetime.datetime.strptime(limit_date, '%Y/%m/%d %H:%M')
-                limit_timestamp = int(str_to_dt.timestamp())
+                    str_to_dt = datetime.datetime.strptime(limit_date, '%Y/%m/%d %H:%M')
+                    limit_timestamp = int(str_to_dt.timestamp())
 
-                if limit_timestamp > time.time():
-                    flash("Timestamp is in the future, cannot continue.", 'error')
+                    if limit_timestamp > time.time():
+                        flash("Timestamp is in the future, cannot continue.", 'error')
+                        return redirect(url_for('scrub_page'))
+
+                except ValueError:
+                    flash('Incorrect date format, cannot continue.', 'error')
                     return redirect(url_for('scrub_page'))
 
-            except ValueError:
-                flash('Incorrect date format, cannot continue.', 'error')
-                return redirect(url_for('scrub_page'))
-
             if confirmtext == 'scrub':
                 changes_removed = 0
                 for uuid, watch in datastore.data['watching'].items():
@@ -380,6 +386,11 @@ def changedetection_app(config=None, datastore_o=None):
             populate_form_from_watch(form, datastore.data['watching'][uuid])
 
         if request.method == 'POST' and form.validate():
+
+            # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
+            if form.minutes_between_check.data == datastore.data['settings']['requests']['minutes_between_check']:
+                form.minutes_between_check.data = None
+
             update_obj = {'url': form.url.data.strip(),
                           'minutes_between_check': form.minutes_between_check.data,
                           'tag': form.tag.data.strip(),
@@ -417,18 +428,34 @@ def changedetection_app(config=None, datastore_o=None):
 
             if form.trigger_check.data:
                 n_object = {'watch_url': form.url.data.strip(),
-                            'notification_urls': form.notification_urls.data}
+                            'notification_urls': form.notification_urls.data,
+                            'uuid': uuid}
                 notification_q.put(n_object)
 
                 flash('Notifications queued.')
 
-            return redirect(url_for('index'))
+            # Diff page [edit] link should go back to diff page
+            if request.args.get("next") and request.args.get("next") == 'diff':
+                return redirect(url_for('diff_history_page', uuid=uuid))
+            else:
+                return redirect(url_for('index'))
 
         else:
             if request.method == 'POST' and not form.validate():
                 flash("An error occurred, please see below.", "error")
 
-            output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], form=form)
+            # Re #110 offer the default minutes
+            using_default_minutes = False
+            if form.minutes_between_check.data == None:
+                form.minutes_between_check.data = datastore.data['settings']['requests']['minutes_between_check']
+                using_default_minutes = True
+
+            output = render_template("edit.html",
+                                     uuid=uuid,
+                                     watch=datastore.data['watching'][uuid],
+                                     form=form,
+                                     using_default_minutes=using_default_minutes
+                                     )
 
         return output
 
@@ -442,42 +469,30 @@ def changedetection_app(config=None, datastore_o=None):
         if request.method == 'GET':
             form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check'])
             form.notification_urls.data = datastore.data['settings']['application']['notification_urls']
+            form.extract_title_as_title.data = datastore.data['settings']['application']['extract_title_as_title']
+            form.notification_title.data = datastore.data['settings']['application']['notification_title']
+            form.notification_body.data = datastore.data['settings']['application']['notification_body']
 
             # Password unset is a GET
-            if request.values.get('removepassword') == 'true':
+            if request.values.get('removepassword') == 'yes':
                 from pathlib import Path
                 datastore.data['settings']['application']['password'] = False
                 flash("Password protection removed.", 'notice')
                 flask_login.logout_user()
+                return redirect(url_for('settings_page'))
 
         if request.method == 'POST' and form.validate():
 
             datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
             datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data
-
-            if len(form.notification_urls.data):
-                import apprise
-                apobj = apprise.Apprise()
-                apobj.debug = True
-
-                # Add each notification
-                for n in datastore.data['settings']['application']['notification_urls']:
-                    apobj.add(n)
-                outcome = apobj.notify(
-                    body='Hello from the worlds best and simplest web page change detection and monitoring service!',
-                    title='Changedetection.io Notification Test',
-                )
-
-                if outcome:
-                    flash("{} Notification URLs reached.".format(len(form.notification_urls.data)), "notice")
-                else:
-                    flash("One or more Notification URLs failed", 'error')
-
+            datastore.data['settings']['application']['extract_title_as_title'] = form.extract_title_as_title.data
+            datastore.data['settings']['application']['notification_title'] = form.notification_title.data
+            datastore.data['settings']['application']['notification_body'] = form.notification_body.data
 
             datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
             datastore.needs_write = True
 
-            if form.trigger_check.data:
+            if form.trigger_check.data and len(form.notification_urls.data):
                 n_object = {'watch_url': "Test from changedetection.io!",
                             'notification_urls': form.notification_urls.data}
                 notification_q.put(n_object)
@@ -494,7 +509,10 @@ def changedetection_app(config=None, datastore_o=None):
         if request.method == 'POST' and not form.validate():
             flash("An error occurred, please see below.", "error")
 
-        output = render_template("settings.html", form=form)
+        # Same as notification.py
+        base_url = os.getenv('BASE_URL', '').strip('"')
+        output = render_template("settings.html", form=form, base_url=base_url)
+
         return output
 
     @app.route("/import", methods=['GET', "POST"])
@@ -550,7 +568,7 @@ def changedetection_app(config=None, datastore_o=None):
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
-        extra_stylesheets = ['/static/styles/diff.css']
+        extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
         try:
             watch = datastore.data['watching'][uuid]
         except KeyError:
@@ -593,7 +611,9 @@ def changedetection_app(config=None, datastore_o=None):
                                  uuid=uuid,
                                  newest_version_timestamp=dates[0],
                                  current_previous_version=str(previous_version),
-                                 current_diff_url=watch['url'])
+                                 current_diff_url=watch['url'],
+                                 extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']),
+                                 left_sticky= True )
 
         return output
 
@@ -605,7 +625,7 @@ def changedetection_app(config=None, datastore_o=None):
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
 
-        extra_stylesheets = ['/static/styles/diff.css']
+        extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
 
         try:
             watch = datastore.data['watching'][uuid]
@@ -699,19 +719,26 @@ def changedetection_app(config=None, datastore_o=None):
     @app.route("/api/add", methods=['POST'])
     @login_required
     def api_watch_add():
+        from backend import forms
+        form = forms.quickWatchForm(request.form)
 
-        url = request.form.get('url').strip()
-        if datastore.url_exists(url):
-            flash('The URL {} already exists'.format(url), "error")
+        if form.validate():
+
+            url = request.form.get('url').strip()
+            if datastore.url_exists(url):
+                flash('The URL {} already exists'.format(url), "error")
+                return redirect(url_for('index'))
+
+            # @todo add_watch should throw a custom Exception for validation etc
+            new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
+            # Straight into the queue.
+            update_q.put(new_uuid)
+
+            flash("Watch added.")
+            return redirect(url_for('index'))
+        else:
+            flash("Error")
             return redirect(url_for('index'))
-
-        # @todo add_watch should throw a custom Exception for validation etc
-        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
-        # Straight into the queue.
-        update_q.put(new_uuid)
-
-        flash("Watch added.")
-        return redirect(url_for('index'))
 
     @app.route("/api/delete", methods=['GET'])
     @login_required
@@ -781,7 +808,9 @@ def check_for_new_version():
         try:
             r = requests.post("https://changedetection.io/check-ver.php",
                               data={'version': datastore.data['version_tag'],
-                                    'app_guid': datastore.data['app_guid']},
+                                    'app_guid': datastore.data['app_guid'],
+                                    'watch_count': len(datastore.data['watching'])
+                                    },
 
                               verify=False)
         except:
@@ -797,39 +826,22 @@ def check_for_new_version():
         app.config.exit.wait(86400)
 
 def notification_runner():
-
     while not app.config.exit.is_set():
         try:
             # At the moment only one thread runs (single runner)
             n_object = notification_q.get(block=False)
         except queue.Empty:
             time.sleep(1)
-            pass
 
         else:
-            import apprise
-
-            # Create an Apprise instance
+            # Process notifications
             try:
-                apobj = apprise.Apprise()
-                for url in n_object['notification_urls']:
-                    apobj.add(url.strip())
-
-                n_body = n_object['watch_url']
-
-                # 65 - Append URL of instance to the notification if it is set.
-                base_url = os.getenv('BASE_URL')
-                if base_url != None:
-                    n_body += "\n" + base_url
-
-                apobj.notify(
-                    body=n_body,
-                    # @todo This should be configurable.
-                    title="ChangeDetection.io Notification - {}".format(n_object['watch_url'])
-                )
+                from backend import notification
+                notification.process_notification(n_object, datastore)
 
             except Exception as e:
-                print("Watch URL: {}  Error {}".format(n_object['watch_url'],e))
+                print("Watch URL: {}  Error {}".format(n_object['watch_url'], e))
+
 
 
 # Thread runner to check every minute, look for new watches to feed into the Queue.
@@ -854,7 +866,7 @@ def ticker_thread_check_time_launch_checks():
         for uuid, watch in datastore.data['watching'].items():
 
             # If they supplied an individual entry minutes to threshold.
-            if 'minutes_between_check' in watch:
+            if 'minutes_between_check' in watch and watch['minutes_between_check'] is not None:
                 max_time = watch['minutes_between_check'] * 60
             else:
                 # Default system wide.

backend/fetch_site_status.py

@@ -3,8 +3,11 @@ import requests
 import hashlib
 from inscriptis import get_text
 import urllib3
+from . import html_tools
+
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
+
 # Some common stuff here that can be moved to a base class
 class perform_site_check():
 
@@ -48,6 +51,7 @@ class perform_site_check():
 
     def run(self, uuid):
         timestamp = int(time.time())  # used for storage etc too
+
         stripped_text_from_html = False
         changed_detected = False
 
@@ -59,7 +63,7 @@ class perform_site_check():
         extra_headers = self.datastore.get_val(uuid, 'headers')
 
         # Tweak the base config with the per-watch ones
-        request_headers = self.datastore.data['settings']['headers']
+        request_headers = self.datastore.data['settings']['headers'].copy()
         request_headers.update(extra_headers)
 
         # https://github.com/psf/requests/issues/4525
@@ -82,18 +86,20 @@ class perform_site_check():
                              timeout=timeout,
                              verify=False)
 
-            # CSS Filter
-            css_filter = self.datastore.data['watching'][uuid]['css_filter']
-            if css_filter and len(css_filter.strip()):
-                from bs4 import BeautifulSoup
-                soup = BeautifulSoup(r.content, "html.parser")
-                stripped_text_from_html = ""
-                for item in soup.select(css_filter):
-                    text = str(item.get_text()).strip() + '\n'
-                    stripped_text_from_html += text
+            html = r.text
 
-            else:
-                stripped_text_from_html = get_text(r.text)
+            is_html = True
+            css_filter_rule = self.datastore.data['watching'][uuid]['css_filter']
+            if css_filter_rule and len(css_filter_rule.strip()):
+                if 'json:' in css_filter_rule:
+                    stripped_text_from_html = html_tools.extract_json_as_string(html, css_filter_rule)
+                    is_html = False
+                else:
+                    # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+                    html = html_tools.css_filter(css_filter=css_filter_rule, html_content=r.content)
+
+            if is_html:
+                stripped_text_from_html = get_text(html)
 
         # Usually from networkIO/requests level
         except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
@@ -142,4 +148,10 @@ class perform_site_check():
 
                 update_obj["previous_md5"] = fetched_md5
 
+            # Extract title as title
+            if self.datastore.data['settings']['application']['extract_title_as_title']:
+                if not self.datastore.data['watching'][uuid]['title'] or not len(self.datastore.data['watching'][uuid]['title']):
+                    update_obj['title'] = html_tools.extract_element(find='title', html_content=html)
+
+
         return changed_detected, update_obj, stripped_text_from_html

backend/forms.py

@@ -75,14 +75,14 @@ class StringDictKeyValue(StringField):
             # Remove empty strings
             cleaned = list(filter(None, valuelist[0].split("\n")))
             for s in cleaned:
-                parts = s.strip().split(':')
+                parts = s.strip().split(':', 1)
                 if len(parts) == 2:
                     self.data.update({parts[0].strip(): parts[1].strip()})
 
         else:
             self.data = {}
 
-class ListRegex(object):
+class ValidateListRegex(object):
     """
     Validates that anything that looks like a regex passes as a regex
     """
@@ -102,19 +102,43 @@ class ListRegex(object):
                     message = field.gettext('RegEx \'%s\' is not a valid regular expression.')
                     raise ValidationError(message % (line))
 
+class ValidateCSSJSONInput(object):
+    """
+    Filter validation
+    @todo CSS validator ;)
+    """
 
-class watchForm(Form):
+    def __init__(self, message=None):
+        self.message = message
+
+    def __call__(self, form, field):
+        if 'json:' in field.data:
+            from jsonpath_ng.exceptions import JsonPathParserError
+            from jsonpath_ng import jsonpath, parse
+
+            input = field.data.replace('json:', '')
+
+            try:
+                parse(input)
+            except JsonPathParserError as e:
+                message = field.gettext('\'%s\' is not a valid JSONPath expression. (%s)')
+                raise ValidationError(message % (input, str(e)))
+
+class quickWatchForm(Form):
     # https://wtforms.readthedocs.io/en/2.3.x/fields/#module-wtforms.fields.html5
     # `require_tld` = False is needed even for the test harness "http://localhost:5005.." to run
 
     url = html5.URLField('URL', [validators.URL(require_tld=False)])
     tag = StringField('Tag', [validators.Optional(), validators.Length(max=35)])
+
+class watchForm(quickWatchForm):
+
     minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
                                                [validators.Optional(), validators.NumberRange(min=1)])
-    css_filter = StringField('CSS Filter')
+    css_filter = StringField('CSS/JSON Filter', [ValidateCSSJSONInput()])
     title = StringField('Title')
 
-    ignore_text = StringListField('Ignore Text', [ListRegex()])
+    ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
     notification_urls = StringListField('Notification URL List')
     headers = StringDictKeyValue('Request Headers')
     trigger_check = BooleanField('Send test notification on save')
@@ -128,4 +152,8 @@ class globalSettingsForm(Form):
                                                [validators.NumberRange(min=1)])
 
     notification_urls = StringListField('Notification URL List')
+    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title')
     trigger_check = BooleanField('Send test notification on save')
+
+    notification_title = StringField('Notification Title')
+    notification_body = TextAreaField('Notification Body')

backend/html_tools.py

@@ -0,0 +1,90 @@
+import json
+from bs4 import BeautifulSoup
+from jsonpath_ng import parse
+
+
+class JSONNotFound(ValueError):
+    def __init__(self, msg):
+        ValueError.__init__(self, msg)
+
+# Given a CSS Rule, and a blob of HTML, return the blob of HTML that matches
+def css_filter(css_filter, html_content):
+    soup = BeautifulSoup(html_content, "html.parser")
+    html_block = ""
+    for item in soup.select(css_filter, separator=""):
+        html_block += str(item)
+
+    return html_block + "\n"
+
+
+# Extract/find element
+def extract_element(find='title', html_content=''):
+
+    #Re #106, be sure to handle when its not found
+    element_text = None
+
+    soup = BeautifulSoup(html_content, 'html.parser')
+    result = soup.find(find)
+    if result and result.string:
+        element_text = result.string.strip()
+
+    return element_text
+
+#
+def _parse_json(json_data, jsonpath_filter):
+    s=[]
+    jsonpath_expression = parse(jsonpath_filter.replace('json:', ''))
+    match = jsonpath_expression.find(json_data)
+
+    # More than one result, we will return it as a JSON list.
+    if len(match) > 1:
+        for i in match:
+            s.append(i.value)
+
+    # Single value, use just the value, as it could be later used in a token in notifications.
+    if len(match) == 1:
+        s = match[0].value
+
+    if not s:
+        raise JSONNotFound("No Matching JSON could be found for the rule {}".format(jsonpath_filter.replace('json:', '')))
+
+    stripped_text_from_html = json.dumps(s, indent=4)
+
+    return stripped_text_from_html
+
+def extract_json_as_string(content, jsonpath_filter):
+
+    stripped_text_from_html = False
+
+    # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded <script type=ldjson>
+    try:
+        stripped_text_from_html = _parse_json(json.loads(content), jsonpath_filter)
+    except json.JSONDecodeError:
+
+        # Foreach <script json></script> blob.. just return the first that matches jsonpath_filter
+        s = []
+        soup = BeautifulSoup(content, 'html.parser')
+        bs_result = soup.findAll('script')
+
+        if not bs_result:
+            raise JSONNotFound("No parsable JSON found in this document")
+
+        for result in bs_result:
+            # Skip empty tags, and things that dont even look like JSON
+            if not result.string or not '{' in result.string:
+                continue
+                
+            try:
+                json_data = json.loads(result.string)
+            except json.JSONDecodeError:
+                # Just skip it
+                continue
+            else:
+                stripped_text_from_html = _parse_json(json_data, jsonpath_filter)
+                if stripped_text_from_html:
+                    break
+
+    if not stripped_text_from_html:
+        raise JSONNotFound("No JSON matching the rule '%s' found" % jsonpath_filter.replace('json:',''))
+
+    return stripped_text_from_html

backend/notification.py

@@ -0,0 +1,57 @@
+import os
+import apprise
+
+def process_notification(n_object, datastore):
+    apobj = apprise.Apprise()
+    for url in n_object['notification_urls']:
+        print (">> Process Notification: AppRise notifying {}".format(url.strip()))
+        apobj.add(url.strip())
+
+    # Get the notification body from datastore
+    n_body = datastore.data['settings']['application']['notification_body']
+    # Get the notification title from the datastore
+    n_title = datastore.data['settings']['application']['notification_title']
+
+    # Insert variables into the notification content
+    notification_parameters = create_notification_parameters(n_object)
+    raw_notification_text = [n_body, n_title]
+
+    parameterised_notification_text = dict(
+        [
+            (i, n.replace(n, n.format(**notification_parameters)))
+            for i, n in zip(['body', 'title'], raw_notification_text)
+        ]
+    )
+
+    apobj.notify(
+        body=parameterised_notification_text["body"],
+        title=parameterised_notification_text["title"]
+    )
+
+
+# Notification title + body content parameters get created here.
+def create_notification_parameters(n_object):
+
+    # in the case we send a test notification from the main settings, there is no UUID.
+    uuid = n_object['uuid'] if 'uuid' in n_object else ''
+
+    # Create URLs to customise the notification with
+    base_url = os.getenv('BASE_URL', '').strip('"')
+    watch_url = n_object['watch_url']
+
+    # Re #148 - Some people have just {base_url} in the body or title, but this may break some notification services
+    #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.
+    if base_url == '':
+        base_url = "<base-url-env-var-not-set>"
+
+    diff_url = "{}/diff/{}".format(base_url, uuid)
+    preview_url = "{}/preview/{}".format(base_url, uuid)
+
+    return {
+        'base_url': base_url,
+        'watch_url': watch_url,
+        'diff_url': diff_url,
+        'preview_url': preview_url,
+        'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else ''
+    }
+

backend/static/js/settings.js

@@ -0,0 +1,16 @@
+window.addEventListener("load", (event) => {
+  // just an example for now
+  function toggleVisible(elem) {
+    // theres better ways todo this
+    var x = document.getElementById(elem);
+    if (x.style.display === "block") {
+      x.style.display = "none";
+    } else {
+      x.style.display = "block";
+    }
+  }
+
+  document.getElementById("toggle-customise-notifications").onclick = function () {
+    toggleVisible("notification-customisation");
+  };
+});

backend/static/styles/diff.css

@@ -12,7 +12,8 @@
     border: 1px solid transparent;
     vertical-align: top;
     font: 1em monospace;
-    text-align: left;
+    text-align: left; }
+  #diff-ui pre {
     white-space: pre-wrap; }
 
 h1 {

backend/static/styles/diff.scss

@@ -16,9 +16,10 @@
         vertical-align: top;
         font: 1em monospace;
         text-align: left;
-        white-space: pre-wrap;
     }
-
+    pre {
+            white-space: pre-wrap;
+    }
 }
 h1 {
 	display: inline;

backend/static/styles/package.json

@@ -10,6 +10,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "node-sass": "^6.0.0"
+    "node-sass": "^6.0.1",
+    "trim-newlines": "^3.0.1"
   }
 }

backend/static/styles/styles.css

@@ -102,6 +102,24 @@ body:after, body:before {
   -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);
   clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%); }
 
+.arrow {
+  border: solid black;
+  border-width: 0 3px 3px 0;
+  display: inline-block;
+  padding: 3px; }
+  .arrow.right {
+    transform: rotate(-45deg);
+    -webkit-transform: rotate(-45deg); }
+  .arrow.left {
+    transform: rotate(135deg);
+    -webkit-transform: rotate(135deg); }
+  .arrow.up {
+    transform: rotate(-135deg);
+    -webkit-transform: rotate(-135deg); }
+  .arrow.down {
+    transform: rotate(45deg);
+    -webkit-transform: rotate(45deg); }
+
 .button-small {
   font-size: 85%; }
 
@@ -115,7 +133,8 @@ body:after, body:before {
   background: #fff;
   padding: 2em;
   margin: 1em;
-  border-radius: 5px; }
+  border-radius: 5px;
+  min-width: 70%; }
 
 .button-secondary {
   color: white;
@@ -165,17 +184,30 @@ body:after, body:before {
   .messages li.notice {
     background: rgba(255, 255, 255, 0.5); }
 
+#notification-customisation {
+  display: block;
+  border: 1px solid #ccc;
+  padding: 1rem;
+  border-radius: 5px; }
+
+#toggle-customise-notifications {
+  cursor: pointer; }
+
+#token-table.pure-table td, #token-table.pure-table th {
+  font-size: 80%; }
+
 #new-watch-form {
   background: rgba(0, 0, 0, 0.05);
   padding: 1em;
   border-radius: 10px;
   margin-bottom: 1em; }
-
-#new-watch-form legend {
-  color: #fff; }
-
-#new-watch-form input {
-  width: auto !important; }
+  #new-watch-form input {
+    width: auto !important;
+    display: inline-block; }
+  #new-watch-form .label {
+    display: none; }
+  #new-watch-form legend {
+    color: #fff; }
 
 #diff-col {
   padding-left: 40px; }
@@ -183,7 +215,7 @@ body:after, body:before {
 #diff-jump {
   position: fixed;
   left: 0px;
-  top: 80px;
+  top: 120px;
   background: #fff;
   padding: 10px;
   border-top-right-radius: 5px;
@@ -208,13 +240,25 @@ footer {
 #feed-icon {
   vertical-align: middle; }
 
-#version {
+#top-right-menu {
+  /*
+    position: absolute;
+    right: 0px;
+    background: linear-gradient(to right, #fff0, #fff 10%);
+    padding-left: 20px;
+    padding-right: 10px;
+    */ }
+
+.sticky-tab {
   position: absolute;
   top: 80px;
-  right: 0px;
   font-size: 8px;
   background: #fff;
   padding: 10px; }
+  .sticky-tab#left-sticky {
+    left: 0px; }
+  .sticky-tab#right-sticky {
+    right: 0px; }
 
 #new-version-text a {
   color: #e07171; }
@@ -237,7 +281,7 @@ footer {
   /* The list of errors */ }
   .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
     padding-bottom: 1em; }
-    .pure-form .pure-control-group dd, .pure-form .pure-group dd, .pure-form .pure-controls dd {
+    .pure-form .pure-control-group div, .pure-form .pure-group div, .pure-form .pure-controls div {
       margin: 0px; }
   .pure-form .error input {
     background-color: #ffebeb; }
@@ -253,18 +297,15 @@ footer {
       color: #dd0000; }
   .pure-form label {
     font-weight: bold; }
-  .pure-form input[type=url] {
-    width: 100%; }
   .pure-form textarea {
-    width: 100%;
-    font-size: 14px; }
+    width: 100%; }
 
 @media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
   .box {
     max-width: 95%; }
   .edit-form {
     padding: 0.5em;
-    margin: 0.5em; }
+    margin: 0; }
   #nav-menu {
     overflow-x: scroll; } }
 
@@ -274,6 +315,8 @@ This query will take effect for any screen smaller than 760px
 and also iPads specifically.
 */
 @media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
+  input[type='text'] {
+    width: 100%; }
   .watch-table {
     /* Force table to not be like tables anymore */
     /* Force table to not be like tables anymore */
@@ -311,3 +354,12 @@ and also iPads specifically.
       background-color: #eee; }
     .watch-table.pure-table-striped tr:nth-child(2n-1) td {
       background-color: inherit; } }
+
+/** Desktop vs mobile input field strategy
+- We dont use 'size' with <input> because `size` is too unreliable to override, and will often push-out
+- Rely always on width in CSS
+*/
+@media only screen and (min-width: 761px) {
+  /* m-d is medium-desktop */
+  .m-d {
+    min-width: 80%; } }

backend/static/styles/styles.scss

@@ -72,7 +72,6 @@ section.content {
   }
 }
 
-
 .watch-tag-list {
   color: #e70069;
   white-space: nowrap;
@@ -137,12 +136,33 @@ body:after, body:before {
   clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)
 }
 
+.arrow {
+  border: solid black;
+  border-width: 0 3px 3px 0;
+  display: inline-block;
+  padding: 3px;
+    &.right {
+      transform: rotate(-45deg);
+      -webkit-transform: rotate(-45deg);
+    }
+    &.left {
+      transform: rotate(135deg);
+      -webkit-transform: rotate(135deg);
+    }
+    &.up {
+      transform: rotate(-135deg);
+      -webkit-transform: rotate(-135deg);
+    }
+    &.down {
+      transform: rotate(45deg);
+      -webkit-transform: rotate(45deg);
+    }
+}
 
 .button-small {
   font-size: 85%;
 }
 
-
 .fetch-error {
   padding-top: 1em;
   font-size: 60%;
@@ -155,6 +175,7 @@ body:after, body:before {
   padding: 2em;
   margin: 1em;
   border-radius: 5px;
+  min-width: 70%;
 }
 
 .button-secondary {
@@ -220,7 +241,24 @@ body:after, body:before {
             background: rgba(255, 255, 255, .5);
         }
     }
+}
 
+#notification-customisation {
+    display: block;
+    border: 1px solid #ccc;
+    padding: 1rem;
+    border-radius: 5px;
+}
+
+#toggle-customise-notifications {
+    cursor: pointer;
+}
+
+
+#token-table {
+    &.pure-table td, &.pure-table th {
+        font-size: 80%;
+    }
 }
 
 #new-watch-form {
@@ -228,15 +266,20 @@ body:after, body:before {
   padding: 1em;
   border-radius: 10px;
   margin-bottom: 1em;
+  input {
+    width: auto !important;
+    display: inline-block;
+  }
+  .label {
+    display: none;
+  }
+  legend {
+    color: #fff;
+  }
 }
 
-#new-watch-form legend {
-  color: #fff;
-}
 
-#new-watch-form input {
-  width: auto !important;
-}
+
 
 #diff-col {
   padding-left: 40px;
@@ -245,7 +288,7 @@ body:after, body:before {
 #diff-jump {
   position: fixed;
   left: 0px;
-  top: 80px;
+  top: 120px;
   background: #fff;
   padding: 10px;
   border-top-right-radius: 5px;
@@ -274,13 +317,31 @@ footer {
   vertical-align: middle;
 }
 
-#version {
+#top-right-menu {
+// Just let flex overflow the x axis for now
+/*
+    position: absolute;
+    right: 0px;
+    background: linear-gradient(to right, #fff0, #fff 10%);
+    padding-left: 20px;
+    padding-right: 10px;
+    */
+}
+
+
+
+.sticky-tab {
   position: absolute;
   top: 80px;
-  right: 0px;
   font-size: 8px;
   background: #fff;
   padding: 10px;
+  &#left-sticky {
+    left: 0px;
+  }
+  &#right-sticky {
+    right: 0px;
+  }
 }
 
 #new-version-text a {
@@ -311,7 +372,7 @@ footer {
 .pure-form {
     .pure-control-group, .pure-group, .pure-controls {
         padding-bottom: 1em;
-        dd {
+        div {
             margin: 0px;
         }
     }
@@ -321,7 +382,8 @@ footer {
         background-color: #ffebeb;
     }
   }
-   /* The list of errors */
+
+  /* The list of errors */
   ul.errors {
     padding: .5em .6em;
     border: 1px solid #dd0000;
@@ -339,13 +401,8 @@ footer {
     font-weight: bold;
   }
 
-  input[type=url] {
-    width: 100%;
-  }
-
   textarea {
     width: 100%;
-    font-size: 14px;
   }
 }
 
@@ -355,7 +412,7 @@ footer {
   }
   .edit-form {
     padding: 0.5em;
-    margin: 0.5em;
+    margin: 0;
   }
   #nav-menu {
     overflow-x: scroll;
@@ -368,9 +425,12 @@ Max width before this PARTICULAR table gets nasty
 This query will take effect for any screen smaller than 760px
 and also iPads specifically.
 */
-@media only screen and (max-width: 760px),
-(min-device-width: 768px) and (max-device-width: 1024px) {
+@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
 
+  input[type='text'] {
+    width: 100%;
+  }
+    
   .watch-table {
     /* Force table to not be like tables anymore */
     thead, tbody, th, td, tr {
@@ -436,4 +496,14 @@ and also iPads specifically.
 }
 
 
+/** Desktop vs mobile input field strategy
+- We dont use 'size' with <input> because `size` is too unreliable to override, and will often push-out
+- Rely always on width in CSS
+*/
+@media only screen and (min-width: 761px) {
+/* m-d is medium-desktop */
+    .m-d {
+        min-width: 80%;
+    }
+}
 

backend/store.py

@@ -38,7 +38,11 @@ class ChangeDetectionStore:
                 },
                 'application': {
                     'password': False,
-                    'notification_urls': [] # Apprise URL list
+                    'extract_title_as_title': False,
+                    'notification_urls': [], # Apprise URL list
+                    # Custom notification content
+                    'notification_title': 'ChangeDetection.io Notification - {watch_url}',
+                    'notification_body': '{watch_url} had a change.'
                 }
             }
         }
@@ -53,7 +57,9 @@ class ChangeDetectionStore:
             'last_viewed': 0,  # history key value of the last viewed via the [diff] link
             'newest_history_key': "",
             'title': None,
-            'minutes_between_check': 3 * 60,  # Default 3 hours
+            # Re #110, so then if this is set to None, we know to use the default value instead
+            # Requires setting to None on submit if it's the same as the default
+            'minutes_between_check': None,
             'previous_md5': "",
             'uuid': str(uuid_builder.uuid4()),
             'headers': {},  # Extra headers to send
@@ -111,7 +117,7 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
                 self.add_watch(url='https://changedetection.io', tag='Tech news')
 
-        self.__data['version_tag'] = "0.36"
+        self.__data['version_tag'] = "0.38.2"
 
         # Helper to remove password protection
         password_reset_lockfile = "{}/removepassword.lock".format(self.datastore_path)
@@ -173,7 +179,6 @@ class ChangeDetectionStore:
 
     @property
     def data(self):
-
         has_unviewed = False
         for uuid, v in self.__data['watching'].items():
             self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
@@ -184,6 +189,10 @@ class ChangeDetectionStore:
                 self.__data['watching'][uuid]['viewed'] = False
                 has_unviewed = True
 
+            # #106 - Be sure this is None on empty string, False, None, etc
+            if not self.__data['watching'][uuid]['title']:
+                self.__data['watching'][uuid]['title'] = None
+
         self.__data['has_unviewed'] = has_unviewed
 
         return self.__data
@@ -346,7 +355,7 @@ class ChangeDetectionStore:
             if self.stop_thread:
                 print("Shutting down datastore thread")
                 return
-            
+
             if self.needs_write:
                 self.sync_to_json()
             time.sleep(3)

backend/templates/_helpers.jinja

@@ -1,6 +1,6 @@
 {% macro render_field(field) %}
-  <dt {% if field.errors %} class="error" {% endif %}>{{ field.label }}
-  <dd {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
+  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
   {% if field.errors %}
     <ul class=errors>
     {% for error in field.errors %}
@@ -8,5 +8,18 @@
     {% endfor %}
     </ul>
   {% endif %}
-  </dd>
+  </div>
+{% endmacro %}
+
+{% macro render_simple_field(field) %}
+  <span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>
+  <span {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </span>
 {% endmacro %}

backend/templates/base.html

@@ -4,9 +4,9 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta name="description" content="Self hosted website change detection.">
-    <title>Change Detection</title>
-    <link rel="stylesheet" href="/static/styles/pure-min.css">
-    <link rel="stylesheet" href="/static/styles/styles.css?ver=1000">
+    <title>Change Detection{{extra_title}}</title>
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='pure-min.css')}}">
+    <link rel="stylesheet" href="{{url_for('static_content', group='styles', filename='styles.css')}}">
     {% if extra_stylesheets %}
         {% for m in extra_stylesheets %}
         <link rel="stylesheet" href="{{ m }}?ver=1000">
@@ -21,31 +21,31 @@
         {% if has_password and not current_user.is_authenticated %}
             <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener"><strong>Change</strong>Detection.io</a>
         {% else %}
-            <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
+            <a class="pure-menu-heading" href="{{url_for('index')}}"><strong>Change</strong>Detection.io</a>
         {% endif %}
         {% if current_diff_url %}
         <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</span></a>
         {% else %}
-        {% if new_version_available %}
+        {% if new_version_available and not (has_password and not current_user.is_authenticated) %}
         <span id="new-version-text" class="pure-menu-heading"><a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a></span>
         {% endif %}
         {% endif %}
 
-        <ul class="pure-menu-list">
+        <ul class="pure-menu-list"  id="top-right-menu">
         {% if current_user.is_authenticated or not has_password %}
             {% if not current_diff_url %}
             <li class="pure-menu-item">
-                <a href="/backup" class="pure-menu-link">BACKUP</a>
+                <a href="{{ url_for('get_backup')}}" class="pure-menu-link">BACKUP</a>
             </li>
             <li class="pure-menu-item">
-                <a href="/import" class="pure-menu-link">IMPORT</a>
+                <a href="{{ url_for('import_page')}}" class="pure-menu-link">IMPORT</a>
             </li>
             <li class="pure-menu-item">
-                <a href="/settings" class="pure-menu-link">SETTINGS</a>
+                <a href="{{ url_for('settings_page')}}" class="pure-menu-link">SETTINGS</a>
             </li>
             {% else %}
             <li class="pure-menu-item">
-                <a href="{{ url_for('edit_page', uuid=uuid) }}" class="pure-menu-link">EDIT</a>
+                <a href="{{ url_for('edit_page', uuid=uuid, next='diff') }}" class="pure-menu-link">EDIT</a>
             </li>
             {% endif %}
         {% else %}
@@ -55,7 +55,7 @@
         {% endif %}
 
         {% if current_user.is_authenticated %}
-            <li class="pure-menu-item"><a href="/logout" class="pure-menu-link">LOG OUT</a></li>
+            <li class="pure-menu-item"><a href="{{url_for('logout')}}" class="pure-menu-link">LOG OUT</a></li>
         {% endif %}
             <li class="pure-menu-item"><a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
                 <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16"
@@ -68,7 +68,9 @@
         </ul>
     </div>
 </div>
-<div id="version">v{{ version }}</div>
+
+{% if left_sticky %}<div class="sticky-tab" id="left-sticky"><a href="{{url_for('preview_page', uuid=uuid)}}">Show current snapshot</a></div> {% endif %}
+{% if right_sticky %}<div class="sticky-tab" id="right-sticky">{{ right_sticky }}</div> {% endif %}
 <section class="content">
     <header>
         {% block header %}{% endblock %}

backend/templates/diff.html

@@ -30,7 +30,6 @@
     </form>
     <del>Removed text</del>
     <ins>Inserted Text</ins>
-    <a href="{{ url_for('preview_page', uuid=uuid) }}">Show current snapshot</a>
 </div>
 
 <div id="diff-jump">
@@ -130,7 +129,8 @@ if ('oninput' in a) {
 
 function onDiffTypeChange(radio) {
 	window.diffType = radio.value;
-	document.title = "Diff " + radio.value.slice(4);
+// Not necessary 
+//	document.title = "Diff " + radio.value.slice(4);
 }
 
 var radio = document.getElementsByName('diff_type');

backend/templates/edit.html

@@ -2,25 +2,33 @@
 {% block content %}
 {% from '_helpers.jinja' import render_field %}
 <div class="edit-form monospaced-textarea">
-    <form class="pure-form pure-form-stacked" action="/edit/{{uuid}}" method="POST">
+    <form class="pure-form pure-form-stacked" action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next') ) }}" method="POST">
         <fieldset>
             <div class="pure-control-group">
-                {{ render_field(form.url, placeholder="https://...", size=30, required=true) }}
+                {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
             </div>
             <div class="pure-control-group">
-                {{ render_field(form.title, size=30) }}
+                {{ render_field(form.title, class="m-d") }}
             </div>
             <div class="pure-control-group">
-                {{ render_field(form.tag, size=10) }}
+                {{ render_field(form.tag) }}
             </div>
             <div class="pure-control-group">
-                {{ render_field(form.minutes_between_check, size=5) }}
+                {{ render_field(form.minutes_between_check) }}
+                {% if using_default_minutes %}
+                    <span class="pure-form-message-inline">Currently using the <a href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
+                {% else %}
+                    <span class="pure-form-message-inline">Set to blank to use the <a href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
+                {% endif %}
             </div>
             <div class="pure-control-group">
-                {{ render_field(form.css_filter, size=25, placeholder=".class-name or #some-id, or other CSS selector rule.") }}
-                <span class="pure-form-message-inline">Limit text to this CSS rule, only text matching this CSS rule is included.<br/>
-                    Please be sure that you thoroughly understand how to write CSS selector rules before filing an issue on GitHub!<br/>
-                    Go <a href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>
+                {{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.", class="m-d") }}
+                <span class="pure-form-message-inline">
+                    <ul>
+                        <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
+                        <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <b>"json:"</b>, <a href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
+                    </ul>
+                    Please be sure that you thoroughly understand how to write CSS or JSONPath selector rules before filing an issue on GitHub! <a href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
                 </span>
             </div>
             <!-- @todo: move to tabs --->
@@ -42,12 +50,14 @@ User-Agent: wonderbra 1.0") }}
             </fieldset>
 
             <div class="pure-control-group">
-                {{ render_field(form.notification_urls, rows=5, placeholder="Gitter - gitter://token/room
+                {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
+Gitter - gitter://token/room
 Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
 AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
 SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
 ") }}
                 <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
+                <span class="pure-form-message-inline">Note: This overrides any global settings notification URLs</span>
             </div>
 
             <div class="pure-controls">
@@ -58,14 +68,13 @@ SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
             </div>
             <br/>
             <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Cancel</a>
-                <a href="/api/delete?uuid={{uuid}}"
+                <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Cancel</a>
+                <a href="{{url_for('api_delete', uuid=uuid)}}"
                    class="pure-button button-small button-error ">Delete</a>
             </div>
         </fieldset>
     </form>
 
-
 </div>
 
 {% endblock %}

backend/templates/import.html

@@ -4,7 +4,7 @@
 <div class="edit-form">
 
 
-    <form class="pure-form pure-form-aligned" action="/import" method="POST">
+    <form class="pure-form pure-form-aligned" action="{{url_for('import_page')}}" method="POST">
 
         <fieldset class="pure-group">
             <legend>One URL per line, URLs that do not pass validation will stay in the textarea.</legend>

backend/templates/login.html

@@ -2,7 +2,7 @@
 
 {% block content %}
 <div class="edit-form">
-    <form class="pure-form pure-form-stacked" action="/login" method="POST">
+    <form class="pure-form pure-form-stacked" action="{{url_for('login')}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
                 <label for="password">Password</label>

backend/templates/scrub.html

@@ -2,7 +2,7 @@
 
 {% block content %}
 <div class="edit-form">
-    <form class="pure-form pure-form-stacked" action="/scrub" method="POST">
+    <form class="pure-form pure-form-stacked" action="{{url_for('scrub_page')}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
                 This will remove all version snapshots/data, but keep your list of URLs. <br/>
@@ -26,7 +26,7 @@
             </div>
             <br/>
             <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Cancel</a>
+                <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Cancel</a>
             </div>
         </fieldset>
     </form>

backend/templates/settings.html

@@ -2,46 +2,103 @@
 
 {% block content %}
 {% from '_helpers.jinja' import render_field %}
-
+<script type="text/javascript" src="static/js/settings.js"></script>
 <div class="edit-form">
-    <form class="pure-form pure-form-stacked settings" action="/settings" method="POST">
+    <form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
-                {{ render_field(form.minutes_between_check, size=5) }}
+                {{ render_field(form.minutes_between_check) }}
+                <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
             </div>
             <div class="pure-control-group">
                 {% if current_user.is_authenticated %}
-                    <a href="/settings?removepassword=true" class="pure-button pure-button-primary">Remove password</a>
+                    <a href="{{url_for('settings_page', removepassword='yes')}}" class="pure-button pure-button-primary">Remove password</a>
                 {% else %}
-                    {{ render_field(form.password, size=10) }}
+                {{ render_field(form.password) }}
+                <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
                 {% endif %}
             </div>
             <div class="pure-control-group">
-                {{ render_field(form.notification_urls, rows=5, placeholder="Gitter - gitter://token/room
+                {{ render_field(form.extract_title_as_title) }}
+                <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
+            </div>
+
+            <div class="field-group">
+
+                <div class="pure-control-group">
+                    {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
+Gitter - gitter://token/room
 Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
 AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
-SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
-") }}
-                <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
-            </div>
-                <div class="pure-controls">
-                    <span class="pure-form-message-inline"><label for="trigger-test-notification" class="pure-checkbox">
-                        <input type="checkbox" id="trigger-test-notification" name="trigger-test-notification"> Send test notification on save.</label></span>
-
+SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com") }}
+                    <div class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!
+                    <a id="toggle-customise-notifications">Customise notification body: <i
+                            class="arrow down"></i></a>
+                        </div>
                 </div>
+                <div id="notification-customisation" style="display:none;">
 
-            <br/>
+                    <div class="pure-control-group">
+                        {{ render_field(form.notification_title, class="m-d") }}
+                        <span class="pure-form-message-inline">Title for all notifications</span>
+                    </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.notification_body , rows=5) }}
+                        <span class="pure-form-message-inline">Body for all notifications</span>
+                    </div>
+                    <div class="pure-controls">
+                        <span class="pure-form-message-inline">
+                            These tokens can be used in the notification body and title to
+                            customise the notification text.
+                        </span>
+                        <table class="pure-table" id="token-table">
+                            <thead>
+                            <tr>
+                                <th>Token</th>
+                                <th>Description</th>
+                            </tr>
+                            </thead>
+                            <tbody>
+                            <tr>
+                                <td><code>{base_url}</code></td>
+                                <td>The URL of the changedetection.io instance you are running.</td>
+                            </tr>
+                            <tr>
+                                <td><code>{watch_url}</code></td>
+                                <td>The URL being watched.</td>
+                            </tr>
+                            <tr>
+                                <td><code>{preview_url}</code></td>
+                                <td>The URL of the preview page generated by changedetection.io.</td>
+                            </tr>
+                            <tr>
+                                <td><code>{diff_url}</code></td>
+                                <td>The URL of the diff page generated by changedetection.io.</td>
+                            </tr>
+                            <tr>
+                                <td><code>{current_snapshot}</code></td>
+                                <td>The current snapshot value, useful when combined with JSON or CSS filters</td>
+                            </tr>
+                            </tbody>
+                        </table>
+                        <span class="pure-form-message-inline">
+                            URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
+                            Your <code>BASE_URL</code> var is currently "{{base_url}}"
+                        </span>
+                    </div>
+                </div>
+                <div class="pure-control-group">
+                    {{ render_field(form.trigger_check) }}
+                </div>
+            </div>
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Save</button>
             </div>
             <br/>
-
             <div class="pure-control-group">
-                <a href="/" class="pure-button button-small button-cancel">Back</a>
-                <a href="/scrub" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
+                <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
+                <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
             </div>
-
-
         </fieldset>
     </form>
 

backend/templates/watch-overview.html

@@ -1,24 +1,24 @@
 {% extends 'base.html' %}
-
 {% block content %}
+{% from '_helpers.jinja' import render_simple_field %}
 
 <div class="box">
 
-    <form class="pure-form" action="/api/add" method="POST" id="new-watch-form">
+    <form class="pure-form" action="{{ url_for('api_watch_add') }}" method="POST" id="new-watch-form">
         <fieldset>
             <legend>Add a new change detection watch</legend>
-            <input type="url" placeholder="https://..." name="url"/>
-            <input type="text" placeholder="tag" size="10" name="tag" value="{{active_tag if active_tag}}"/>
+                {{ render_simple_field(form.url, placeholder="https://...", required=true) }}
+                {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="tag") }}
             <button type="submit" class="pure-button pure-button-primary">Watch</button>
         </fieldset>
         <!-- add extra stuff, like do a http POST and send headers -->
         <!-- user/pass r = requests.get('https://api.github.com/user', auth=('user', 'pass')) -->
     </form>
     <div>
-        <a href="/" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
+        <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
         {% for tag in tags %}
             {% if tag != "" %}
-                <a href="/?tag={{ tag}}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
+                <a href="{{url_for('index', tag=tag) }}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
             {% endif %}
         {% endfor %}
     </div>
@@ -45,7 +45,8 @@
                 {% if watch.paused is defined and watch.paused != False %}paused{% endif %}
                 {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
                 <td class="inline">{{ loop.index }}</td>
-                <td class="inline paused-state state-{{watch.paused}}"><a href="/?pause={{ watch.uuid}}{% if active_tag %}&tag={{active_tag}}{% endif %}"><img src="/static/images/pause.svg" alt="Pause"/></a></td>
+                <td class="inline paused-state state-{{watch.paused}}"><a href="{{url_for('index', pause=watch.uuid, tag=active_tag)}}"><img src="{{url_for('static_content', group='images', filename='pause.svg')}}" alt="Pause"/></a></td>
+
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.url }}"></a>
                     {% if watch.last_error is defined and watch.last_error != False %}
@@ -63,14 +64,14 @@
                     {% endif %}
                 </td>
                 <td>
-                    <a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
+                    <a href="{{ url_for('api_watch_checknow', uuid=watch.uuid, tag=request.args.get('tag')) }}"
                        class="pure-button button-small pure-button-primary">Recheck</a>
-                    <a href="/edit/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
+                    <a href="{{ url_for('edit_page', uuid=watch.uuid)}}" class="pure-button button-small pure-button-primary">Edit</a>
                     {% if watch.history|length >= 2 %}
-                    <a href="/diff/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
+                    <a href="{{ url_for('diff_history_page', uuid=watch.uuid) }}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
                     {% else %}
                         {% if watch.history|length == 1 %}
-                            <a href="/preview/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
+                            <a href="{{ url_for('preview_page', uuid=watch.uuid)}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Preview</a>
                         {% endif %}
                     {% endif %}
                 </td>
@@ -81,15 +82,15 @@
         <ul id="post-list-buttons">
             {% if has_unviewed %}
             <li>
-                <a href="/api/mark-all-viewed" class="pure-button button-tag ">Mark all viewed</a>
+                <a href="{{url_for('mark_all_viewed', tag=request.args.get('tag')) }}" class="pure-button button-tag ">Mark all viewed</a>
             </li>
             {% endif %}
             <li>
-               <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
+               <a href="{{ url_for('api_watch_checknow', tag=active_tag) }}" class="pure-button button-tag ">Recheck
                 all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
             </li>
             <li>
-                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="/static/images/Generic_Feed-icon.svg" height="15px"></a>
+                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="{{url_for('static_content', group='images', filename='Generic_Feed-icon.svg')}}" height="15px"></a>
             </li>
         </ul>
     </div>

backend/tests/conftest.py

@@ -12,6 +12,21 @@ import os
 
 global app
 
+
+def cleanup(datastore_path):
+    # Unlink test output files
+    files = ['output.txt',
+             'url-watches.json',
+             'notification.txt',
+             'count.txt',
+             'endpoint-content.txt']
+    for file in files:
+        try:
+            os.unlink("{}/{}".format(datastore_path, file))
+            x = 1
+        except FileNotFoundError:
+            pass
+
 @pytest.fixture(scope='session')
 def app(request):
     """Create application for the tests."""
@@ -22,10 +37,10 @@ def app(request):
     except FileExistsError:
         pass
 
-    try:
-        os.unlink("{}/url-watches.json".format(datastore_path))
-    except FileNotFoundError:
-        pass
+    # Enable a BASE_URL for notifications to work (so we can look for diff/ etc URLs)
+    os.environ["BASE_URL"] = "http://mysite.com/"
+
+    cleanup(datastore_path)
 
     app_config = {'datastore_path': datastore_path}
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
@@ -35,13 +50,8 @@ def app(request):
     def teardown():
         datastore.stop_thread = True
         app.config.exit.set()
-        for fname in ["url-watches.json", "count.txt", "output.txt"]:
-            try:
-                os.unlink("{}/{}".format(datastore_path, fname))
-            except FileNotFoundError:
-                # This is fine in the case of a failure.
-                pass
-
+        cleanup(datastore_path)
+        
     request.addfinalizer(teardown)
     yield app
 

backend/tests/test_access_control.py

@@ -46,7 +46,7 @@ def test_check_access_control(app, client):
         assert b"LOG OUT" in res.data
 
         # Now remove the password so other tests function, @todo this should happen before each test automatically
-        res = c.get(url_for("settings_page", removepassword="true"),
+        res = c.get(url_for("settings_page", removepassword="yes"),
               follow_redirects=True)
         assert b"Password protection removed." in res.data
 
@@ -93,7 +93,7 @@ def test_check_access_no_remote_access_to_remove_password(app, client):
         assert b"Password protection enabled." in res.data
         assert b"Login" in res.data
 
-        res = c.get(url_for("settings_page", removepassword="true"),
+        res = c.get(url_for("settings_page", removepassword="yes"),
               follow_redirects=True)
         assert b"Password protection removed." not in res.data
 

backend/tests/test_backend.py

@@ -8,8 +8,6 @@ from . util import set_original_response, set_modified_response, live_server_set
 sleep_time_for_fetch_thread = 3
 
 
-
-
 def test_check_basic_change_detection_functionality(client, live_server):
     set_original_response()
     live_server_setup(live_server)
@@ -82,15 +80,27 @@ def test_check_basic_change_detection_functionality(client, live_server):
         # It should report nothing found (no new 'unviewed' class)
         res = client.get(url_for("index"))
         assert b'unviewed' not in res.data
+        assert b'head title' not in res.data # Should not be present because this is off by default
         assert b'test-endpoint' in res.data
 
     set_original_response()
 
+    # Enable auto pickup of <title> in settings
+    res = client.post(
+        url_for("settings_page"),
+        data={"extract_title_as_title": "1", "minutes_between_check": 180},
+        follow_redirects=True
+    )
+
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
+
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
+    # It should have picked up the <title>
+    assert b'head title' in res.data
 
+    #
     # Cleanup everything
     res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

backend/tests/test_css_selector.py

@@ -4,6 +4,8 @@ import time
 from flask import url_for
 from . util import live_server_setup
 
+from ..html_tools import *
+
 def test_setup(live_server):
     live_server_setup(live_server)
 
@@ -43,6 +45,30 @@ def set_modified_response():
     return None
 
 
+# Test that the CSS extraction works how we expect, important here is the right placing of new lines \n's
+def test_css_filter_output():
+    from backend import fetch_site_status
+    from inscriptis import get_text
+
+    # Check text with sub-parts renders correctly
+    content = """<html> <body><div id="thingthing" >  Some really <b>bold</b> text  </div> </body> </html>"""
+    html_blob = css_filter(css_filter="#thingthing", html_content=content)
+    text = get_text(html_blob)
+    assert text == "  Some really bold text"
+
+    content = """<html> <body>
+    <p>foo bar blah</p>
+    <div class="parts">Block A</div> <div class="parts">Block B</div></body> 
+    </html>
+"""
+    html_blob = css_filter(css_filter=".parts", html_content=content)
+    text = get_text(html_blob)
+
+    # Divs are converted to 4 whitespaces by inscriptis
+    assert text == "    Block A\n    Block B"
+
+
+# Tests the whole stack works with the CSS Filter
 def test_check_markup_css_filter_restriction(client, live_server):
     sleep_time_for_fetch_thread = 3
 

backend/tests/test_headers.py

@@ -0,0 +1,79 @@
+import json
+import time
+from flask import url_for
+from . util import set_original_response, set_modified_response, live_server_setup
+
+# Hard to just add more live server URLs when one test is already running (I think)
+# So we add our test here (was in a different file)
+def test_headers_in_request(client, live_server):
+    live_server_setup(live_server)
+
+    # Add our URL to the import page
+    test_url = url_for('test_headers', _external=True)
+
+    # Add the test URL twice, we will check
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;'
+
+
+    # Add some headers to a request
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+              "url": test_url,
+              "tag": "",
+              "headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+
+    # Give the thread time to pick up the first version
+    time.sleep(5)
+
+    # The service should echo back the request headers
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Flask will convert the header key to uppercase
+    assert b"Xxx:ooo" in res.data
+    assert b"Cool:yeah" in res.data
+
+    # The test call service will return the headers as the body
+    from html import escape
+    assert escape(cookie_header).encode('utf-8') in res.data
+
+    time.sleep(5)
+
+    # Re #137 -  Examine the JSON index file, it should have only one set of headers entered
+    watches_with_headers = 0
+    with open('test-datastore/url-watches.json') as f:
+        app_struct = json.load(f)
+        for uuid in app_struct['watching']:
+            if (len(app_struct['watching'][uuid]['headers'])):
+                watches_with_headers += 1
+
+    # Should be only one with headers set
+    assert watches_with_headers==1
+
+
+
+
+
+
+

backend/tests/test_jsonpath_selector.py

@@ -0,0 +1,160 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from . util import live_server_setup
+import pytest
+
+def test_unittest_inline_html_extract():
+    # So lets pretend that the JSON we want is inside some HTML
+    content="""
+    <html>
+    
+    food and stuff and more
+    <script>
+    alert('nothing really good here');
+    </script>
+    
+    <script type="application/ld+json">
+  xx {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","description":"During the first year of life, nutrition is critical for your baby. NAN OPTIPRO 1 is tailored to ensure your formula fed infant receives balanced, high quality nutrition.<br />Starter infant formula. The age optimised protein source (whey dominant) is from cow’s milk.<br />Backed by more than 150 years of Nestlé expertise.<br />For hygiene and convenience, it is available in an innovative packaging format with a separate storage area for the scoop, and a semi-transparent window which allows you to see how much powder is left in the can without having to open it.","image":"https://cdn0.woolworths.media/content/wowproductimages/large/155536.jpg","brand":{"@context":"http://schema.org","@type":"Organization","name":"Nan"},"gtin13":"7613287517388","offers":{"@context":"http://schema.org","@type":"Offer","potentialAction":{"@context":"http://schema.org","@type":"BuyAction"},"availability":"http://schema.org/InStock","itemCondition":"http://schema.org/NewCondition","price":23.5,"priceCurrency":"AUD"},"review":[],"sku":"155536"}
+</script>
+<body>
+and it can also be repeated
+<script type="application/ld+json">
+  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","description":"During the first year of life, nutrition is critical for your baby. NAN OPTIPRO 1 is tailored to ensure your formula fed infant receives balanced, high quality nutrition.<br />Starter infant formula. The age optimised protein source (whey dominant) is from cow’s milk.<br />Backed by more than 150 years of Nestlé expertise.<br />For hygiene and convenience, it is available in an innovative packaging format with a separate storage area for the scoop, and a semi-transparent window which allows you to see how much powder is left in the can without having to open it.","image":"https://cdn0.woolworths.media/content/wowproductimages/large/155536.jpg","brand":{"@context":"http://schema.org","@type":"Organization","name":"Nan"},"gtin13":"7613287517388","offers":{"@context":"http://schema.org","@type":"Offer","potentialAction":{"@context":"http://schema.org","@type":"BuyAction"},"availability":"http://schema.org/InStock","itemCondition":"http://schema.org/NewCondition","price":23.5,"priceCurrency":"AUD"},"review":[],"sku":"155536"}
+</script>
+<h4>ok</h4>
+</body>
+</html>
+
+    """
+    from .. import html_tools
+
+    # See that we can find the second <script> one, which is not broken, and matches our filter
+    text = html_tools.extract_json_as_string(content, "$.offers.price")
+    assert text == "23.5"
+
+    text = html_tools.extract_json_as_string('{"id":5}', "$.id")
+    assert text == "5"
+
+    # When nothing at all is found, it should throw JSONNOTFound
+    # Which is caught and shown to the user in the watch-overview table
+    with pytest.raises(html_tools.JSONNotFound) as e_info:
+        html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "$.id")
+
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def set_original_response():
+    test_return_data = """
+    {
+      "employees": [
+        {
+          "id": 1,
+          "name": "Pankaj",
+          "salary": "10000"
+        },
+        {
+          "name": "David",
+          "salary": "5000",
+          "id": 2
+        }
+      ],
+      "boss": {
+        "name": "Fat guy"
+      }
+    }
+    """
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+    return None
+
+def set_modified_response():
+    test_return_data = """
+    {
+      "employees": [
+        {
+          "id": 1,
+          "name": "Pankaj",
+          "salary": "10000"
+        },
+        {
+          "name": "David",
+          "salary": "5000",
+          "id": 2
+        }
+      ],
+      "boss": {
+        "name": "Foobar"
+      }
+    }
+        """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
+
+
+
+def test_check_json_filter(client, live_server):
+
+    json_filter = 'json:boss.name'
+
+    set_original_response()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": json_filter, "url": test_url, "tag": "", "headers": ""},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(json_filter.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(3)
+    #  Make a change
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    # Give the thread time to pick it up
+    time.sleep(3)
+
+    # It should have 'unviewed' still
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
+    # Should not see this, because its not in the JSONPath we entered
+    res = client.get(url_for("diff_history_page", uuid="first"))
+    # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
+    assert b'Foobar' in res.data

backend/tests/test_notification.py

@@ -1,4 +1,4 @@
-
+import os
 import time
 from flask import url_for
 from . util import set_original_response, set_modified_response, live_server_setup
@@ -22,7 +22,7 @@ def test_check_notification(client, live_server):
     )
     assert b"1 Imported" in res.data
 
-    # Give the thread time to pick it up
+    # Give the thread time to pick up the first version
     time.sleep(3)
 
     # Goto the edit page, add our ignore text
@@ -33,16 +33,37 @@ def test_check_notification(client, live_server):
     print (">>>> Notification URL: "+notification_url)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"notification_urls": notification_url, "url": test_url, "tag": "", "headers": ""},
+        data={"notification_urls": notification_url,
+              "url": test_url,
+              "tag": "",
+              "headers": "",
+              "trigger_check": "y"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
+    assert b"Notifications queued" in res.data
 
     # Hit the edit page, be sure that we saved it
     res = client.get(
         url_for("edit_page", uuid="first"))
     assert bytes(notification_url.encode('utf-8')) in res.data
 
+
+    # Because we hit 'send test notification on save'
+    time.sleep(3)
+
+    notification_submission = None
+
+    # Verify what was sent as a notification, this file should exist
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+        # Did we see the URL that had a change, in the notification?
+
+    assert test_url in notification_submission
+
+    os.unlink("test-datastore/notification.txt")
+
+
     set_modified_response()
 
     # Trigger a check
@@ -57,16 +78,53 @@ def test_check_notification(client, live_server):
 
     assert bytes("just now".encode('utf-8')) in res.data
 
+    notification_submission=None
+    # Verify what was sent as a notification
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+        # Did we see the URL that had a change, in the notification?
 
-    # Check it triggered
-    res = client.get(
-        url_for("test_notification_counter"),
+    assert test_url in notification_submission
+
+        # Re #65 - did we see our foobar.com BASE_URL ?
+        #assert bytes("https://foobar.com".encode('utf-8')) in notification_submission
+
+
+    ##  Now configure something clever, we go into custom config (non-default) mode
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(";jasdhflkjadshf kjhsdfkjl ahslkjf haslkjd hfaklsj hf\njl;asdhfkasj stuff we will detect\n")
+
+    res = client.post(
+        url_for("settings_page"),
+        data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
+              "notification_body": "{base_url}\n{watch_url}\n{preview_url}\n{diff_url}\n{current_snapshot}\n:-)",
+              "notification_urls": "json://foobar.com", #Re #143 should not see that it sent without [test checkbox]
+              "minutes_between_check": 180},
+        follow_redirects=True
     )
+    assert b"Settings updated." in res.data
+    # Re #143 - should not see this if we didnt hit the test box
+    assert b"Notifications queued" not in res.data
 
-    assert bytes("we hit it".encode('utf-8')) in res.data
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
-    # Did we see the URL that had a change, in the notification?
-    assert bytes("test-endpoint".encode('utf-8')) in res.data
+    # Give the thread time to pick it up
+    time.sleep(3)
 
-    # Re #65 - did we see our foobar.com BASE_URL ?
-    assert bytes("https://foobar.com".encode('utf-8')) in res.data
+    # Did the front end see it?
+    res = client.get(
+        url_for("index"))
+
+    assert bytes("just now".encode('utf-8')) in res.data
+
+    with open("test-datastore/notification.txt", "r") as f:
+        notification_submission = f.read()
+
+        assert "diff/" in notification_submission
+        assert "preview/" in notification_submission
+        assert ":-)" in notification_submission
+        assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
+        # This should insert the {current_snapshot}
+        assert "stuff we will detect" in notification_submission

backend/tests/test_watch_fields_storage.py

@@ -50,3 +50,96 @@ def test_check_watch_field_storage(client, live_server):
 
 
 
+# Re https://github.com/dgtlmoon/changedetection.io/issues/110
+def test_check_recheck_global_setting(client, live_server):
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 1566,
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Now add a record
+
+    test_url = "http://somerandomsitewewatch.com"
+
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Now visit the edit page, it should have the default minutes
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should show the default minutes
+    assert b"change to another value if you want to be specific" in res.data
+    assert b"1566" in res.data
+
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 222,
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+
+    # Should show the default minutes
+    assert b"change to another value if you want to be specific" in res.data
+    assert b"222" in res.data
+
+    # Now change it specifically, it should show the new minutes
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url,
+              "minutes_between_check": 55,
+              },
+        follow_redirects=True
+    )
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"55" in res.data
+
+    # Now submit an empty field, it should give back the default global minutes
+    res = client.post(
+        url_for("settings_page"),
+        data={
+               "minutes_between_check": 666,
+               },
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"url": test_url,
+              "minutes_between_check": "",
+              },
+        follow_redirects=True
+    )
+
+    assert b"Updated watch." in res.data
+
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b"666" in res.data
+

backend/tests/util.py

@@ -3,7 +3,8 @@
 
 def set_original_response():
     test_return_data = """<html>
-       <body>
+    <head><title>head title</title></head>
+    <body>
      Some initial text</br>
      <p>Which is across multiple lines</p>
      </br>
@@ -18,7 +19,8 @@ def set_original_response():
 
 def set_modified_response():
     test_return_data = """<html>
-       <body>
+    <head><title>modified head title</title></head>
+    <body>
      Some initial text</br>
      <p>which has this one new line</p>
      </br>
@@ -35,33 +37,38 @@ def set_modified_response():
 
 def live_server_setup(live_server):
 
+
     @live_server.app.route('/test-endpoint')
     def test_endpoint():
         # Tried using a global var here but didn't seem to work, so reading from a file instead.
         with open("test-datastore/output.txt", "r") as f:
             return f.read()
 
+    # Just return the headers in the request
+    @live_server.app.route('/test-headers')
+    def test_headers():
+
+        from flask import request
+        output= []
+
+        for header in request.headers:
+             output.append("{}:{}".format(str(header[0]),str(header[1])   ))
+
+        return "\n".join(output)
+
+
+    # Where we POST to as a notification
     @live_server.app.route('/test_notification_endpoint', methods=['POST'])
     def test_notification_endpoint():
         from flask import request
 
-        with open("test-datastore/count.txt", "w") as f:
-            f.write("we hit it\n")
+        with open("test-datastore/notification.txt", "wb") as f:
             # Debug method, dump all POST to file also, used to prove #65
             data = request.stream.read()
             if data != None:
-                f.write(str(data))
+                f.write(data)
 
         print("\n>> Test notification endpoint was hit.\n")
         return "Text was set"
 
-    # And this should return not zero.
-    @live_server.app.route('/test_notification_counter')
-    def test_notification_counter():
-        try:
-            with open("test-datastore/count.txt", "r") as f:
-                return f.read()
-        except FileNotFoundError:
-            return "nope :("
-
-    live_server.start()
+    live_server.start()

backend/update_worker.py

@@ -31,33 +31,49 @@ class update_worker(threading.Thread):
                     try:
                         changed_detected, result, contents = update_handler.run(uuid)
 
-                    except PermissionError as s:
-                        self.app.logger.error("File permission error updating", uuid, str(s))
+                    except PermissionError as e:
+                        self.app.logger.error("File permission error updating", uuid, str(e))
+                    except Exception as e:
+                        self.app.logger.error("Exception reached", uuid, str(e))
                     else:
                         if result:
                             try:
                                 self.datastore.update_watch(uuid=uuid, update_obj=result)
                                 if changed_detected:
-                                    # A change was detected
-                                    self.datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
 
+                                    # A change was detected
+                                    newest_version_file_contents = ""
+                                    self.datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
                                     watch = self.datastore.data['watching'][uuid]
 
+                                    print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))
+
+                                    # Get the newest snapshot data to be possibily used in a notification
+                                    newest_key = self.datastore.get_newest_history_key(uuid)
+                                    if newest_key:
+                                        with open(watch['history'][newest_key], 'r') as f:
+                                            newest_version_file_contents = f.read().strip()
+
+                                    n_object = {
+                                        'watch_url': watch['url'],
+                                        'uuid': uuid,
+                                        'current_snapshot': newest_version_file_contents
+                                    }
+
                                     # Did it have any notification alerts to hit?
                                     if len(watch['notification_urls']):
-                                        print("Processing notifications for UUID: {}".format(uuid))
-                                        n_object = {'watch_url': self.datastore.data['watching'][uuid]['url'],
-                                                    'notification_urls': watch['notification_urls']}
+                                        print(">>> Notifications queued for UUID from watch {}".format(uuid))
+                                        n_object['notification_urls'] = watch['notification_urls']
                                         self.notification_q.put(n_object)
 
-
                                     # No? maybe theres a global setting, queue them all
                                     elif len(self.datastore.data['settings']['application']['notification_urls']):
-                                        print("Processing GLOBAL notifications for UUID: {}".format(uuid))
-                                        n_object = {'watch_url': self.datastore.data['watching'][uuid]['url'],
-                                                    'notification_urls': self.datastore.data['settings']['application'][
-                                                        'notification_urls']}
+                                        print(">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}".format(uuid))
+                                        n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
                                         self.notification_q.put(n_object)
+                                    else:
+                                        print(">>> NO notifications queued, watch and global notification URLs were empty.")
+
                             except Exception as e:
                                 print("!!!! Exception in update_worker !!!\n", e)
 

changedetection.py

@@ -14,7 +14,7 @@ from backend import store
 
 def main(argv):
     ssl_mode = False
-    port = 5000
+    port = os.environ.get('PORT') or 5000
     do_cleanup = False
 
     # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
@@ -60,11 +60,23 @@ def main(argv):
 
     @app.context_processor
     def inject_version():
-        return dict(version=datastore.data['version_tag'],
+        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
                     new_version_available=app.config['NEW_VERSION_AVAILABLE'],
                     has_password=datastore.data['settings']['application']['password'] != False
                     )
 
+    # Proxy sub-directory support
+    # Set environment var USE_X_SETTINGS=1 on this script
+    # And then in your proxy_pass settings
+    #
+    #         proxy_set_header Host "localhost";
+    #         proxy_set_header X-Forwarded-Prefix /app;
+
+    if os.getenv('USE_X_SETTINGS'):
+        print ("USE_X_SETTINGS is ENABLED\n")
+        from werkzeug.middleware.proxy_fix import ProxyFix
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
+
     if ssl_mode:
         # @todo finalise SSL config, but this should get you in the right direction if you need it.
         eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen(('', port)),
@@ -73,7 +85,7 @@ def main(argv):
                                                server_side=True), app)
 
     else:
-        eventlet.wsgi.server(eventlet.listen(('', port)), app)
+        eventlet.wsgi.server(eventlet.listen(('', int(port))), app)
 
 
 if __name__ == '__main__':

docker-compose.yml

@@ -7,6 +7,9 @@ services:
       volumes:
         - changedetection-data:/datastore
   #    environment:
+  #        Default listening port, can also be changed with the -p option
+  #      - PORT=5000
+
   #      - PUID=1000
   #      - PGID=1000
   #        Proxy support example.
@@ -14,11 +17,17 @@ services:
   #      - HTTPS_PROXY="socks5h://10.10.1.10:1080"
   #        An exclude list (useful for notification URLs above) can be specified by with
   #      - NO_PROXY="localhost,192.168.0.0/24"
-  #        Base URL of your changedetection.io install (Added to notification alert
+  #        Base URL of your changedetection.io install (Added to the notification alert)
   #      - BASE_URL="https://mysite.com"
 
+  #        Respect proxy_pass type settings, `proxy_set_header Host "localhost";` and `proxy_set_header X-Forwarded-Prefix /app;`
+  #        More here https://github.com/dgtlmoon/changedetection.io/wiki/Running-changedetection.io-behind-a-reverse-proxy-sub-directory
+  #      - USE_X_SETTINGS=1
+
+      # Comment out ports: when using behind a reverse proxy , enable networks: etc.
       ports:
         - 5000:5000
+
       restart: always
 
 volumes:

heroku.yml

@@ -0,0 +1,5 @@
+build:
+  docker:
+    changedetection: Dockerfile
+run:
+  changedetection: python ./changedetection.py -d /datastore -p $PORT

requirements.txt

@@ -6,18 +6,18 @@ eventlet>=0.31.0
 requests[socks] ~= 2.15
 validators
 timeago ~=1.0
-inscriptis ~= 1.1
+inscriptis ~= 1.2
 feedgen ~= 0.9
 flask-login ~= 0.5
 pytz
 urllib3
 wtforms ~= 2.3.3
-
+jsonpath-ng ~= 1.5.3
 
 
 # Notification library
 apprise ~= 0.9
 
-# Used for CSS filtering
+# Used for CSS filtering, replace with soupsieve and lxml for xpath
 bs4