Fixing workflow tag syntax issues

Re #152 ability to over-ride env var BASE_URL, with UI+tests

.github/workflows/containers.yml

@@ -0,0 +1,125 @@
+name: Build and push containers
+
+on:
+  # Automatically triggered by a testing workflow passing
+  workflow_run:
+    workflows: ["ChangeDetection.io Test"]
+    types: [completed]
+
+  # Or a new tagged release
+  release:
+    types: [created, published, edited]
+
+jobs:
+  metadata:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Show metadata
+      run: |
+        echo SHA ${{ github.sha }}
+        echo github.ref:  ${{ github.ref }}
+        echo github_ref: $GITHUB_REF
+        echo Event name: ${{ github.event_name }}
+        echo Ref ${{ github.ref }}
+        echo c: ${{ github.event.workflow_run.conclusion }}
+        echo r: ${{ github.event.workflow_run }}
+        echo tname: ${{ github.event.release.tag_name }}
+        set
+
+  on-success:
+    runs-on: ubuntu-latest
+    # If the testing workflow has a success, then we build to :latest
+    # Or if we are in a tagged release scenario.
+    if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+      - name: Create release metadata
+        run: |
+          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
+          echo ${{ github.sha }} > changedetectionio/source.txt
+          echo ${{ github.ref }} > changedetectionio/tag.txt
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: all
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to Docker Hub Container Registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+          version: latest
+          driver-opts: image=moby/buildkit:master
+
+      # master always builds :latest
+      - name: Build and push :latest
+        id: docker_build
+        if: ${{ github.ref == 'refs/heads/master'}}
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
+            ghcr.io/${{ github.repository }}:latest
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+
+      # A new tagged release is required, which builds :tag
+      - name: Build and push :tag
+        id: docker_build_tag_release
+        if: ${{ github.event.release.tag_name }} != ''
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
+            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+
+
+      - name: Image digest
+        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+

.github/workflows/image-javascript.yml

@@ -1,88 +0,0 @@
-name: Javascript/Webdriver support - Test, build and push to Docker Hub :javascript tag
-
-on:
-  push:
-    branches: [  javascript-browser ]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install flake8 pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-
-      - name: Create release metadata
-        run: |
-          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
-          echo ${{ github.sha }} > changedetectionio/source.txt
-          echo ${{ github.ref }} > changedetectionio/tag.txt
-
-      - name: Test with pytest
-        run: |
-          # Each test is totally isolated and performs its own cleanup/reset
-          cd changedetectionio; ./run_all_tests.sh
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          image: tonistiigi/binfmt:latest
-          platforms: all
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-          version: latest
-          driver-opts: image=moby/buildkit:master
-
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./
-          file: ./Dockerfile
-          push: true
-          tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:javascript-dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
-
-      - name: Image digest
-        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
-
-# failed: Cache service responded with 503
-#      - name: Cache Docker layers
-#        uses: actions/cache@v2
-#        with:
-#          path: /tmp/.buildx-cache
-#          key: ${{ runner.os }}-buildx-${{ github.sha }}
-#          restore-keys: |
-#            ${{ runner.os }}-buildx-
-
-

.github/workflows/image-tag.yml

@@ -1,92 +0,0 @@
-name: Test, build and push tagged release to Docker Hub
-
-on:
-  push:
-    tags:
-      - '*.*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - uses: olegtarasov/get-tag@v2.1
-        id: tagName
-
-#        with:
-#          tagRegex: "foobar-(.*)"  # Optional. Returns specified group text as tag name. Full tag string is returned if regex is not defined.
-#          tagRegexGroup: 1 # Optional. Default is 1.
-
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install flake8 pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-
-      - name: Create release metadata
-        run: |
-          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
-          echo ${{ github.sha }} > changedetectionio/source.txt
-          echo ${{ github.ref }} > changedetectionio/tag.txt
-
-      - name: Test with pytest
-        run: |
-          # Each test is totally isolated and performs its own cleanup/reset
-          cd changedetectionio; ./run_all_tests.sh
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          image: tonistiigi/binfmt:latest
-          platforms: all
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-          version: latest
-          driver-opts: image=moby/buildkit:master
-
-      - name: tag
-        run : echo ${{ github.event.release.tag_name }}
-
-      - name: Build and push tagged version
-        id: docker_build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./
-          file: ./Dockerfile
-          push: true
-          tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ steps.tagName.outputs.tag }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
-        env:
-            SOURCE_NAME: ${{ steps.branch_name.outputs.SOURCE_NAME }}
-            SOURCE_BRANCH: ${{ steps.branch_name.outputs.SOURCE_BRANCH }}
-            SOURCE_TAG: ${{ steps.branch_name.outputs.SOURCE_TAG }}
-
-      - name: Image digest
-        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}

.github/workflows/image.yml

@@ -1,88 +0,0 @@
-name: Test, build and push to Docker Hub
-
-on:
-  push:
-    branches: [ master, arm-build ]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install flake8 pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-
-      - name: Create release metadata
-        run: |
-          # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py
-          echo ${{ github.sha }} > changedetectionio/source.txt
-          echo ${{ github.ref }} > changedetectionio/tag.txt
-
-      - name: Test with pytest
-        run: |
-          # Each test is totally isolated and performs its own cleanup/reset
-          cd changedetectionio; ./run_all_tests.sh
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          image: tonistiigi/binfmt:latest
-          platforms: all
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-          version: latest
-          driver-opts: image=moby/buildkit:master
-
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./
-          file: ./Dockerfile
-          push: true
-          tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
-
-      - name: Image digest
-        run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
-
-# failed: Cache service responded with 503
-#      - name: Cache Docker layers
-#        uses: actions/cache@v2
-#        with:
-#          path: /tmp/.buildx-cache
-#          key: ${{ runner.os }}-buildx-${{ github.sha }}
-#          restore-keys: |
-#            ${{ runner.os }}-buildx-
-
-

.github/workflows/pypi.yml

@@ -0,0 +1,44 @@
+name: PyPi Test and Push tagged release
+
+# Triggers the workflow on push or pull request events
+on:
+  workflow_run:
+    workflows: ["ChangeDetection.io Test"]
+    tags: '*.*'
+    types: [completed]
+
+
+jobs:
+  test-build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+#      - name: Install dependencies
+#        run: |
+#          python -m pip install --upgrade pip
+#          pip install flake8 pytest
+#          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+#          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+      - name: Test that pip builds without error
+        run: |
+          pip3 --version
+          python3 -m pip install wheel
+          python3 setup.py bdist_wheel
+          python3 -m pip install dist/changedetection.io-*-none-any.whl --force
+          changedetection.io -d /tmp -p 10000 &
+          sleep 3
+          curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+          killall -9 changedetection.io
+
+      # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
+      # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
+
+      # https://github.com/docker/buildx/issues/495#issuecomment-918925854
+#if: ${{ github.event_name == 'release'}}

.github/workflows/test-only.yml

@@ -4,7 +4,7 @@ name: ChangeDetection.io Test
 on: [push, pull_request]
 
 jobs:
-  build:
+  test-build:
     runs-on: ubuntu-latest
     steps:
 
@@ -32,15 +32,8 @@ jobs:
           # Each test is totally isolated and performs its own cleanup/reset
           cd changedetectionio; ./run_all_tests.sh
 
-      - name: Test that pip builds without error
-        run: |
-          pip3 --version
-          python3 -m pip install wheel
-          python3 setup.py bdist_wheel
-          python3 -m pip install dist/changedetection.io-*-none-any.whl --force
-          changedetection.io -d /tmp -p 10000 &
-          sleep 3
-          curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
-          killall -9 changedetection.io
 
+      # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
+      # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
 
+      # https://github.com/docker/buildx/issues/495#issuecomment-918925854

MANIFEST.in

@@ -1,4 +1,6 @@
 recursive-include changedetectionio/templates *
 recursive-include changedetectionio/static *
 include changedetection.py
-global-exclude *.pyc
+global-exclude *.pyc
+global-exclude *node_modules*
+global-exclude venv

README.md

@@ -65,8 +65,23 @@ Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
 
 _Now with per-site configurable support for using a fast built in HTTP fetcher or use a Chrome based fetcher for monitoring of JavaScript websites!_
 
- 
-### Screenshots
+## Updating changedetection.io
+
+### Docker
+```
+docker pull dgtlmoon/changedetection.io
+docker kill $(docker ps -a|grep changedetection.io|awk '{print $1}')
+docker rm $(docker ps -a|grep changedetection.io|awk '{print $1}')
+docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
+```
+
+### docker-compose
+
+```bash
+docker-compose pull && docker-compose up -d
+```
+
+## Screenshots
 
 Examining differences in content.
 

changedetectionio/__init__.py

@@ -28,8 +28,9 @@ from feedgen.feed import FeedGenerator
 from flask import make_response
 import datetime
 import pytz
+from copy import deepcopy
 
-__version__ = '0.39.1'
+__version__ = '0.39.2'
 
 datastore = None
 
@@ -88,8 +89,7 @@ def populate_form_from_watch(form, watch):
         if i[0] != '_':
             p = getattr(form, i)
             if hasattr(p, 'data') and i in watch:
-                if not p.data:
-                    setattr(p, "data", watch[i])
+                setattr(p, "data", watch[i])
 
 
 # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
@@ -269,11 +269,14 @@ def changedetection_app(config=None, datastore_o=None):
 
             for watch in sorted_watches:
                 if not watch['viewed']:
+                    # Re #239 - GUID needs to be individual for each event
+                    # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228)
+                    guid = "{}/{}".format(watch['uuid'], watch['last_changed'])
                     fe = fg.add_entry()
                     fe.title(watch['url'])
                     fe.link(href=watch['url'])
                     fe.description(watch['url'])
-                    fe.guid(watch['uuid'], permalink=False)
+                    fe.guid(guid, permalink=False)
                     dt = datetime.datetime.fromtimestamp(int(watch['newest_history_key']))
                     dt = dt.replace(tzinfo=pytz.UTC)
                     fe.pubDate(dt)
@@ -411,7 +414,9 @@ def changedetection_app(config=None, datastore_o=None):
                           'fetch_backend': form.fetch_backend.data,
                           'trigger_text': form.trigger_text.data,
                           'notification_title': form.notification_title.data,
-                          'notification_body': form.notification_body.data
+                          'notification_body': form.notification_body.data,
+                          'extract_title_as_title': form.extract_title_as_title.data
+
                           }
 
             # Notification URLs
@@ -434,9 +439,8 @@ def changedetection_app(config=None, datastore_o=None):
                 if len(datastore.data['watching'][uuid]['history']):
                     update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
-
             datastore.data['watching'][uuid].update(update_obj)
-            datastore.needs_write = True
+
             flash("Updated watch.")
 
             # Queue the watch for immediate recheck
@@ -493,6 +497,7 @@ def changedetection_app(config=None, datastore_o=None):
             form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
             form.notification_title.data = datastore.data['settings']['application']['notification_title']
             form.notification_body.data = datastore.data['settings']['application']['notification_body']
+            form.base_url.data = datastore.data['settings']['application']['base_url']
 
             # Password unset is a GET
             if request.values.get('removepassword') == 'yes':
@@ -510,9 +515,8 @@ def changedetection_app(config=None, datastore_o=None):
             datastore.data['settings']['application']['fetch_backend'] = form.fetch_backend.data
             datastore.data['settings']['application']['notification_title'] = form.notification_title.data
             datastore.data['settings']['application']['notification_body'] = form.notification_body.data
-
             datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
-            datastore.needs_write = True
+            datastore.data['settings']['application']['base_url'] = form.base_url.data
 
             if form.trigger_check.data and len(form.notification_urls.data):
                 n_object = {'watch_url': "Test from changedetection.io!",
@@ -529,14 +533,13 @@ def changedetection_app(config=None, datastore_o=None):
                 flask_login.logout_user()
                 return redirect(url_for('index'))
 
+            datastore.needs_write = True
             flash("Settings updated.")
 
         if request.method == 'POST' and not form.validate():
             flash("An error occurred, please see below.", "error")
 
-        # Same as notification.py
-        base_url = os.getenv('BASE_URL', '').strip('"')
-        output = render_template("settings.html", form=form, base_url=base_url)
+        output = render_template("settings.html", form=form)
 
         return output
 
@@ -894,15 +897,19 @@ def ticker_thread_check_time_launch_checks():
             if t.current_uuid:
                 running_uuids.append(t.current_uuid)
 
+        # Re #232 - Deepcopy the data incase it changes while we're iterating through it all
+        copied_datastore = deepcopy(datastore)
+
         # Check for watches outside of the time threshold to put in the thread queue.
-        for uuid, watch in datastore.data['watching'].items():
+        for uuid, watch in copied_datastore.data['watching'].items():
 
             # If they supplied an individual entry minutes to threshold.
             if 'minutes_between_check' in watch and watch['minutes_between_check'] is not None:
-                max_time = watch['minutes_between_check'] * 60
+                # Cast to int just incase
+                max_time = int(watch['minutes_between_check']) * 60
             else:
                 # Default system wide.
-                max_time = datastore.data['settings']['requests']['minutes_between_check'] * 60
+                max_time = int(copied_datastore.data['settings']['requests']['minutes_between_check']) * 60
 
             threshold = time.time() - max_time
 

changedetectionio/fetch_site_status.py

@@ -170,9 +170,10 @@ class perform_site_check():
                 update_obj["previous_md5"] = fetched_md5
 
             # Extract title as title
-            if is_html and self.datastore.data['settings']['application']['extract_title_as_title']:
-                if not watch['title'] or not len(watch['title']):
-                    update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
+            if is_html:
+                if self.datastore.data['settings']['application']['extract_title_as_title'] or watch['extract_title_as_title']:
+                    if not watch['title'] or not len(watch['title']):
+                        update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
 
 
         return changed_detected, update_obj, stripped_text_from_html

changedetectionio/forms.py

@@ -203,6 +203,7 @@ class commonSettingsForm(Form):
     notification_body = TextAreaField('Notification Body', default='{watch_url} had a change.', validators=[validators.Optional(), ValidateTokensList()])
     trigger_check = BooleanField('Send test notification on save')
     fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+    extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
 
 class watchForm(commonSettingsForm):
 
@@ -222,7 +223,7 @@ class watchForm(commonSettingsForm):
 class globalSettingsForm(commonSettingsForm):
 
     password = SaltyPasswordField()
-
     minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
                                                [validators.NumberRange(min=1)])
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title')
+    base_url = StringField('Base URL', validators=[validators.Optional()])

changedetectionio/notification.py

@@ -57,7 +57,8 @@ def create_notification_parameters(n_object, datastore):
         watch_tag = ''
 
     # Create URLs to customise the notification with
-    base_url = os.getenv('BASE_URL', '').strip('"')
+    base_url = datastore.data['settings']['application']['base_url']
+
     watch_url = n_object['watch_url']
 
     # Re #148 - Some people have just {base_url} in the body or title, but this may break some notification services

changedetectionio/run_all_tests.sh

@@ -9,11 +9,15 @@
 # exit when any command fails
 set -e
 
-# Re #65 - Ability to include a link back to the installation, in the notification.
-export BASE_URL="https://foobar.com"
 
 find tests/test_*py -type f|while read test_name
 do
   echo "TEST RUNNING $test_name"
   pytest $test_name
 done
+
+
+# Now re-run some tests with BASE_URL enabled
+# Re #65 - Ability to include a link back to the installation, in the notification.
+export BASE_URL="https://really-unique-domain.io"
+pytest tests/test_notification.py

changedetectionio/store.py

@@ -7,7 +7,7 @@ from copy import deepcopy
 import logging
 import time
 import threading
-
+import os
 
 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
 # Open a github issue if you know something :)
@@ -38,6 +38,7 @@ class ChangeDetectionStore:
                 },
                 'application': {
                     'password': False,
+                    'base_url' : None,
                     'extract_title_as_title': False,
                     'fetch_backend': 'html_requests',
                     'notification_urls': [], # Apprise URL list
@@ -73,6 +74,7 @@ class ChangeDetectionStore:
             'css_filter': "",
             'trigger_text': [],  # List of text or regex to wait for until a change is detected
             'fetch_backend': None,
+            'extract_title_as_title': False
         }
 
         if path.isfile('changedetectionio/source.txt'):
@@ -196,13 +198,15 @@ class ChangeDetectionStore:
                 has_unviewed = True
 
             # #106 - Be sure this is None on empty string, False, None, etc
-            if not self.__data['watching'][uuid]['title']:
-                self.__data['watching'][uuid]['title'] = None
-
             # Default var for fetch_backend
             if not self.__data['watching'][uuid]['fetch_backend']:
                 self.__data['watching'][uuid]['fetch_backend'] = self.__data['settings']['application']['fetch_backend']
 
+        # Re #152, Return env base_url if not overriden, @todo also prefer the proxy pass url
+        env_base_url = os.getenv('BASE_URL','')
+        if self.__data['settings']['application']['base_url'] is None and len(env_base_url) >0:
+            self.__data['settings']['application']['base_url'] = env_base_url.strip('" ')
+
         self.__data['has_unviewed'] = has_unviewed
 
         return self.__data

changedetectionio/templates/_common_fields.jinja

@@ -3,9 +3,6 @@
 
 {% macro render_notifications_field(form) %}
 
-
-<fieldset>
-                    <div class="field-group">
                         <div class="pure-control-group">
                             {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
     Gitter - gitter://token/room
@@ -84,7 +81,4 @@
                         <div class="pure-control-group">
                             {{ render_field(form.trigger_check) }}
                         </div>
-                    </div>
-
-                </fieldset>
-{% endmacro %}
+{% endmacro %}

changedetectionio/templates/edit.html

@@ -56,12 +56,19 @@ User-Agent: wonderbra 1.0") }}
                             <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server. </p>
                         </span>
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.extract_title_as_title) }}
+                    </div>
                 </fieldset>
             </div>
 
             <div class="tab-pane-inner" id="notifications">
                 <strong>Note: <i>These settings override the global settings.</i></strong>
-                {{ render_notifications_field(form) }}
+                <fieldset>
+                    <div class="field-group">
+                        {{ render_notifications_field(form) }}
+                    </div>
+                </fieldset>
             </div>
 
             <div class="tab-pane-inner" id="filters">

changedetectionio/templates/settings.html

@@ -32,6 +32,15 @@
                         <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
                         {% endif %}
                     </div>
+                    <div class="pure-control-group">
+                        {{ render_field(form.base_url, placeholder="http://yoursite.com:5000/",
+                        class="m-d") }}
+                        <span class="pure-form-message-inline">
+                            Base URL used for the {base_url} token in notifications, default value is the ENV var 'base_url',
+                            <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
+                        </span>
+                    </div>
+
                     <div class="pure-control-group">
                         {{ render_field(form.extract_title_as_title) }}
                         <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
@@ -40,7 +49,11 @@
             </div>
 
             <div class="tab-pane-inner" id="notifications">
-                {{ render_notifications_field(form) }}
+                <fieldset>
+                    <div class="field-group">
+                        {{ render_notifications_field(form) }}
+                    </div>
+                </fieldset>
             </div>
 
             <div class="tab-pane-inner" id="fetching">

changedetectionio/tests/conftest.py

@@ -36,11 +36,8 @@ def app(request):
     except FileExistsError:
         pass
 
-    # Enable a BASE_URL for notifications to work (so we can look for diff/ etc URLs)
-    os.environ["BASE_URL"] = "http://mysite.com/"
     cleanup(datastore_path)
 
-
     app_config = {'datastore_path': datastore_path}
     cleanup(app_config['datastore_path'])
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)

changedetectionio/tests/test_notification.py

@@ -3,6 +3,7 @@ import time
 import re
 from flask import url_for
 from . util import set_original_response, set_modified_response, live_server_setup
+import logging
 
 # Hard to just add more live server URLs when one test is already running (I think)
 # So we add our test here (was in a different file)
@@ -14,14 +15,24 @@ def test_check_notification(client, live_server):
     # Give the endpoint time to spin up
     time.sleep(3)
 
+    # When test mode is in BASE_URL env mode, we should see this already configured
+    env_base_url = os.getenv('BASE_URL', '').strip()
+    if len(env_base_url):
+        logging.debug(">>> BASE_URL enabled, looking for %s", env_base_url)
+        res = client.get(url_for("settings_page"))
+        assert bytes(env_base_url.encode('utf-8')) in res.data
+    else:
+        logging.debug(">>> SKIPPING BASE_URL check")
+
+    # re #242 - when you edited an existing new entry, it would not correctly show the notification settings
     # Add our URL to the import page
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
+        url_for("api_watch_add"),
+        data={"url": test_url, "tag": ''},
         follow_redirects=True
     )
-    assert b"1 Imported" in res.data
+    assert b"Watch added" in res.data
 
     # Give the thread time to pick up the first version
     time.sleep(3)
@@ -61,6 +72,10 @@ def test_check_notification(client, live_server):
         url_for("edit_page", uuid="first"))
     assert bytes(notification_url.encode('utf-8')) in res.data
 
+    # Re #242 - wasnt saving?
+    assert bytes("New ChangeDetection.io Notification".encode('utf-8')) in res.data
+
+
 
     # Because we hit 'send test notification on save'
     time.sleep(3)
@@ -98,9 +113,12 @@ def test_check_notification(client, live_server):
 
     assert test_url in notification_submission
 
-        # Re #65 - did we see our foobar.com BASE_URL ?
-        #assert bytes("https://foobar.com".encode('utf-8')) in notification_submission
-
+    if env_base_url:
+        # Re #65 - did we see our BASE_URl ?
+        logging.debug (">>> BASE_URL checking in notification: %s", env_base_url)
+        assert env_base_url in notification_submission
+    else:
+        logging.debug(">>> Skipping BASE_URL check")
 
     ##  Now configure something clever, we go into custom config (non-default) mode, this is returned by the endpoint
     with open("test-datastore/endpoint-content.txt", "w") as f:

requirements.txt

@@ -16,6 +16,10 @@ jsonpath-ng ~= 1.5.3
 # Notification library
 apprise ~= 0.9
 
+# Pinned version of cryptography otherwise
+# ERROR: Could not build wheels for cryptography which use PEP 517 and cannot be installed directly
+cryptography ~= 3.4
+
 # Used for CSS filtering, replace with soupsieve and lxml for xpath
 bs4