0.45.2

Code/Test - Improve testing for creating backups
UI - Re-order notification field settings
2025-10-31 14:47:21 +00:00 · 2023-09-22 09:45:55 +02:00 · 2023-09-22 09:21:07 +02:00 · 2023-09-14 14:34:44 +02:00 · 2023-09-14 14:16:08 +02:00 · 2023-09-14 13:55:48 +02:00
58 changed files with 1444 additions and 289 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,2 +1,18 @@
 .git
 .github
+changedetectionio/processors/__pycache__
+changedetectionio/api/__pycache__
+changedetectionio/model/__pycache__
+changedetectionio/blueprint/price_data_follower/__pycache__
+changedetectionio/blueprint/tags/__pycache__
+changedetectionio/blueprint/__pycache__
+changedetectionio/blueprint/browser_steps/__pycache__
+changedetectionio/fetchers/__pycache__
+changedetectionio/tests/visualselector/__pycache__
+changedetectionio/tests/restock/__pycache__
+changedetectionio/tests/__pycache__
+changedetectionio/tests/fetchers/__pycache__
+changedetectionio/tests/unit/__pycache__
+changedetectionio/tests/proxy_list/__pycache__
+changedetectionio/__pycache__
+
--- a/.github/workflows/containers.yml
+++ b/.github/workflows/containers.yml
@@ -95,7 +95,7 @@ jobs:
          push: true
          tags: |
            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:dev,ghcr.io/${{ github.repository }}:dev
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,dest=/tmp/.buildx-cache
 # Looks like this was disabled
@@ -115,7 +115,7 @@ jobs:
            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
            ghcr.io/dgtlmoon/changedetection.io:latest
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,dest=/tmp/.buildx-cache
 # Looks like this was disabled
--- a/.github/workflows/test-container-build.yml
+++ b/.github/workflows/test-container-build.yml
@@ -62,7 +62,7 @@ jobs:
          with:
            context: ./
            file: ./Dockerfile
-            platforms: linux/arm/v7,linux/arm/v6,linux/amd64,linux/arm64,
+            platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/arm/v8
            cache-from: type=local,src=/tmp/.buildx-cache
            cache-to: type=local,dest=/tmp/.buildx-cache

--- a/.github/workflows/test-only.yml
+++ b/.github/workflows/test-only.yml
@@ -36,10 +36,16 @@ jobs:
        run: |         
          # Build a changedetection.io container and start testing inside
          docker build . -t test-changedetectionio
+          # Debug info
+          docker run test-changedetectionio  bash -c 'pip list'
+
+      - name: Spin up ancillary SMTP+Echo message test server
+        run: |
+          # Debug SMTP server/echo message back server
+          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py' 

      - name: Test built container with pytest
        run: |
-          
          # Unit tests
          docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
          
@@ -58,11 +64,16 @@ jobs:
          # Settings headers playwright tests - Call back in from Browserless, check headers
          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'          
          
          # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it
          docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'

+      - name: Test SMTP notification mime types
+        run: |
+          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
+          docker run --rm  --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
+
      - name: Test with puppeteer fetcher and disk cache
        run: |
          docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -13,3 +13,6 @@ include changedetection.py
 global-exclude *.pyc
 global-exclude node_modules
 global-exclude venv
+
+global-exclude test-datastore
+global-exclude changedetection.io*dist-info
--- a/README-pip.md
+++ b/README-pip.md
@@ -54,5 +54,5 @@ $ changedetection.io -d /path/to/empty/data/dir -p 5000

 Then visit http://127.0.0.1:5000 , You should now be able to access the UI.

-See https://github.com/dgtlmoon/changedetection.io for more information.
+See https://changedetection.io for more information.

--- a/README.md
+++ b/README.md
@@ -67,13 +67,14 @@ Requires Playwright to be enabled.
 - Get alerts when new job positions are open on Bamboo HR and other job platforms
 - Website defacement monitoring
 - Pokémon Card Restock Tracker / Pokémon TCG Tracker
+- RegTech - stay ahead of regulatory changes, regulatory compliance

 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_

 #### Key Features

 - Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
- Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
+- Target elements with xPath(1.0) and CSS Selectors, Easily monitor complex JSON with JSONPath or jq
 - Switch between fast non-JS and Chrome JS based "fetchers"
 - Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)
 - Easily specify how often a site should be checked
@@ -85,6 +86,8 @@ _Need an actual Chrome runner with Javascript support? We support fetching via W

 We [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.

+[Oxylabs](https://oxylabs.go2cloud.org/SH2d) is also an excellent proxy provider and well worth using, they offer Residental, ISP, Rotating and many other proxy types to suit your project. 
+
 Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/

 ## Installation
@@ -144,7 +147,7 @@ See the wiki for more information https://github.com/dgtlmoon/changedetection.io

 ## Filters

-XPath, JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. 
+XPath(1.0), JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. 
 (We support LXML `re:test`, `re:match` and `re:replace`.)

 ## Notifications
@@ -183,7 +186,7 @@ This will re-parse the JSON and apply formatting to the text, making it super ea

 ### JSONPath or jq?

-For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more specifc information on jq.
+For more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more specific information on jq.

 One big advantage of `jq` is that you can use logic in your JSON filter, such as filters to only show items that have a value greater than/less than etc.

@@ -223,7 +226,7 @@ The application also supports notifying you that it can follow this information

 ## Proxy Configuration

-See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [BrightData proxy services where possible]( https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support)
+See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [Bright Data proxy services where possible]( https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support)

 ## Raspberry Pi support?

--- a/changedetectionio/init.py
+++ b/changedetectionio/init.py
@@ -38,7 +38,9 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1

-__version__ = '0.43'
+__version__ = '0.45.2'
+
+from changedetectionio.store import BASE_URL_NOT_SET_TEXT

 datastore = None

@@ -355,11 +357,11 @@ def changedetection_app(config=None, datastore_o=None):

                # Include a link to the diff page, they will have to login here to see if password protection is enabled.
                # Description is the page you watch, link takes you to the diff JS UI page
-                base_url = datastore.data['settings']['application']['base_url']
-                if base_url == '':
-                    base_url = "<base-url-env-var-not-set>"
+                # Dict val base_url will get overriden with the env var if it is set.
+                ext_base_url = datastore.data['settings']['application'].get('active_base_url')

-                diff_link = {'href': "{}{}".format(base_url, url_for('diff_history_page', uuid=watch['uuid']))}
+                # Because we are called via whatever web server, flask should figure out the right path (
+                diff_link = {'href': url_for('diff_history_page', uuid=watch['uuid'], _external=True)}

                fe.link(link=diff_link)

@@ -712,7 +714,6 @@ def changedetection_app(config=None, datastore_o=None):
            output = render_template("edit.html",
                                     available_processors=processors.available_processors(),
                                     browser_steps_config=browser_step_ui_config,
-                                     current_base_url=datastore.data['settings']['application']['base_url'],
                                     emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
                                     form=form,
                                     has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False,
@@ -802,7 +803,6 @@ def changedetection_app(config=None, datastore_o=None):

        output = render_template("settings.html",
                                 form=form,
-                                 current_base_url = datastore.data['settings']['application']['base_url'],
                                 hide_remove_pass=os.getenv("SALTED_PASS", False),
                                 api_key=datastore.data['settings']['application'].get('api_access_token'),
                                 emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
@@ -1268,10 +1268,10 @@ def changedetection_app(config=None, datastore_o=None):
                update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
            i = 1

-        elif tag != None:
+        elif tag:
            # Items that have this current tag
            for watch_uuid, watch in datastore.data['watching'].items():
-                if (tag != None and tag in watch.get('tags', {})):
+                if tag in watch.get('tags', {}):
                    if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                        update_q.put(
                            queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False})
@@ -1430,6 +1430,27 @@ def changedetection_app(config=None, datastore_o=None):
        # paste in etc
        return redirect(url_for('index'))

+    @app.route("/highlight_submit_ignore_url", methods=['POST'])
+    def highlight_submit_ignore_url():
+        import re
+        mode = request.form.get('mode')
+        selection = request.form.get('selection')
+
+        uuid = request.args.get('uuid','')
+        if datastore.data["watching"].get(uuid):
+            if mode == 'exact':
+                for l in selection.splitlines():
+                    datastore.data["watching"][uuid]['ignore_text'].append(l.strip())
+            elif mode == 'digit-regex':
+                for l in selection.splitlines():
+                    # Replace any series of numbers with a regex
+                    s = re.escape(l.strip())
+                    s = re.sub(r'[0-9]+', r'\\d+', s)
+                    datastore.data["watching"][uuid]['ignore_text'].append('/' + s + '/')
+
+        return f"<a href={url_for('preview_page', uuid=uuid)}>Click to preview</a>"
+
+
    import changedetectionio.blueprint.browser_steps as browser_steps
    app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')

@@ -1439,6 +1460,10 @@ def changedetection_app(config=None, datastore_o=None):
    import changedetectionio.blueprint.tags as tags
    app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags')

+    import changedetectionio.blueprint.check_proxies as check_proxies
+    app.register_blueprint(check_proxies.construct_blueprint(datastore=datastore), url_prefix='/check_proxy')
+
+
    # @todo handle ctrl break
    ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
    threading.Thread(target=notification_runner).start()
--- a/changedetectionio/api/api_v1.py
+++ b/changedetectionio/api/api_v1.py
@@ -1,3 +1,6 @@
+import os
+from distutils.util import strtobool
+
 from flask_expects_json import expects_json
 from changedetectionio import queuedWatchMetaData
 from flask_restful import abort, Resource
@@ -33,7 +36,7 @@ class Watch(Resource):
    @auth.check_token
    def get(self, uuid):
        """
-        @api {get} /api/v1/watch/:uuid Get a single watch data
+        @api {get} /api/v1/watch/:uuid Single watch - get data, recheck, pause, mute.
        @apiDescription Retrieve watch information and set muted/paused status
        @apiExample {curl} Example usage:
            curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
@@ -209,7 +212,9 @@ class CreateWatch(Resource):
        json_data = request.get_json()
        url = json_data['url'].strip()

-        if not validators.url(json_data['url'].strip()):
+        # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+        if not validators.url(url, simple_host=allow_simplehost):
            return "Invalid or unsupported URL", 400

        if json_data.get('proxy'):
@@ -219,13 +224,15 @@ class CreateWatch(Resource):

        extras = copy.deepcopy(json_data)

-        # Because we renamed 'tag' to 'tags' but dont want to change the API (can do this in v2 of the API)
+        # Because we renamed 'tag' to 'tags' but don't want to change the API (can do this in v2 of the API)
+        tags = None
        if extras.get('tag'):
-            extras['tags'] = extras.get('tag')
+            tags = extras.get('tag')
+            del extras['tag']

        del extras['url']

-        new_uuid = self.datastore.add_watch(url=url, extras=extras)
+        new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags)
        if new_uuid:
            self.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
            return {'uuid': new_uuid}, 201
--- a/changedetectionio/blueprint/check_proxies/init.py
+++ b/changedetectionio/blueprint/check_proxies/init.py
@@ -0,0 +1,116 @@
+from concurrent.futures import ThreadPoolExecutor
+
+from functools import wraps
+
+from flask import Blueprint
+from flask_login import login_required
+
+from changedetectionio.processors import text_json_diff
+from changedetectionio.store import ChangeDetectionStore
+
+
+STATUS_CHECKING = 0
+STATUS_FAILED = 1
+STATUS_OK = 2
+THREADPOOL_MAX_WORKERS = 3
+_DEFAULT_POOL = ThreadPoolExecutor(max_workers=THREADPOOL_MAX_WORKERS)
+
+
+# Maybe use fetch-time if its >5 to show some expected load time?
+def threadpool(f, executor=None):
+    @wraps(f)
+    def wrap(*args, **kwargs):
+        return (executor or _DEFAULT_POOL).submit(f, *args, **kwargs)
+
+    return wrap
+
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    check_proxies_blueprint = Blueprint('check_proxies', __name__)
+    checks_in_progress = {}
+
+    @threadpool
+    def long_task(uuid, preferred_proxy):
+        import time
+        from changedetectionio import content_fetcher
+
+        status = {'status': '', 'length': 0, 'text': ''}
+        from jinja2 import Environment, BaseLoader
+
+        contents = ''
+        now = time.time()
+        try:
+            update_handler = text_json_diff.perform_site_check(datastore=datastore)
+            changed_detected, update_obj, contents = update_handler.run(uuid, preferred_proxy=preferred_proxy, skip_when_checksum_same=False)
+        # title, size is len contents not len xfer
+        except content_fetcher.Non200ErrorCodeReceived as e:
+            if e.status_code == 404:
+                status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but 404 (page not found)"})
+            elif e.status_code == 403 or e.status_code == 401:
+                status.update({'status': 'ERROR', 'length': len(contents), 'text': f"{e.status_code} - Access denied"})
+            else:
+                status.update({'status': 'ERROR', 'length': len(contents), 'text': f"Status code: {e.status_code}"})
+        except text_json_diff.FilterNotFoundInResponse:
+            status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but CSS/xPath filter not found (page changed layout?)"})
+        except content_fetcher.EmptyReply as e:
+            if e.status_code == 403 or e.status_code == 401:
+                status.update({'status': 'ERROR OTHER', 'length': len(contents), 'text': f"Got empty reply with code {e.status_code} - Access denied"})
+            else:
+                status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': f"Empty reply with code {e.status_code}, needs chrome?"})
+
+        except Exception as e:
+            status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': 'Error: '+str(e)})
+        else:
+            status.update({'status': 'OK', 'length': len(contents), 'text': ''})
+
+        if status.get('text'):
+            status['text'] = Environment(loader=BaseLoader()).from_string('{{text|e}}').render({'text': status['text']})
+
+        status['time'] = "{:.2f}s".format(time.time() - now)
+
+        return status
+
+    def _recalc_check_status(uuid):
+
+        results = {}
+        for k, v in checks_in_progress.get(uuid, {}).items():
+            try:
+                r_1 = v.result(timeout=0.05)
+            except Exception as e:
+                # If timeout error?
+                results[k] = {'status': 'RUNNING'}
+
+            else:
+                results[k] = r_1
+
+        return results
+
+    @login_required
+    @check_proxies_blueprint.route("/<string:uuid>/status", methods=['GET'])
+    def get_recheck_status(uuid):
+        results = _recalc_check_status(uuid=uuid)
+        return results
+
+    @login_required
+    @check_proxies_blueprint.route("/<string:uuid>/start", methods=['GET'])
+    def start_check(uuid):
+
+        if not datastore.proxy_list:
+            return
+
+        if checks_in_progress.get(uuid):
+            state = _recalc_check_status(uuid=uuid)
+            for proxy_key, v in state.items():
+                if v.get('status') == 'RUNNING':
+                    return state
+        else:
+            checks_in_progress[uuid] = {}
+
+        for k, v in datastore.proxy_list.items():
+            if not checks_in_progress[uuid].get(k):
+                checks_in_progress[uuid][k] = long_task(uuid=uuid, preferred_proxy=k)
+
+        results = _recalc_check_status(uuid=uuid)
+        return results
+
+    return check_proxies_blueprint
--- a/changedetectionio/blueprint/tags/init.py
+++ b/changedetectionio/blueprint/tags/init.py
@@ -76,6 +76,16 @@ def construct_blueprint(datastore: ChangeDetectionStore):
        flash(f"Tag unlinked removed from {unlinked} watches")
        return redirect(url_for('tags.tags_overview_page'))

+    @tags_blueprint.route("/delete_all", methods=['GET'])
+    @login_optionally_required
+    def delete_all():
+        for watch_uuid, watch in datastore.data['watching'].items():
+            watch['tags'] = []
+        datastore.data['settings']['application']['tags'] = {}
+
+        flash(f"All tags deleted")
+        return redirect(url_for('tags.tags_overview_page'))
+
    @tags_blueprint.route("/edit/<string:uuid>", methods=['GET'])
    @login_optionally_required
    def form_tag_edit(uuid):
--- a/changedetectionio/blueprint/tags/templates/edit-tag.html
+++ b/changedetectionio/blueprint/tags/templates/edit-tag.html
@@ -2,6 +2,10 @@
 {% block content %}
 {% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
+<script>
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+</script>
+
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script>

--- a/changedetectionio/content_fetcher.py
+++ b/changedetectionio/content_fetcher.py
@@ -201,7 +201,8 @@ class Fetcher():
            dest = os.path.join(self.browser_steps_screenshot_path, 'step_*.jpeg')
            files = glob.glob(dest)
            for f in files:
-                os.unlink(f)
+                if os.path.isfile(f):
+                    os.unlink(f)


 #   Maybe for the future, each fetcher provides its own diff output, could be used for text, image
--- a/changedetectionio/diff.py
+++ b/changedetectionio/diff.py
@@ -35,15 +35,19 @@ def customSequenceMatcher(before, after, include_equal=False, include_removed=Tr

 # only_differences - only return info about the differences, no context
 # line_feed_sep could be "<br>" or "<li>" or "\n" etc
-def render_diff(previous_version_file_contents, newest_version_file_contents, include_equal=False, include_removed=True, include_added=True, include_replaced=True, line_feed_sep="\n", include_change_type_prefix=True):
+def render_diff(previous_version_file_contents, newest_version_file_contents, include_equal=False, include_removed=True, include_added=True, include_replaced=True, line_feed_sep="\n", include_change_type_prefix=True, patch_format=False):

    newest_version_file_contents = [line.rstrip() for line in newest_version_file_contents.splitlines()]

    if previous_version_file_contents:
-            previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
+        previous_version_file_contents = [line.rstrip() for line in previous_version_file_contents.splitlines()]
    else:
        previous_version_file_contents = ""

+    if patch_format:
+        patch = difflib.unified_diff(previous_version_file_contents, newest_version_file_contents)
+        return line_feed_sep.join(patch)
+
    rendered_diff = customSequenceMatcher(before=previous_version_file_contents,
                                          after=newest_version_file_contents,
                                          include_equal=include_equal,
@@ -54,4 +58,5 @@ def render_diff(previous_version_file_contents, newest_version_file_contents, in

    # Recursively join lists
    f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
-    return f(rendered_diff)
+    p= f(rendered_diff)
+    return p
--- a/changedetectionio/forms.py
+++ b/changedetectionio/forms.py
@@ -1,5 +1,6 @@
 import os
 import re
+from distutils.util import strtobool

 from wtforms import (
    BooleanField,
@@ -257,9 +258,10 @@ class validateURL(object):

    def __call__(self, form, field):
        import validators
-
+        # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
        try:
-            validators.url(field.data.strip())
+            validators.url(field.data.strip(), simple_host=allow_simplehost)
        except validators.ValidationFailure:
            message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
            raise ValidationError(message)
@@ -500,7 +502,10 @@ class globalSettingsRequestForm(Form):
 class globalSettingsApplicationForm(commonSettingsForm):

    api_access_token_enabled = BooleanField('API access token security check enabled', default=True, validators=[validators.Optional()])
-    base_url = StringField('Base URL', validators=[validators.Optional()])
+    base_url = StringField('Notification base URL override',
+                           validators=[validators.Optional()],
+                           render_kw={"placeholder": os.getenv('BASE_URL', 'Not set')}
+                           )
    empty_pages_are_a_change =  BooleanField('Treat empty pages as a change?', default=False)
    fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
    global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
--- a/changedetectionio/html_tools.py
+++ b/changedetectionio/html_tools.py
@@ -191,42 +191,50 @@ def extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None
 #
 # wordlist - list of regex's (str) or words (str)
 def strip_ignore_text(content, wordlist, mode="content"):
-    ignore = []
-    ignore_regex = []
-
-    # @todo check this runs case insensitive
-    for k in wordlist:
-
-        # Is it a regex?
-        if k[0] == '/':
-            ignore_regex.append(k.strip(" /"))
-        else:
-            ignore.append(k)
-
    i = 0
    output = []
+    ignore_text = []
+    ignore_regex = []
+
    ignored_line_numbers = []
+
+    for k in wordlist:
+        # Is it a regex?
+        x = re.search('^\/(.*)\/(.*)', k.strip())
+        if x:
+            # Starts with / but doesn't look like a regex
+            p = x.group(1)
+            try:
+                # @Todo python regex options can go before the regex str, but not really many of the options apply on a per-line basis
+                ignore_regex.append(re.compile(rf"{p}", re.IGNORECASE))
+            except Exception as e:
+                # Badly formed regex, treat as text
+                ignore_text.append(k.strip())
+        else:
+            # Had a / but doesn't work as regex
+            ignore_text.append(k.strip())
+
    for line in content.splitlines():
        i += 1
        # Always ignore blank lines in this mode. (when this function gets called)
+        got_match = False
        if len(line.strip()):
-            regex_matches = False
+            for l in ignore_text:
+                if l.lower() in line.lower():
+                    got_match = True

-            # if any of these match, skip
-            for regex in ignore_regex:
-                try:
-                    if re.search(regex, line, re.IGNORECASE):
-                        regex_matches = True
-                except Exception as e:
-                    continue
+            if not got_match:
+                for r in ignore_regex:
+                    if r.search(line):
+                        got_match = True

-            if not regex_matches and not any(skip_text.lower() in line.lower() for skip_text in ignore):
+            if not got_match:
+                # Not ignored
                output.append(line.encode('utf8'))
            else:
                ignored_line_numbers.append(i)


-
    # Used for finding out what to highlight
    if mode == "line numbers":
        return ignored_line_numbers
--- a/changedetectionio/importer.py
+++ b/changedetectionio/importer.py
@@ -85,7 +85,8 @@ class import_distill_io_json(Importer):
        now = time.time()
        self.new_uuids=[]

-
+        # @todo Use JSONSchema like in the API to validate here.
+        
        try:
            data = json.loads(data.strip())
        except json.decoder.JSONDecodeError:
@@ -120,11 +121,8 @@ class import_distill_io_json(Importer):
                except IndexError:
                    pass

-# Does this need to be here anymore?
-                if d.get('tags', False):
-                    extras['tags'] = ", ".join(d['tags'])
-
                new_uuid = datastore.add_watch(url=d['uri'].strip(),
+                                               tag=",".join(d.get('tags', [])),
                                               extras=extras,
                                               write_to_disk_now=False)

--- a/changedetectionio/notification.py
+++ b/changedetectionio/notification.py
@@ -9,6 +9,7 @@ valid_tokens = {
    'diff': '',
    'diff_added': '',
    'diff_full': '',
+    'diff_patch': '',
    'diff_removed': '',
    'diff_url': '',
    'preview_url': '',
@@ -92,7 +93,13 @@ def process_notification(n_object, datastore):
        n_object.get('notification_format', default_notification_format),
        valid_notification_formats[default_notification_format],
    )
-    
+
+    # If we arrived with 'System default' then look it up
+    if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
+        # Initially text or whatever
+        n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
+
+
    # https://github.com/caronc/apprise/wiki/Development_LogCapture
    # Anything higher than or equal to WARNING (which covers things like Connection errors)
    # raise it as an exception
@@ -145,9 +152,12 @@ def process_notification(n_object, datastore):
                    # Apprise will default to HTML, so we need to override it
                    # So that whats' generated in n_body is in line with what is going to be sent.
                    # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-                    if not 'format=' in url and (n_format == 'text' or n_format == 'markdown'):
+                    if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
                        prefix = '?' if not '?' in url else '&'
+                        # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
+                        n_format = n_format.tolower()
                        url = "{}{}format={}".format(url, prefix, n_format)
+                    # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only

                apobj.add(url)

@@ -168,7 +178,7 @@ def process_notification(n_object, datastore):
                log_value = logs.getvalue()
                if log_value and 'WARNING' in log_value or 'ERROR' in log_value:
                    raise Exception(log_value)
-                
+
                sent_objs.append({'title': n_title,
                                  'body': n_body,
                                  'url' : url,
@@ -198,15 +208,11 @@ def create_notification_parameters(n_object, datastore):
        watch_tag = ''

    # Create URLs to customise the notification with
-    base_url = datastore.data['settings']['application']['base_url']
+    # active_base_url - set in store.py data property
+    base_url = datastore.data['settings']['application'].get('active_base_url')

    watch_url = n_object['watch_url']

-    # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services
-    #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.
-    if base_url == '':
-        base_url = "<base-url-env-var-not-set>"
-
    diff_url = "{}/diff/{}".format(base_url, uuid)
    preview_url = "{}/preview/{}".format(base_url, uuid)

@@ -216,11 +222,12 @@ def create_notification_parameters(n_object, datastore):
    # Valid_tokens also used as a field validator
    tokens.update(
        {
-            'base_url': base_url if base_url is not None else '',
+            'base_url': base_url,
            'current_snapshot': n_object['current_snapshot'] if 'current_snapshot' in n_object else '',
            'diff': n_object.get('diff', ''),  # Null default in the case we use a test
            'diff_added': n_object.get('diff_added', ''),  # Null default in the case we use a test
            'diff_full': n_object.get('diff_full', ''),  # Null default in the case we use a test
+            'diff_patch': n_object.get('diff_patch', ''),  # Null default in the case we use a test
            'diff_removed': n_object.get('diff_removed', ''),  # Null default in the case we use a test
            'diff_url': diff_url,
            'preview_url': preview_url,
--- a/changedetectionio/processors/init.py
+++ b/changedetectionio/processors/init.py
@@ -9,7 +9,7 @@ class difference_detection_processor():
        super().__init__(*args, **kwargs)

    @abstractmethod
-    def run(self, uuid, skip_when_checksum_same=True):
+    def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
        update_obj = {'last_notification_error': False, 'last_error': False}
        some_data = 'xxxxx'
        update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()
--- a/changedetectionio/processors/text_json_diff.py
+++ b/changedetectionio/processors/text_json_diff.py
@@ -50,7 +50,7 @@ class perform_site_check(difference_detection_processor):

        return regex

-    def run(self, uuid, skip_when_checksum_same=True):
+    def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
        changed_detected = False
        screenshot = False  # as bytes
        stripped_text_from_html = ""
@@ -105,7 +105,11 @@ class perform_site_check(difference_detection_processor):
            # If the klass doesnt exist, just use a default
            klass = getattr(content_fetcher, "html_requests")

-        proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
+        if preferred_proxy:
+            proxy_id = preferred_proxy
+        else:
+            proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
+
        proxy_url = None
        if proxy_id:
            proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')
--- a/changedetectionio/run_proxy_tests.sh
+++ b/changedetectionio/run_proxy_tests.sh
@@ -2,6 +2,8 @@

 # exit when any command fails
 set -e
+# enable debug
+set -x

 # Test proxy list handling, starting two squids on different ports
 # Each squid adds a different header to the response, which is the main thing we test for.
@@ -19,7 +21,6 @@ docker run --network changedet-network -d \


 ## 2nd test actually choose the preferred proxy from proxies.json
-
 docker run --network changedet-network \
  -v `pwd`/tests/proxy_list/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json \
  test-changedetectionio \
@@ -44,7 +45,6 @@ fi


 # Test the UI configurable proxies
-
 docker run --network changedet-network \
  test-changedetectionio \
  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_select_custom_proxy.py'
@@ -58,4 +58,25 @@ then
  exit 1
 fi

+# Test "no-proxy" option
+docker run --network changedet-network \
+  test-changedetectionio \
+  bash -c 'cd changedetectionio && pytest tests/proxy_list/test_noproxy.py'
+
+# We need to handle grep returning 1
+set +e
+# Check request was never seen in any container
+for c in $(echo "squid-one squid-two squid-custom"); do
+  echo Checking $c
+  docker logs $c &> $c.txt
+  grep noproxy $c.txt
+  if [ $? -ne 1 ]
+  then
+    echo "Saw request for noproxy in $c container"
+    cat $c.txt
+    exit 1
+  fi
+done
+
+
 docker kill squid-one squid-two squid-custom
--- a/changedetectionio/static/images/brightdata.svg
+++ b/changedetectionio/static/images/brightdata.svg
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   height="59.553207"
+   viewBox="-0.36 95.21 25.082135 59.553208"
+   width="249.99138"
+   version="1.1"
+   id="svg12"
+   sodipodi:docname="brightdata.svg"
+   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs16" />
+  <sodipodi:namedview
+     id="namedview14"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0"
+     inkscape:zoom="0.9464"
+     inkscape:cx="22.189349"
+     inkscape:cy="-90.870668"
+     inkscape:window-width="1920"
+     inkscape:window-height="1051"
+     inkscape:window-x="1920"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg12" />
+  <path
+     d="m -34.416031,129.28 c -3.97,-2.43 -5.1,-6.09 -4.32,-10.35 0.81,-4.4 3.95,-6.75 8.04,-7.75 4.23,-1.04 8.44,-0.86 12.3,1.5 0.63,0.39 0.93,0.03 1.31,-0.29 1.5,-1.26 3.27,-1.72 5.189999,-1.83 0.79,-0.05 1.04,0.24 1.01,1.01 -0.05,1.31 -0.04,2.63 0,3.95 0.02,0.65 -0.19,0.93 -0.87,0.89 -0.889999,-0.04 -1.789999,0.03 -2.669999,-0.02 -0.82,-0.04 -1.08,0.1 -0.88,1.04 0.83,3.9 -0.06,7.37 -3.1,10.06 -2.76,2.44 -6.13,3.15 -9.72,3.04 -0.51,-0.02 -1.03,-0.02 -1.52,-0.13 -1.22,-0.25 -1.96,0.14 -2.19,1.41 -0.28,1.54 0.16,2.62 1.37,3.07 0.84,0.31 1.74,0.35 2.63,0.39 2.97,0.13 5.95,-0.18 8.91,0.21 2.93,0.39 5.69,1.16 6.85,4.25 1.269999,3.38 0.809999,6.62 -1.48,9.47 -2.73,3.39 -6.52,4.78 -10.66,5.33 -3.53,0.48 -7.04,0.27 -10.39,-1.11 -3.89,-1.6 -5.75,-4.95 -4.84,-8.72 0.51,-2.11 1.85,-3.58 3.69,-4.65 0.38,-0.22 0.93,-0.32 0.28,-0.96 -2.91,-2.83 -2.85,-6.16 0.1,-8.95 0.28,-0.26 0.6,-0.53 0.96,-0.86 z m 8.07,21.5 c 0.95,0.04 1.87,-0.13 2.78,-0.33 1.89,-0.42 3.51,-1.3 4.49,-3.06 1.82,-3.25 0.24,-6.2 -3.37,-6.58 -2.88,-0.3 -5.76,0.24 -8.63,-0.13 -0.53,-0.07 -0.75,0.34 -0.95,0.71 -1.16,2.24 -1.08,4.53 0,6.73 1.15,2.34 3.46,2.48 5.68,2.66 z m -5,-30.61 c -0.03,1.67 0.08,3.19 0.74,4.61 0.76,1.62 2.17,2.42 4.03,2.31 1.62,-0.1 2.9,-1.12 3.36,-2.84 0.66,-2.46 0.69,-4.95 0.01,-7.42 -0.49,-1.76 -1.7,-2.64 -3.56,-2.7 -2.08,-0.07 -3.37,0.7 -4.04,2.42 -0.47,1.21 -0.6,2.47 -0.54,3.62 z m 32.9399993,6.56 c 0,2.59 0.05,5.18 -0.02,7.77 -0.03,1.03 0.31,1.46 1.32,1.52 0.65,0.04 1.61,-0.09 1.82,0.57 0.26,0.81 0.11,1.76 0.06,2.65 -0.03,0.48 -0.81,0.39 -0.81,0.39 l -11.47,0.01 c 0,0 -0.95,-0.21 -0.88,-0.88 0.03,-0.29 0.04,-0.6 0,-0.89 -0.19,-1.24 0.21,-1.92 1.58,-1.9 0.99,0.01 1.28,-0.52 1.28,-1.53 -0.05,-8.75 -0.05,-17.49 0,-26.24 0.01,-1.15 -0.36,-1.62 -1.44,-1.67 -0.17,-0.01 -0.34,-0.04 -0.5,-0.07 -1.43,-0.22 -2.12,-1.57 -1.53,-2.91 0.15,-0.35 0.43,-0.36 0.72,-0.4 2.94,-0.41 5.88,-0.81 8.82000002,-1.23 0.81999998,-0.12 0.99999998,0.27 0.98999998,1.01 -0.02,3.35 0,6.71 0.02,10.06 0,0.35 -0.23,0.84 0.18,1.03 0.38,0.17 0.69,-0.25 0.99,-0.45 2.56,-1.74 5.33,-2.73 8.4900007,-2.56 3.51005,0.19 5.65005,1.95 6.35005,5.46 0.42,2.09 0.52,4.21 0.51,6.33 -0.02,3.86 0.05,7.73 -0.04,11.59 -0.02,1.12 0.37,1.5 1.39,1.6 0.61,0.05 1.55,-0.13 1.74,0.47 0.26,0.85 0.12,1.84 0.1,2.77 -0.01,0.41 -0.69,0.37 -0.69,0.37 l -11.4700504,0.01 c 0,0 -0.81,-0.29 -0.8,-0.85 0.01,-0.38 0.04,-0.77 -0.01,-1.15 -0.13,-1.01 0.32,-1.52 1.31,-1.56 1.0600004,-0.05 1.3800004,-0.55 1.3500004,-1.63 -0.14,-4.84 0.16,-9.68 -0.18,-14.51 -0.26,-3.66 -2.1100004,-4.95 -5.6700007,-3.99 -0.25,0.07 -0.49,0.15 -0.73,0.22 -2.57,0.8 -2.79,1.09 -2.79,3.71 0.01,2.3 0.01,4.59 0.01,6.88 z M -109.26603,122.56 c 0,-4.75 -0.02,-9.51 0.02,-14.26 0.01,-0.92 -0.17,-1.47 -1.19,-1.45 -0.16,0 -0.33,-0.07 -0.5,-0.1 -1.56,-0.27 -2.24,-1.47 -1.69,-2.92 0.14,-0.37 0.41,-0.38 0.7,-0.42 2.98,-0.41 5.97,-0.81 8.94,-1.24 0.85,-0.12 0.88,0.33 0.88,0.96 -0.01,3.01 -0.01,6.03 0,9.04 0,0.4 -0.18,0.96 0.27,1.16 0.36,0.16 0.66,-0.3 0.96,-0.52 4.729999,-3.51 12.459999,-2.61 14.889999,4.48 1.89,5.51 1.91,11.06 -0.96,16.28 -2.37,4.31 -6.19,6.49 -11.15,6.59 -3.379999,0.07 -6.679999,-0.3 -9.909999,-1.37 -0.93,-0.31 -1.3,-0.78 -1.28,-1.83 0.05,-4.81 0.02,-9.6 0.02,-14.4 z m 7.15,3.89 c 0,2.76 0.02,5.52 -0.01,8.28 -0.01,0.76 0.18,1.29 0.91,1.64 1.899999,0.9 4.299999,0.5 5.759999,-1.01 0.97,-1 1.56,-2.21 1.96,-3.52 1.03,-3.36 0.97,-6.78 0.61,-10.22 a 9.991,9.991 0 0 0 -0.93,-3.29 c -1.47,-3.06 -4.67,-3.85 -7.439999,-1.86 -0.6,0.43 -0.88,0.93 -0.87,1.7 0.04,2.76 0.01,5.52 0.01,8.28 z"
+     fill="#4280f6"
+     id="path2" />
+  <path
+     d="m 68.644019,137.2 c -1.62,1.46 -3.41,2.56 -5.62,2.96 -4.4,0.8 -8.7,-1.39 -10.49,-5.49 -2.31,-5.31 -2.3,-10.67 -0.1,-15.98 2.31,-5.58 8.29,-8.65 14.24,-7.46 1.71,0.34 1.9,0.18 1.9,-1.55 0,-0.68 -0.05,-1.36 0.01,-2.04 0.09,-1.02 -0.25,-1.54 -1.34,-1.43 -0.64,0.06 -1.26,-0.1 -1.88,-0.21 -1.32,-0.24 -1.6,-0.62 -1.37,-1.97 0.07,-0.41 0.25,-0.57 0.65,-0.62 2.63,-0.33 5.27,-0.66 7.9,-1.02 1.04,-0.14 1.17,0.37 1.17,1.25 -0.02,10.23 -0.02,20.45 -0.01,30.68 v 1.02 c 0.02,0.99 0.35,1.6 1.52,1.47 0.52,-0.06 1.35,-0.27 1.25,0.73 -0.08,0.8 0.58,1.93 -0.94,2.18 -1.29,0.22 -2.51,0.69 -3.86,0.65 -2.04,-0.06 -2.3,-0.23 -2.76,-2.19 -0.09,-0.3 0.06,-0.67 -0.27,-0.98 z m -0.07,-12.46 c 0,-2.8 -0.04,-5.6 0.02,-8.39 0.02,-0.9 -0.28,-1.47 -1.05,-1.81 -3.18,-1.4 -7.54,-0.8 -9.3,2.87 -0.83,1.74 -1.31,3.54 -1.49,5.46 -0.28,2.93 -0.38,5.83 0.61,8.65 0.73,2.09 1.81,3.9 4.11,4.67 2.49,0.83 4.55,-0.04 6.5,-1.48 0.54,-0.4 0.62,-0.95 0.61,-1.57 -0.02,-2.8 -0.01,-5.6 -0.01,-8.4 z m 28.79,2.53 c 0,3.24 0.04,5.83 -0.02,8.41 -0.02,1 0.19,1.49 1.309998,1.41 0.55,-0.04 1.460003,-0.46 1.520003,0.73 0.05,1.02 0.1,1.89 -1.330003,2.08 -1.289998,0.17 -2.559998,0.51 -3.889998,0.48 -1.88,-0.05 -2.15,-0.26 -2.42,-2.15 -0.04,-0.27 0.14,-0.65 -0.22,-0.79 -0.34,-0.13 -0.5,0.24 -0.72,0.42 -3.61,3 -8.15,3.4 -11.64,1.08 -1.61,-1.07 -2.49,-2.63 -2.67,-4.43 -0.51,-5.13 0.77,-7.91 6.3,-10.22 2.44,-1.02 5.07,-1.27 7.68,-1.49 0.77,-0.07 1.03,-0.28 1.02,-1.05 -0.03,-1.48 -0.05,-2.94 -0.64,-4.36 -0.59,-1.42 -1.67,-1.92 -3.08,-2.03 -3.04,-0.24 -5.88,0.5 -8.63,1.71 -0.51,0.23 -1.19,0.75 -1.48,-0.13 -0.26,-0.77 -1.35,-1.61 0.05,-2.47 3.27,-2 6.7,-3.44 10.61,-3.42 1.44,0.01 2.88,0.27 4.21,0.81 2.67,1.08 3.44,3.4 3.8,5.99 0.46,3.37 0.1,6.73 0.24,9.42 z m -5.09,2.9 c 0,-1.23 -0.01,-2.46 0,-3.69 0,-0.52 -0.06,-0.98 -0.75,-0.84 -1.45,0.3 -2.93,0.28 -4.37,0.69 -3.71,1.04 -5.46,4.48 -3.97,8.03 0.51,1.22 1.48,1.98 2.79,2.16 2.01,0.28 3.86,-0.29 5.6,-1.28 0.54,-0.31 0.73,-0.76 0.72,-1.37 -0.05,-1.23 -0.02,-2.47 -0.02,-3.7 z m 43.060001,-2.89 c 0,2.72 0.01,5.43 -0.01,8.15 0,0.66 0.02,1.21 0.91,1.12 0.54,-0.06 0.99,0.12 0.86,0.75 -0.15,0.71 0.56,1.7 -0.58,2.09 -1.55,0.52 -3.16,0.59 -4.77,0.4 -0.99,-0.12 -1.12,-1.01 -1.18,-1.73 -0.08,-1.15 -0.16,-1.45 -1.24,-0.54 -3.41,2.87 -8.05,3.17 -11.43,0.88 -1.75,-1.18 -2.49,-2.91 -2.7,-4.94 -0.64,-6.24 3.16,-8.74 7.83,-10.17 2.04,-0.62 4.14,-0.8 6.24,-0.99 0.81,-0.07 1,-0.36 0.98,-1.09 -0.04,-1.31 0.04,-2.62 -0.42,-3.89 -0.57,-1.57 -1.53,-2.34 -3.18,-2.45 -3.03,-0.21 -5.88,0.46 -8.64,1.66 -0.6,0.26 -1.25,0.81 -1.68,-0.2 -0.34,-0.8 -1.08,-1.61 0.16,-2.36 4.12,-2.5 8.44,-4.16 13.36,-3.07 3.21,0.71 4.89,2.91 5.26,6.34 0.18,1.69 0.22,3.37 0.22,5.07 0.01,1.66 0.01,3.32 0.01,4.97 z m -5.09,2.54 c 0,-1.27 -0.03,-2.54 0.01,-3.81 0.02,-0.74 -0.27,-1.02 -0.98,-0.92 -1.21,0.17 -2.43,0.28 -3.62,0.55 -3.72,0.83 -5.47,3.48 -4.82,7.21 0.29,1.66 1.57,2.94 3.21,3.16 2.02,0.27 3.85,-0.34 5.57,-1.34 0.49,-0.29 0.64,-0.73 0.63,-1.29 -0.02,-1.18 0,-2.37 0,-3.56 z"
+     fill="#c8dbfb"
+     id="path4" />
+  <path
+     d="m 26.314019,125.77 c 0,-2.89 -0.05,-5.77 0.02,-8.66 0.03,-1.04 -0.33,-1.39 -1.31,-1.24 a 0.7,0.7 0 0 1 -0.25,0 c -0.57,-0.18 -1.44,0.48 -1.68,-0.58 -0.35,-1.48 -0.02,-2.3 1.21,-2.7 1.3,-0.43 2.16,-1.26 2.76,-2.46 0.78,-1.56 1.44,-3.17 1.91,-4.84 0.18,-0.63 0.47,-0.86 1.15,-0.88 3.28,-0.09 3.27,-0.11 3.32,3.17 0.01,1.06 0.09,2.12 0.09,3.18 -0.01,0.67 0.27,0.89 0.91,0.88 1.61,-0.02 3.23,0.03 4.84,-0.02 0.77,-0.02 1.01,0.23 1.03,1.01 0.08,3.27 0.1,3.27 -3.09,3.27 -0.93,0 -1.87,0.03 -2.8,-0.01 -0.67,-0.02 -0.89,0.26 -0.88,0.91 0.04,5.43 0.04,10.86 0.12,16.29 0.02,1.7 0.75,2.26 2.46,2.1 1.1,-0.1 2.19,-0.26 3.23,-0.65 0.59,-0.22 0.89,-0.09 1.14,0.53 0.93,2.29 0.92,2.37 -1.32,3.52 -2.54,1.3 -5.22,1.99 -8.1,1.79 -2.27,-0.16 -3.68,-1.27 -4.35,-3.45 -0.3,-0.98 -0.41,-1.99 -0.41,-3.01 z m -97.67005,-8.99 c 0.57,-0.84 1.11,-1.74 1.76,-2.55 1.68,-2.09 3.68,-3.62 6.54,-3.66 1.08,-0.01 1.63,0.28 1.57,1.52 -0.1,2.08 -0.05,4.16 -0.02,6.24 0.01,0.74 -0.17,0.96 -0.96,0.76 -2.36,-0.59 -4.71,-0.42 -7.03,0.28 -0.8,0.24 -1.16,0.62 -1.15,1.52 0.05,4.5 0.04,9 0,13.5 -0.01,0.89 0.29,1.16 1.15,1.2 1.23,0.06 2.44,0.32 3.67,0.39 0.75,0.05 0.91,0.38 0.89,1.04 -0.06,2.86 0.29,2.28 -2.25,2.3 -4.2,0.04 -8.41,-0.02 -12.61,0.03 -0.91,0.01 -1.39,-0.18 -1.22,-1.18 0.02,-0.12 0,-0.25 0,-0.38 0.02,-2.1 -0.24,-1.88 1.77,-2.04 1.33,-0.11 1.6,-0.67 1.58,-1.9 -0.07,-5.35 -0.04,-10.7 -0.02,-16.05 0,-0.78 -0.17,-1.2 -1,-1.46 -2.21,-0.68 -2.7,-1.69 -2.22,-3.99 0.11,-0.52 0.45,-0.56 0.82,-0.62 2.22,-0.34 4.44,-0.7 6.67,-0.99 0.99,-0.13 1.82,0.7 1.84,1.76 0.03,1.4 0.03,2.8 0.04,4.2 -0.01,0.02 0.06,0.04 0.18,0.08 z m 25.24,6.59 c 0,3.69 0.04,7.38 -0.03,11.07 -0.02,1.04 0.31,1.48 1.32,1.49 0.29,0 0.59,0.12 0.88,0.13 0.93,0.01 1.18,0.47 1.16,1.37 -0.05,2.19 0,2.19 -2.24,2.19 -3.48,0 -6.96,-0.04 -10.44,0.03 -1.09,0.02 -1.47,-0.33 -1.3,-1.36 0.02,-0.12 0.02,-0.26 0,-0.38 -0.28,-1.39 0.39,-1.96 1.7,-1.9 1.36,0.06 1.76,-0.51 1.74,-1.88 -0.09,-5.17 -0.08,-10.35 0,-15.53 0.02,-1.22 -0.32,-1.87 -1.52,-2.17 -0.57,-0.14 -1.47,-0.11 -1.57,-0.85 -0.15,-1.04 -0.05,-2.11 0.01,-3.17 0.02,-0.34 0.44,-0.35 0.73,-0.39 2.81,-0.39 5.63,-0.77 8.44,-1.18 0.92,-0.14 1.15,0.2 1.14,1.09 -0.04,3.8 -0.02,7.62 -0.02,11.44 z"
+     fill="#4280f6"
+     id="path6" />
+  <path
+     d="m 101.44402,125.64 c 0,-3.18 -0.03,-6.37 0.02,-9.55 0.02,-0.94 -0.26,-1.36 -1.22,-1.22 -0.21,0.03 -0.430003,0.04 -0.630003,0 -0.51,-0.12 -1.35,0.39 -1.44,-0.55 -0.08,-0.85 -0.429998,-1.87 0.93,-2.24 2.080003,-0.57 2.720003,-2.39 3.350003,-4.17 0.31,-0.88 0.62,-1.76 0.87,-2.66 0.18,-0.64 0.52,-0.85 1.19,-0.84 2.46,0.05 2,-0.15 2.04,2.04 0.02,1.1 0.08,2.21 -0.02,3.31 -0.11,1.16 0.46,1.52 1.46,1.53 1.78,0.01 3.57,0.04 5.35,-0.01 0.82,-0.02 1.12,0.23 1.11,1.08 -0.05,2.86 0.19,2.49 -2.42,2.51 -1.53,0.01 -3.06,0.02 -4.59,-0.01 -0.65,-0.01 -0.9,0.22 -0.9,0.89 0.02,5.52 0,11.04 0.03,16.56 0,0.67 0.14,1.34 0.25,2.01 0.17,1.04 1.17,1.62 2.59,1.42 1.29,-0.19 2.57,-0.49 3.86,-0.69 0.43,-0.07 1.05,-0.47 1.19,0.4 0.12,0.75 1.05,1.61 -0.09,2.24 -2.09,1.16 -4.28,2.07 -6.71,2.16 -1.05,0.04 -2.13,0.2 -3.16,-0.14 -1.92,-0.65 -3.03,-2.28 -3.05,-4.51 -0.02,-3.19 -0.01,-6.37 -0.01,-9.56 z"
+     fill="#c8dbfb"
+     id="path8" />
+  <path
+     d="m -50.816031,95.21 c 0.19,2.160002 1.85,3.240002 2.82,4.740002 0.25,0.379998 0.48,0.109998 0.67,-0.16 0.21,-0.31 0.6,-1.21 1.15,-1.28 -0.35,1.38 -0.04,3.149998 0.16,4.449998 0.49,3.05 -1.22,5.64 -4.07,6.18 -3.38,0.65 -6.22,-2.21 -5.6,-5.62 0.23,-1.24 1.37,-2.5 0.77,-3.699998 -0.85,-1.7 0.54,-0.52 0.79,-0.22 1.04,1.199998 1.21,0.09 1.45,-0.55 0.24,-0.63 0.31,-1.31 0.47,-1.97 0.19,-0.770002 0.55,-1.400002 1.39,-1.870002 z"
+     fill="#4280f6"
+     id="path10" />
+</svg>
--- a/changedetectionio/static/images/oxylabs.svg
+++ b/changedetectionio/static/images/oxylabs.svg
--- a/changedetectionio/static/js/diff-overview.js
+++ b/changedetectionio/static/js/diff-overview.js
@@ -1,4 +1,13 @@
 $(document).ready(function () {
+    var csrftoken = $('input[name=csrf_token]').val();
+    $.ajaxSetup({
+        beforeSend: function (xhr, settings) {
+            if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken)
+            }
+        }
+    })
+
    // Load it when the #screenshot tab is in use, so we dont give a slow experience when waiting for the text diff to load
    window.addEventListener('hashchange', function (e) {
        toggle(location.hash);
@@ -15,11 +24,71 @@ $(document).ready(function () {
            $("#settings").hide();
        } else if (hash_name === '#extract') {
            $("#settings").hide();
-        }
-
-
-        else {
+        } else {
            $("#settings").show();
        }
    }
+
+    const article = $('.highlightable-filter')[0];
+
+    // We could also add the  'touchend' event for touch devices, but since
+    // most iOS/Android browsers already show a dialog when you select
+    // text (often with a Share option) we'll skip that
+    article.addEventListener('mouseup', dragTextHandler, false);
+    article.addEventListener('mousedown', clean, false);
+
+    function clean(event) {
+      $("#highlightSnippet").remove();
+    }
+
+
+    function dragTextHandler(event) {
+        console.log('mouseupped');
+
+        // Check if any text was selected
+        if (window.getSelection().toString().length > 0) {
+
+            // Find out how much (if any) user has scrolled
+            var scrollTop = (window.pageYOffset !== undefined) ? window.pageYOffset : (document.documentElement || document.body.parentNode || document.body).scrollTop;
+
+            // Get cursor position
+            const posX = event.clientX;
+            const posY = event.clientY + 20 + scrollTop;
+
+            // Append HTML to the body, create the "Tweet Selection" dialog
+            document.body.insertAdjacentHTML('beforeend', '<div id="highlightSnippet" style="position: absolute; top: ' + posY + 'px; left: ' + posX + 'px;"><div class="pure-form-message-inline" style="font-size: 70%">Ignore any change on any line which contains the selected text.</div><br><a data-mode="exact" href="javascript:void(0);" class="pure-button button-secondary button-xsmall">Ignore exact text</a>&nbsp;</div>');
+
+            if (/\d/.test(window.getSelection().toString())) {
+                // Offer regex replacement
+                document.getElementById("highlightSnippet").insertAdjacentHTML('beforeend', '<a data-mode="digit-regex"  href="javascript:void(0);" class="pure-button button-secondary button-xsmall">Ignore text including number changes</a>');
+            }
+
+            $('#highlightSnippet a').bind('click', function (e) {
+                if(!window.getSelection().toString().trim().length) {
+                    alert('Oops no text selected!');
+                    return;
+                }
+
+                $.ajax({
+                    type: "POST",
+                    url: highlight_submit_ignore_url,
+                    data: {'mode': $(this).data('mode'), 'selection': window.getSelection().toString()},
+                    statusCode: {
+                        400: function () {
+                            // More than likely the CSRF token was lost when the server restarted
+                            alert("There was a problem processing the request, please reload the page.");
+                        }
+                    }
+                }).done(function (data) {
+                    $("#highlightSnippet").html(data)
+                }).fail(function (data) {
+                    console.log(data);
+                    alert('There was an error communicating with the server.');
+                });
+            });
+
+        }
+    }
+
+
 });
--- a/changedetectionio/static/js/global-settings.js
+++ b/changedetectionio/static/js/global-settings.js
@@ -32,5 +32,10 @@ $(document).ready(function () {
        window.getSelection().removeAllRanges();

    });
+
+    $("#notification-token-toggle").click(function (e) {
+        e.preventDefault();
+        $('#notification-tokens-info').toggle();
+    });
 });

--- a/changedetectionio/static/js/recheck-proxy.js
+++ b/changedetectionio/static/js/recheck-proxy.js
@@ -0,0 +1,87 @@
+$(function () {
+    /* add container before each proxy location to show status */
+
+    var option_li = $('.fetch-backend-proxy li').filter(function() {
+        return $("input",this)[0].value.length >0;
+    });
+
+    //var option_li = $('.fetch-backend-proxy li');
+    var isActive = false;
+    $(option_li).prepend('<div class="proxy-status"></div>');
+    $(option_li).append('<div class="proxy-timing"></div><div class="proxy-check-details"></div>');
+
+    function set_proxy_check_status(proxy_key, state) {
+        // select input by value name
+        const proxy_li = $('input[value="' + proxy_key + '" ]').parent();
+        if (state['status'] === 'RUNNING') {
+            $('.proxy-status', proxy_li).html('<span class="spinner"></span>');
+        }
+        if (state['status'] === 'OK') {
+            $('.proxy-status', proxy_li).html('<span style="color: green; font-weight: bold" >OK</span>');
+            $('.proxy-check-details', proxy_li).html(state['text']);
+        }
+        if (state['status'] === 'ERROR' || state['status'] === 'ERROR OTHER') {
+            $('.proxy-status', proxy_li).html('<span style="color: red; font-weight: bold" >X</span>');
+            $('.proxy-check-details', proxy_li).html(state['text']);
+        }
+        $('.proxy-timing', proxy_li).html(state['time']);
+    }
+
+
+    function pollServer() {
+        if (isActive) {
+            window.setTimeout(function () {
+                $.ajax({
+                    url: proxy_recheck_status_url,
+                    success: function (data) {
+                        var all_done = true;
+                        $.each(data, function (proxy_key, state) {
+                            set_proxy_check_status(proxy_key, state);
+                            if (state['status'] === 'RUNNING') {
+                                all_done = false;
+                            }
+                        });
+
+                        if (all_done) {
+                            console.log("Shutting down poller, all done.")
+                            isActive = false;
+                        } else {
+                            pollServer();
+                        }
+                    },
+                    error: function () {
+                        //ERROR HANDLING
+                        pollServer();
+                    }
+                });
+            }, 2000);
+        }
+    }
+
+    $('#check-all-proxies').click(function (e) {
+        e.preventDefault()
+        $('body').addClass('proxy-check-active');
+        $('.proxy-check-details').html('');
+        $('.proxy-status').html('<span class="spinner"></span>').fadeIn();
+        $('.proxy-timing').html('');
+
+        // Request start, needs CSRF?
+        $.ajax({
+            type: "GET",
+            url: recheck_proxy_start_url,
+        }).done(function (data) {
+            $.each(data, function (proxy_key, state) {
+                set_proxy_check_status(proxy_key, state['status'])
+            });
+            isActive = true;
+            pollServer();
+
+        }).fail(function (data) {
+            console.log(data);
+            alert('There was an error communicating with the server.');
+        });
+
+    });
+
+});
+
--- a/changedetectionio/static/js/watch-settings.js
+++ b/changedetectionio/static/js/watch-settings.js
@@ -42,4 +42,8 @@ $(document).ready(function () {
        $('#notification_urls').val('');
        e.preventDefault();
    });
+    $("#notification-token-toggle").click(function (e) {
+        e.preventDefault();
+        $('#notification-tokens-info').toggle();
+    });
 });
--- a/changedetectionio/static/styles/diff.css
+++ b/changedetectionio/static/styles/diff.css
@@ -218,3 +218,10 @@ td#diff-col div {
  text-align: center; }
  .tab-pane-inner#screenshot img {
    max-width: 99%; }
+
+#highlightSnippet {
+  background: var(--color-background);
+  padding: 1em;
+  border-radius: 5px;
+  background: var(--color-background);
+  box-shadow: 1px 1px 4px var(--color-shadow-jump); }
--- a/changedetectionio/static/styles/scss/diff.scss
+++ b/changedetectionio/static/styles/scss/diff.scss
@@ -119,3 +119,11 @@ td#diff-col div {
    max-width: 99%;
  }
 }
+
+#highlightSnippet {
+  background: var(--color-background);
+  padding: 1em;
+  border-radius: 5px;
+  background: var(--color-background);
+  box-shadow: 1px 1px 4px var(--color-shadow-jump);
+}
--- a/changedetectionio/static/styles/scss/parts/_darkmode.scss
+++ b/changedetectionio/static/styles/scss/parts/_darkmode.scss
@@ -0,0 +1,25 @@
+
+#toggle-light-mode {
+  width: 3rem;
+  /* default */
+  .icon-dark {
+    display: none;
+  }
+
+}
+
+html[data-darkmode="true"] {
+  #toggle-light-mode {
+    .icon-light {
+      display: none;
+    }
+
+    .icon-dark {
+      display: block;
+    }
+  }
+}
+
+
+
+
--- a/changedetectionio/static/styles/scss/parts/_extra_proxies.scss
+++ b/changedetectionio/static/styles/scss/parts/_extra_proxies.scss
@@ -7,6 +7,7 @@ ul#requests-extra_proxies {
    }

  }
+
  /* each proxy entry is a `table` */
  table {
    tr {
@@ -15,3 +16,47 @@ ul#requests-extra_proxies {
  }
 }

+#request {
+  /* Auto proxy scan/checker */
+  label[for=proxy] {
+    display: inline-block;
+  }
+}
+
+body.proxy-check-active {
+  #request {
+    .proxy-status {
+      width: 2em;
+    }
+
+    .proxy-check-details {
+      font-size: 80%;
+      color: #555;
+      display: block;
+      padding-left: 4em;
+    }
+
+    .proxy-timing {
+      font-size: 80%;
+      padding-left: 1rem;
+      color: var(--color-link);
+    }
+  }
+}
+
+
+#recommended-proxy {
+  display: grid;
+  gap: 2rem;
+    @media  (min-width: 991px) {
+      grid-template-columns: repeat(2, 1fr);
+    }
+
+  > div {
+    border: 1px #aaa solid;
+    border-radius: 4px;
+    padding: 1em;
+  }
+
+  padding-bottom: 1em;
+}
--- a/changedetectionio/static/styles/scss/styles.scss
+++ b/changedetectionio/static/styles/scss/styles.scss
@@ -8,6 +8,7 @@
@import "parts/_pagination";
@import "parts/_spinners";
@import "parts/_variables";
+@import "parts/_darkmode";

 body {
  color: var(--color-text);
@@ -54,22 +55,6 @@ a.github-link {
  }
 }

-#toggle-light-mode {
-  width: 3rem;
-  .icon-dark {
-    display: none;
-  }
-
-  &.dark {
-    .icon-light {
-      display: none;
-    }
-
-    .icon-dark {
-      display: block;
-    }
-  }
-}

 #toggle-search {
  width: 2rem;
--- a/changedetectionio/static/styles/styles.css
+++ b/changedetectionio/static/styles/styles.css
@@ -95,6 +95,37 @@ ul#requests-extra_proxies {
  ul#requests-extra_proxies table tr {
    display: inline; }

+#request {
+  /* Auto proxy scan/checker */ }
+  #request label[for=proxy] {
+    display: inline-block; }
+
+body.proxy-check-active #request .proxy-status {
+  width: 2em; }
+
+body.proxy-check-active #request .proxy-check-details {
+  font-size: 80%;
+  color: #555;
+  display: block;
+  padding-left: 4em; }
+
+body.proxy-check-active #request .proxy-timing {
+  font-size: 80%;
+  padding-left: 1rem;
+  color: var(--color-link); }
+
+#recommended-proxy {
+  display: grid;
+  gap: 2rem;
+  padding-bottom: 1em; }
+  @media (min-width: 991px) {
+    #recommended-proxy {
+      grid-template-columns: repeat(2, 1fr); } }
+  #recommended-proxy > div {
+    border: 1px #aaa solid;
+    border-radius: 4px;
+    padding: 1em; }
+
 .pagination-page-info {
  color: #fff;
  font-size: 0.85rem;
@@ -283,10 +314,6 @@ html[data-darkmode="true"] {
  --color-icon-github-hover: var(--color-grey-700);
  --color-watch-table-error: var(--color-light-red);
  --color-watch-table-row-text: var(--color-grey-800); }
-  html[data-darkmode="true"] #toggle-light-mode .icon-light {
-    display: none; }
-  html[data-darkmode="true"] #toggle-light-mode .icon-dark {
-    display: block; }
  html[data-darkmode="true"] .icon-spread {
    filter: hue-rotate(-10deg) brightness(1.5); }
  html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
@@ -301,6 +328,18 @@ html[data-darkmode="true"] {
    html[data-darkmode="true"] .watch-table .unviewed.error {
      color: var(--color-watch-table-error); }

+#toggle-light-mode {
+  width: 3rem;
+  /* default */ }
+  #toggle-light-mode .icon-dark {
+    display: none; }
+
+html[data-darkmode="true"] #toggle-light-mode .icon-light {
+  display: none; }
+
+html[data-darkmode="true"] #toggle-light-mode .icon-dark {
+  display: block; }
+
 body {
  color: var(--color-text);
  background: var(--color-background-page); }
@@ -335,11 +374,6 @@ a.github-link {
  a.github-link:hover {
    color: var(--color-icon-github-hover); }

-#toggle-light-mode {
-  width: 3rem; }
-  #toggle-light-mode .icon-dark {
-    display: none; }
-
 #toggle-search {
  width: 2rem; }

--- a/changedetectionio/store.py
+++ b/changedetectionio/store.py
@@ -1,3 +1,5 @@
+from distutils.util import strtobool
+
 from flask import (
    flash
 )
@@ -16,6 +18,9 @@ import threading
 import time
 import uuid as uuid_builder

+# Because the server will run as a daemon and wont know the URL for notification links when firing off a notification
+BASE_URL_NOT_SET_TEXT = '("Base URL" not set - see settings - notifications)'
+
 dictfilt = lambda x, y: dict([ (i,x[i]) for i in x if i in set(y) ])

 # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
@@ -173,12 +178,21 @@ class ChangeDetectionStore:

    @property
    def data(self):
-        # Re #152, Return env base_url if not overriden, @todo also prefer the proxy pass url
-        env_base_url = os.getenv('BASE_URL','')
-        if not self.__data['settings']['application']['base_url']:
-          self.__data['settings']['application']['base_url'] = env_base_url.strip('" ')
+        # Re #152, Return env base_url if not overriden
+        # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services
+        #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.

-        return self.__data
+        active_base_url = BASE_URL_NOT_SET_TEXT
+        if self.__data['settings']['application'].get('base_url'):
+            active_base_url = self.__data['settings']['application'].get('base_url')
+        elif os.getenv('BASE_URL'):
+            active_base_url = os.getenv('BASE_URL')
+
+        # I looked at various ways todo the following, but in the end just copying the dict seemed simplest/most reliable
+        # even given the memory tradeoff - if you know a better way.. maybe return d|self.__data.. or something
+        d = self.__data
+        d['settings']['application']['active_base_url'] = active_base_url.strip('" ')
+        return d

    # Delete a single watch by UUID
    def delete(self, uuid):
@@ -205,10 +219,9 @@ class ChangeDetectionStore:

    # Clone a watch by UUID
    def clone(self, uuid):
-        url = self.data['watching'][uuid]['url']
-        tag = self.data['watching'][uuid].get('tags',[])
+        url = self.data['watching'][uuid].get('url')
        extras = self.data['watching'][uuid]
-        new_uuid = self.add_watch(url=url, tag_uuids=tag, extras=extras)
+        new_uuid = self.add_watch(url=url, extras=extras)
        return new_uuid

    def url_exists(self, url):
@@ -248,12 +261,9 @@ class ChangeDetectionStore:
        if extras is None:
            extras = {}

-        # should always be str
-        if tag is None or not tag:
-            tag = ''
-
        # Incase these are copied across, assume it's a reference and deepcopy()
        apply_extras = deepcopy(extras)
+        apply_extras['tags'] = [] if not apply_extras.get('tags') else apply_extras.get('tags')

        # Was it a share link? try to fetch the data
        if (url.startswith("https://changedetection.io/share/")):
@@ -303,20 +313,22 @@ class ChangeDetectionStore:
            flash('Watch protocol is not permitted by SAFE_PROTOCOL_REGEX', 'error')
            return None

-
-        # #Re 569
-        # Could be in 'tags',  var or extras, smash them together and strip
-        apply_extras['tags'] = []
-        if tag or extras.get('tags'):
-            tags = list(filter(None, list(set().union(tag.split(','), extras.get('tags', '').split(',')))))
-            for t in list(map(str.strip, tags)):
+        if tag and type(tag) == str:
+            # Then it's probably a string of the actual tag by name, split and add it
+            for t in tag.split(','):
                # for each stripped tag, add tag as UUID
-                apply_extras['tags'].append(self.add_tag(t))
+                for a_t in t.split(','):
+                    tag_uuid = self.add_tag(a_t)
+                    apply_extras['tags'].append(tag_uuid)

        # Or if UUIDs given directly
        if tag_uuids:
            apply_extras['tags'] = list(set(apply_extras['tags'] + tag_uuids))

+        # Make any uuids unique
+        if apply_extras.get('tags'):
+            apply_extras['tags'] = list(set(apply_extras.get('tags')))
+
        new_watch = Watch.model(datastore_path=self.datastore_path, url=url)

        new_uuid = new_watch.get('uuid')
@@ -327,6 +339,9 @@ class ChangeDetectionStore:
            if k in apply_extras:
                del apply_extras[k]

+        if not apply_extras.get('date_created'):
+            apply_extras['date_created'] = int(time.time())
+
        new_watch.update(apply_extras)
        new_watch.ensure_data_dir_exists()
        self.__data['watching'][new_uuid] = new_watch
@@ -470,6 +485,8 @@ class ChangeDetectionStore:
                    k = "ui-" + str(i) + proxy.get('proxy_name')
                    proxy_list[k] = {'label': proxy.get('proxy_name'), 'url': proxy.get('proxy_url')}

+        if proxy_list and strtobool(os.getenv('ENABLE_NO_PROXY_OPTION', 'True')):
+            proxy_list["no-proxy"] = {'label': "No proxy", 'url': ''}

        return proxy_list if len(proxy_list) else None

@@ -487,6 +504,9 @@ class ChangeDetectionStore:
        # If it's a valid one
        watch = self.data['watching'].get(uuid)

+        if strtobool(os.getenv('ENABLE_NO_PROXY_OPTION', 'True')) and watch.get('proxy') == "no-proxy":
+            return None
+
        if watch.get('proxy') and watch.get('proxy') in list(self.proxy_list.keys()):
            return watch.get('proxy')

@@ -568,7 +588,7 @@ class ChangeDetectionStore:
    def add_tag(self, name):
        # If name exists, return that
        n = name.strip().lower()
-        print (f">>> Adding new tag - '{n}")
+        print (f">>> Adding new tag - '{n}'")
        if not n:
            return False

@@ -780,15 +800,6 @@ class ChangeDetectionStore:
                continue
        return

-    # We don't know when the date_created was in the past until now, so just add an index number for now.
-    def update_11(self):
-        i = 0
-        for uuid, watch in self.data['watching'].items():
-            if not watch.get('date_created'):
-                watch['date_created'] = i
-            i+=1
-        return
-
    # Create tag objects and their references from existing tag text
    def update_12(self):
        i = 0
@@ -802,3 +813,11 @@ class ChangeDetectionStore:

                self.data['watching'][uuid]['tags'] = tag_uuids

+    # #1775 - Update 11 did not update the records correctly when adding 'date_created' values for sorting
+    def update_13(self):
+        i = 0
+        for uuid, watch in self.data['watching'].items():
+            if not watch.get('date_created'):
+                self.data['watching'][uuid]['date_created'] = i
+            i+=1
+        return
--- a/changedetectionio/templates/_common_fields.jinja
+++ b/changedetectionio/templates/_common_fields.jinja
@@ -13,9 +13,9 @@
                            <div class="pure-form-message-inline">
                              <ul>
                                <li>Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service! <i><a target=_new href="https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes">Please read the notification services wiki here for important configuration notes</a></i>.</li>
-                                <li><code>discord://</code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
-                                <li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
-                                <li><code>tgram://</code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_discord">discord://</a></code> (or <code>https://discord.com/api/webhooks...</code>)) </code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> bots can't send messages to other bots, so you should specify chat ID of non-bot user.</li>
+                                <li><code><a target=_new href="https://github.com/caronc/apprise/wiki/Notify_telegram">tgram://</a></code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
                                  <li>Accepts the <code>{{ '{{token}}' }}</code> placeholders listed below</li>
                              </ul>
@@ -35,18 +35,14 @@
                            </div>
                            <div class="pure-control-group">
                                {{ render_field(form.notification_body , rows=5, class="notification-body", placeholder=settings_application['notification_body']) }}
-                                <span class="pure-form-message-inline">Body for all notifications</span>
-                            </div>
-                            <div class="pure-control-group">
-                            <!-- unsure -->
-                                {{ render_field(form.notification_format , class="notification-format") }}
-                                <span class="pure-form-message-inline">Format for all notifications</span>
+                                <span class="pure-form-message-inline">Body for all notifications &dash; You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL, and tokens from below.
+                                </span>
+
                            </div>
                            <div class="pure-controls">
-                            <p class="pure-form-message-inline">
-                                You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL.
-                            </p>
-
+                                <div id="notification-token-toggle" class="pure-button button-tag button-xsmall">Show token/placeholders</div>
+                            </div>
+                            <div class="pure-controls" style="display: none;" id="notification-tokens-info">
                                <table class="pure-table" id="token-table">
                                    <thead>
                                    <tr>
@@ -99,6 +95,10 @@
                                        <td><code>{{ '{{diff_full}}' }}</code></td>
                                        <td>The diff output - full difference output</td>
                                    </tr>
+                                    <tr>
+                                        <td><code>{{ '{{diff_patch}}' }}</code></td>
+                                        <td>The diff output - patch in unified format</td>
+                                    </tr>
                                    <tr>
                                        <td><code>{{ '{{current_snapshot}}' }}</code></td>
                                        <td>The current snapshot value, useful when combined with JSON or CSS filters
@@ -111,12 +111,15 @@
                                    </tbody>
                                </table>
                                <div class="pure-form-message-inline">
-                                    <br>
-                                    URLs generated by changedetection.io (such as <code>{{ '{{diff_url}}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br>
-                                    Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
-									<br>
-									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
+                                    <p>
+									Warning: Contents of <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, and <code>{{ '{{diff_added}}' }}</code> depend on how the difference algorithm perceives the change. <br>
+                                    For example, an addition or removal could be perceived as a change in some cases. <a target="_new" href="https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens">More Here</a> <br>
+                                    </p>
                                </div>
                            </div>
+                            <div class="pure-control-group">
+                                {{ render_field(form.notification_format , class="notification-format") }}
+                                <span class="pure-form-message-inline">Format for all notifications</span>
+                            </div>
                        </div>
 {% endmacro %}
--- a/changedetectionio/templates/_helpers.jinja
+++ b/changedetectionio/templates/_helpers.jinja
@@ -1,7 +1,6 @@
 {% macro render_field(field) %}
-  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
-
+  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
  {% if field.errors %}
    <ul class=errors>
    {% for error in field.errors %}
@@ -25,18 +24,6 @@
  </div>
 {% endmacro %}

-{% macro render_field(field) %}
-  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
-  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
-  {% if field.errors %}
-    <ul class=errors>
-    {% for error in field.errors %}
-      <li>{{ error }}</li>
-    {% endfor %}
-    </ul>
-  {% endif %}
-  </div>
-{% endmacro %}

 {% macro render_simple_field(field) %}
  <span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>
--- a/changedetectionio/templates/base.html
+++ b/changedetectionio/templates/base.html
@@ -37,7 +37,7 @@
    <div class="header">
      <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed" id="nav-menu">
        {% if has_password and not current_user.is_authenticated %}
-          <a class="pure-menu-heading" href="https://github.com/dgtlmoon/changedetection.io" rel="noopener">
+          <a class="pure-menu-heading" href="https://changedetection.io" rel="noopener">
            <strong>Change</strong>Detection.io</a>
        {% else %}
          <a class="pure-menu-heading" href="{{url_for('index')}}">
@@ -49,7 +49,7 @@
        {% else %}
          {% if new_version_available and not(has_password and not current_user.is_authenticated) %}
            <span id="new-version-text" class="pure-menu-heading">
-              <a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a>
+              <a href="https://changedetection.io">A new version is available</a>
            </span>
          {% endif %}
        {% endif %}
@@ -77,7 +77,7 @@
            {% endif %}
          {% else %}
            <li class="pure-menu-item">
-              <a class="pure-menu-link" href="https://github.com/dgtlmoon/changedetection.io">Website Change Detection and Notification.</a>
+              <a class="pure-menu-link" href="https://changedetection.io">Website Change Detection and Notification.</a>
            </li>
          {% endif %}
          {% if current_user.is_authenticated %}
--- a/changedetectionio/templates/diff.html
+++ b/changedetectionio/templates/diff.html
@@ -6,6 +6,9 @@
    {% if last_error_screenshot %}
    const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
    {% endif %}
+
+    const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
+
 </script>
 <script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>

@@ -76,7 +79,7 @@
    </div>

     <div class="tab-pane-inner" id="text">
-         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored.</div>
+         <div class="tip">Pro-tip: Use <strong>show current snapshot</strong> tab to visualise what will be ignored, highlight text to add to ignore filters</div>

         {% if password_enabled_and_share_is_off %}
           <div class="tip">Pro-tip: You can enable <strong>"share access when password is enabled"</strong> from settings</div>
@@ -91,7 +94,7 @@
                 <td id="a" style="display: none;">{{previous}}</td>
                 <td id="b" style="display: none;">{{newest}}</td>
                 <td id="diff-col">
-                     <span id="result"></span>
+                     <span id="result" class="highlightable-filter"></span>
                 </td>
             </tr>
             </tbody>
--- a/changedetectionio/templates/edit.html
+++ b/changedetectionio/templates/edit.html
@@ -4,18 +4,19 @@
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
-    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
-    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
-    const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
-
-{% if emailprefix %}
-    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
-{% endif %}

    const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
    const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
    const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
+{% if emailprefix %}
+    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
+{% endif %}
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+    const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
+    const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
+    const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
+    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
+    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";

 </script>

@@ -27,6 +28,8 @@
 <script src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
 {% endif %}

+<script src="{{url_for('static_content', group='js', filename='recheck-proxy.js')}}" defer></script>
+
 <div class="edit-form monospaced-textarea">

    <div class="tabs collapsable">
@@ -111,7 +114,8 @@
                    </div>
                {% if form.proxy %}
                    <div class="pure-control-group inline-radio">
-                        {{ render_field(form.proxy, class="fetch-backend-proxy") }}
+                          <div>{{ form.proxy.label }} <a href="" id="check-all-proxies" class="pure-button button-secondary button-xsmall" >Check/Scan all</a></div>
+                          <div>{{ form.proxy(class="fetch-backend-proxy") }}</div>
                        <span class="pure-form-message-inline">
                        Choose a proxy for this watch
                        </span>
--- a/changedetectionio/templates/preview.html
+++ b/changedetectionio/templates/preview.html
@@ -6,6 +6,7 @@
    {% if last_error_screenshot %}
    const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}";
    {% endif %}
+    const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}";
 </script>
 <script src="{{url_for('static_content', group='js', filename='diff-overview.js')}}" defer></script>

@@ -20,7 +21,7 @@
        {% endif %}
    </ul>
 </div>
-
+<form><input type="hidden" name="csrf_token" value="{{ csrf_token() }}"></form>
 <div id="diff-ui">
    <div class="tab-pane-inner" id="error-text">
        <div class="snapshot-age error">{{watch.error_text_ctime|format_seconds_ago}} seconds ago</div>
@@ -36,11 +37,12 @@

    <div class="tab-pane-inner" id="text">
        <div class="snapshot-age">{{watch.snapshot_text_ctime|format_timestamp_timeago}}</div>
-        <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span>
+        <span class="ignored">Grey lines are ignored</span> <span class="triggered">Blue lines are triggers</span> <span class="tip"><strong>Pro-tip</strong>: Highlight text to add to ignore filters</span>
+
        <table>
            <tbody>
            <tr>
-                <td id="diff-col">
+                <td id="diff-col" class="highlightable-filter">
                    {% for row in content %}
                    <div class="{{row.classes}}">{{row.line}}</div>
                    {% endfor %}
--- a/changedetectionio/templates/settings.html
+++ b/changedetectionio/templates/settings.html
@@ -62,14 +62,6 @@
                        <span class="pure-form-message-inline">Allow access to view watch diff page when password is enabled (Good for sharing the diff page)
                        </span>
                    </div>
-                    <div class="pure-control-group">
-                        {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
-                        class="m-d") }}
-                        <span class="pure-form-message-inline">
-                            Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notifications and RSS links.<br>Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"),
-                            <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
-                        </span>
-                    </div>
                    <div class="pure-control-group">
                        {{ render_field(form.application.form.pager_size) }}
                        <span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
@@ -100,6 +92,13 @@
                        {{ render_common_settings_form(form.application.form, emailprefix, settings_application) }}
                    </div>
                </fieldset>
+                <div class="pure-control-group" id="notification-base-url">
+                    {{ render_field(form.application.form.base_url, class="m-d") }}
+                    <span class="pure-form-message-inline">
+                        Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notification links.<br>
+                        Default value is the system environment variable '<code>BASE_URL</code>' - <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
+                    </span>
+                </div>
            </div>

            <div class="tab-pane-inner" id="fetching">
@@ -181,20 +180,56 @@ nav
                </div>
            </div>
            <div class="tab-pane-inner" id="proxies">
+                <div id="recommended-proxy">
+                    <div>
+                        <img style="height: 2em;" src="{{url_for('static_content', group='images', filename='brightdata.svg')}}" alt="BrightData Proxy Provider">
+                        <p>BrightData offer world-class proxy services, "Data Center" proxies are a very affordable way to proxy your requests, whilst <strong><a href="https://brightdata.grsm.io/n0r16zf7eivq">WebUnlocker</a></strong> can help solve most CAPTCHAs.</p>
+                        <p>
+                            BrightData offer many <a href="https://brightdata.com/proxy-types" target="new">many different types of proxies</a>, it is worth reading about what is best for your use-case.
+                        </p>

-                <p><strong>Tip</strong>: You can connect to websites using <a href="https://brightdata.grsm.io/n0r16zf7eivq">BrightData</a> proxies, their service <strong>WebUnlocker</strong> will solve most CAPTCHAs, whilst their <strong>Residential Proxies</strong> may help to avoid CAPTCHA altogether. </p>
-                <p>It may be easier to try <strong>WebUnlocker</strong> first, WebUnlocker also supports country selection.</p>
+                        <p>
+                            When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the access Proxy URL into the "Extra Proxies" boxes below.<br>
+                        </p>
+                        <p>
+                            The Proxy URL with BrightData should start with <code>http://brd-customer...</code>
+                        </p>
+                        <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p>
+                    </div>
+                    <div>
+                        <img style="height: 2em;"
+                             src="{{url_for('static_content', group='images', filename='oxylabs.svg')}}"
+                             alt="Oxylabs Proxy Provider">
+                        <p>
+                            Collect public data at scale with industry-leading web scraping solutions and the world’s
+                            largest ethical proxy network.
+                        </p>
+                        <p>
+                            Oxylabs also provide a <a href="https://oxylabs.io/products/web-unblocker"><strong>WebUnlocker</strong></a>
+                            proxy that bypasses sophisticated anti-bot systems, so you don’t have to.<br>
+                        </p>
+                        <p>
+                            Serve over <a href="https://oxylabs.io/location-proxy">195 countries</a>, providing <a
+                                href="https://oxylabs.io/products/residential-proxy-pool">Residential</a>, <a
+                                href="https://oxylabs.io/products/mobile-proxies">Mobile</a> and <a
+                                href="https://oxylabs.io/products/rotating-isp-proxies">ISP proxies</a> and much more.
+                        </p>
+                        <p>
+                            Use the promo code <strong>boost35</strong> with this link <a href="https://oxylabs.go2cloud.org/SH2d">https://oxylabs.go2cloud.org/SH2d</a> for 35% off Residential, Mobile proxies, Web Unblocker, and Scraper APIs. Built-in proxies enable you to access data from all around the world and help overcome anti-bot solutions.
+
+                        </p>
+
+                        
+                    </div>
+                </div>
                <p>
-                    When you have <a href="https://brightdata.grsm.io/n0r16zf7eivq">registered</a>, enabled the required services, visit the <A href="https://brightdata.com/cp/api_example?">API example page</A>, then select <strong>Python</strong>, set the country you wish to use, then copy+paste the example URL below<br>
-                    The Proxy URL with BrightData should start with <code>http://brd-customer...</code>
+                    Your proxy provider may need to whitelist our IP of <code>204.15.192.195</code>
                </p>
-
-                <p>When you sign up using <a href="https://brightdata.grsm.io/n0r16zf7eivq">https://brightdata.grsm.io/n0r16zf7eivq</a> BrightData will match any first deposit up to $150</p>
-
+               <p><strong>Tip</strong>: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites.

                <div class="pure-control-group">
-                    {{ render_field(form.requests.form.extra_proxies) }}
-                    <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span>
+                {{ render_field(form.requests.form.extra_proxies) }}
+                <span class="pure-form-message-inline">"Name" will be used for selecting the proxy in the Watch Edit settings</span>
                </div>
            </div>
            <div id="actions">
--- a/changedetectionio/tests/proxy_list/test_noproxy.py
+++ b/changedetectionio/tests/proxy_list/test_noproxy.py
@@ -0,0 +1,77 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client
+
+
+def test_noproxy_option(client, live_server):
+    live_server_setup(live_server)
+    # Run by run_proxy_tests.sh
+    # Call this URL then scan the containers that it never went through them
+    url = "http://noproxy.changedetection.io"
+
+    # Should only be available when a proxy is setup
+    res = client.get(
+        url_for("edit_page", uuid="first", unpause_on_save=1))
+    assert b'No proxy' not in res.data
+
+    # Setup a proxy
+    res = client.post(
+        url_for("settings_page"),
+        data={
+            "requests-time_between_check-minutes": 180,
+            "application-ignore_whitespace": "y",
+            "application-fetch_backend": "html_requests",
+            "requests-extra_proxies-0-proxy_name": "custom-one-proxy",
+            "requests-extra_proxies-0-proxy_url": "http://test:awesome@squid-one:3128",
+            "requests-extra_proxies-1-proxy_name": "custom-two-proxy",
+            "requests-extra_proxies-1-proxy_url": "http://test:awesome@squid-two:3128",
+            "requests-extra_proxies-2-proxy_name": "custom-proxy",
+            "requests-extra_proxies-2-proxy_url": "http://test:awesome@squid-custom:3128",
+        },
+        follow_redirects=True
+    )
+
+    assert b"Settings updated." in res.data
+
+    # Should be available as an option
+    res = client.get(
+        url_for("settings_page", unpause_on_save=1))
+    assert b'No proxy' in res.data
+
+
+    # This will add it paused
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
+        follow_redirects=True
+    )
+    assert b"Watch added in Paused state, saving will unpause" in res.data
+    uuid = extract_UUID_from_client(client)
+    res = client.get(
+        url_for("edit_page", uuid=uuid, unpause_on_save=1))
+    assert b'No proxy' in res.data
+
+    res = client.post(
+        url_for("edit_page", uuid=uuid, unpause_on_save=1),
+        data={
+                "include_filters": "",
+                "fetch_backend": "html_requests",
+                "headers": "",
+                "proxy": "no-proxy",
+                "tags": "",
+                "url": url,
+              },
+        follow_redirects=True
+    )
+    assert b"unpaused" in res.data
+    wait_for_all_checks(client)
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+    # Now the request should NOT appear in the second-squid logs (handled by the run_test_proxies.sh script)
+
+    # Prove that it actually checked
+
+    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] != 0
+
--- a/changedetectionio/tests/smtp/smtp-test-server.py
+++ b/changedetectionio/tests/smtp/smtp-test-server.py
@@ -0,0 +1,42 @@
+#!/usr/bin/python3
+import smtpd
+import asyncore
+
+# Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
+
+last_received_message = b"Nothing"
+
+
+class CustomSMTPServer(smtpd.SMTPServer):
+
+    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
+        global last_received_message
+        last_received_message = data
+        print('Receiving message from:', peer)
+        print('Message addressed from:', mailfrom)
+        print('Message addressed to  :', rcpttos)
+        print('Message length        :', len(data))
+        print(data.decode('utf8'))
+        return
+
+
+# Just print out the last message received on plain TCP socket server
+class EchoServer(asyncore.dispatcher):
+
+    def __init__(self, host, port):
+        asyncore.dispatcher.__init__(self)
+        self.create_socket()
+        self.set_reuse_addr()
+        self.bind((host, port))
+        self.listen(5)
+
+    def handle_accepted(self, sock, addr):
+        global last_received_message
+        print('Incoming connection from %s' % repr(addr))
+        sock.send(last_received_message)
+        last_received_message = b''
+
+
+server = CustomSMTPServer(('0.0.0.0', 11025), None)  # SMTP mail goes here
+server2 = EchoServer('0.0.0.0', 11080)  # Echo back last message received
+asyncore.loop()
--- a/changedetectionio/tests/smtp/test_notification_smtp.py
+++ b/changedetectionio/tests/smtp/test_notification_smtp.py
@@ -0,0 +1,165 @@
+import json
+import os
+import time
+import re
+from flask import url_for
+from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
+    wait_for_all_checks, \
+    set_longer_modified_response
+from changedetectionio.tests.util import extract_UUID_from_client
+import logging
+import base64
+
+# NOTE - RELIES ON mailserver as hostname running, see github build recipes
+smtp_test_server = 'mailserver'
+
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+    valid_notification_formats,
+)
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def get_last_message_from_smtp_server():
+    import socket
+    global smtp_test_server
+    port = 11080  # socket server port number
+
+    client_socket = socket.socket()  # instantiate
+    client_socket.connect((smtp_test_server, port))  # connect to the server
+
+    data = client_socket.recv(50024).decode()  # receive response
+    client_socket.close()  # close the connection
+    return data
+
+
+# Requires running the test SMTP server
+
+def test_check_notification_email_formats_default_HTML(client, live_server):
+    # live_server_setup(live_server)
+    set_original_response()
+
+    global smtp_test_server
+    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_body": "fallback-body<br> " + default_notification_body,
+              "application-notification_format": 'HTML',
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Add a watch and trigger a HTTP POST
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": 'nice one'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    wait_for_all_checks(client)
+    set_longer_modified_response()
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    time.sleep(3)
+
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
+
+    # The email should have two bodies, and the text/html part should be <br>
+    assert 'Content-Type: text/plain' in msg
+    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+    assert 'Content-Type: text/html' in msg
+    assert '(added) So let\'s see what happens.<br>' in msg  # the html part
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+
+def test_check_notification_email_formats_default_Text_override_HTML(client, live_server):
+    # live_server_setup(live_server)
+
+    # HTML problems? see this
+    # https://github.com/caronc/apprise/issues/633
+
+    set_original_response()
+    global smtp_test_server
+    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_body": default_notification_body,
+              "application-notification_format": 'Text',
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Add a watch and trigger a HTTP POST
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": 'nice one'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    wait_for_all_checks(client)
+    set_longer_modified_response()
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    time.sleep(3)
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
+    #    with open('/tmp/m.txt', 'w') as f:
+    #        f.write(msg)
+
+    # The email should not have two bodies, should be TEXT only
+
+    assert 'Content-Type: text/plain' in msg
+    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+
+    set_original_response()
+    # Now override as HTML format
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "notification_format": 'HTML',
+            'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+    wait_for_all_checks(client)
+
+    time.sleep(3)
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
+
+    # The email should have two bodies, and the text/html part should be <br>
+    assert 'Content-Type: text/plain' in msg
+    assert '(removed) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+    assert 'Content-Type: text/html' in msg
+    assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
--- a/changedetectionio/tests/test_api.py
+++ b/changedetectionio/tests/test_api.py
@@ -267,7 +267,7 @@ def test_api_watch_PUT_update(client, live_server):

    #live_server_setup(live_server)
    api_key = extract_api_key_from_UI(client)
-    time.sleep(1)
+
    # Create a watch
    set_original_response()
    test_url = url_for('test_endpoint', _external=True,
@@ -283,7 +283,6 @@ def test_api_watch_PUT_update(client, live_server):

    assert res.status_code == 201

-    time.sleep(1)

    # Get a listing, it will be the first one
    res = client.get(
--- a/changedetectionio/tests/test_backup.py
+++ b/changedetectionio/tests/test_backup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python3

-from .util import set_original_response, set_modified_response, live_server_setup
+from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks
 from flask import url_for
 from urllib.request import urlopen
 from zipfile import ZipFile
@@ -19,12 +19,12 @@ def test_backup(client, live_server):
    # Add our URL to the import page
    res = client.post(
        url_for("import_page"),
-        data={"urls": url_for('test_endpoint', _external=True)},
+        data={"urls": url_for('test_endpoint', _external=True)+"?somechar=őőőőőőőő"},
        follow_redirects=True
    )

    assert b"1 Imported" in res.data
-    time.sleep(3)
+    wait_for_all_checks(client)

    res = client.get(
        url_for("get_backup"),
--- a/changedetectionio/tests/test_filter_exist_changes.py
+++ b/changedetectionio/tests/test_filter_exist_changes.py
@@ -15,7 +15,7 @@ def set_response_without_filter():
     <p>Which is across multiple lines</p>
     <br>
     So let's see what happens.  <br>
-     <div id="nope-doesnt-exist">Some text thats the same</div>     
+     <div id="nope-doesnt-exist">Some text thats the same</div>
     </body>
     </html>
    """
@@ -32,7 +32,7 @@ def set_response_with_filter():
     <p>Which is across multiple lines</p>
     <br>
     So let's see what happens.  <br>
-     <div class="ticket-available">Ticket now on sale!</div>     
+     <div class="ticket-available">Ticket now on sale!</div>
     </body>
     </html>
    """
@@ -84,6 +84,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
                                                   "Snapshot: {{current_snapshot}}\n"
                                                   "Diff: {{diff}}\n"
                                                   "Diff Full: {{diff_full}}\n"
+                                                   "Diff as Patch: {{diff_patch}}\n"
                                                   ":-)",
                              "notification_format": "Text"}

--- a/changedetectionio/tests/test_filter_failure_notification.py
+++ b/changedetectionio/tests/test_filter_failure_notification.py
@@ -12,7 +12,7 @@ def set_response_with_filter():
     <p>Which is across multiple lines</p>
     <br>
     So let's see what happens.  <br>
-     <div id="nope-doesnt-exist">Some text thats the same</div>     
+     <div id="nope-doesnt-exist">Some text thats the same</div>
     </body>
     </html>
    """
@@ -66,6 +66,7 @@ def run_filter_test(client, content_filter):
                                                   "Snapshot: {{current_snapshot}}\n"
                                                   "Diff: {{diff}}\n"
                                                   "Diff Full: {{diff_full}}\n"
+                                                   "Diff as Patch: {{diff_patch}}\n"
                                                   ":-)",
                              "notification_format": "Text"}

--- a/changedetectionio/tests/test_group.py
+++ b/changedetectionio/tests/test_group.py
@@ -2,7 +2,7 @@

 import time
 from flask import url_for
-from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name
+from .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name, extract_UUID_from_client
 import os


@@ -15,7 +15,7 @@ def set_original_response():
     Some initial text<br>
     <p id="only-this">Should be only this</p>
     <br>
-     <p id="not-this">And never this</p>     
+     <p id="not-this">And never this</p>
     </body>
     </html>
    """
@@ -30,7 +30,7 @@ def set_modified_response():
     Some initial text<br>
     <p id="only-this">Should be REALLY only this</p>
     <br>
-     <p id="not-this">And never this</p>     
+     <p id="not-this">And never this</p>
     </body>
     </html>
    """
@@ -154,6 +154,10 @@ def test_tag_add_in_ui(client, live_server):
    )
    assert b"Tag added" in res.data
    assert b"new-test-tag" in res.data
+
+    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
+    assert b'All tags deleted' in res.data
+
    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
    assert b'Deleted' in res.data

@@ -185,6 +189,7 @@ def test_group_tag_notification(client, live_server):
                                                   "Diff Added: {{diff_added}}\n"
                                                   "Diff Removed: {{diff_removed}}\n"
                                                   "Diff Full: {{diff_full}}\n"
+                                                   "Diff as Patch: {{diff_patch}}\n"
                                                   ":-)",
                              "notification_screenshot": True,
                              "notification_format": "Text",
@@ -219,12 +224,10 @@ def test_group_tag_notification(client, live_server):
    assert "test-tag" in notification_submission
    assert "other-tag" in notification_submission

-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
    #@todo Test that multiple notifications fired
    #@todo Test that each of multiple notifications with different settings
-
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

 def test_limit_tag_ui(client, live_server):
    #live_server_setup(live_server)
@@ -260,3 +263,61 @@ def test_limit_tag_ui(client, live_server):
    assert b'test-tag' in res.data
    assert res.data.count(b'processor-text_json_diff') == 20
    assert b"object at" not in res.data
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
+    assert b'All tags deleted' in res.data
+def test_clone_tag_on_import(client, live_server):
+    #live_server_setup(live_server)
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url + " test-tag, another-tag\r\n"},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    res = client.get(url_for("index"))
+    assert b'test-tag' in res.data
+    assert b'another-tag' in res.data
+
+    watch_uuid = extract_UUID_from_client(client)
+    res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
+
+    assert b'Cloned' in res.data
+    # 2 times plus the top link to tag
+    assert res.data.count(b'test-tag') == 3
+    assert res.data.count(b'another-tag') == 3
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+def test_clone_tag_on_quickwatchform_add(client, live_server):
+    #live_server_setup(live_server)
+
+    test_url = url_for('test_endpoint', _external=True)
+
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": ' test-tag, another-tag      '},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    res = client.get(url_for("index"))
+    assert b'test-tag' in res.data
+    assert b'another-tag' in res.data
+
+    watch_uuid = extract_UUID_from_client(client)
+    res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True)
+
+    assert b'Cloned' in res.data
+    # 2 times plus the top link to tag
+    assert res.data.count(b'test-tag') == 3
+    assert res.data.count(b'another-tag') == 3
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+    res = client.get(url_for("tags.delete_all"), follow_redirects=True)
+    assert b'All tags deleted' in res.data
--- a/changedetectionio/tests/test_ignore_regex_text.py
+++ b/changedetectionio/tests/test_ignore_regex_text.py
@@ -15,11 +15,24 @@ def test_strip_regex_text_func():
    but sometimes we want to remove the lines.
    
    but 1 lines
+    skip 5 lines
+    really? yes man
+#/not this tries weirdly formed regex or just strings starting with /
+/not this
    but including 1234 lines
    igNORe-cAse text we dont want to keep    
    but not always."""

-    ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"]
+
+    ignore_lines = [
+        "sometimes",
+        "/\s\d{2,3}\s/",
+        "/ignore-case text/",
+        "really?",
+        "/skip \d lines/i",
+        "/not"
+    ]
+

    fetcher = fetch_site_status.perform_site_check(datastore=False)
    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)
@@ -27,4 +40,10 @@ def test_strip_regex_text_func():
    assert b"but 1 lines" in stripped_content
    assert b"igNORe-cAse text" not in stripped_content
    assert b"but 1234 lines" not in stripped_content
+    assert b"really" not in stripped_content
+    assert b"not this" not in stripped_content
+
+    # Check line number reporting
+    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines, mode="line numbers")
+    assert stripped_content == [2, 5, 6, 7, 8, 10]

--- a/changedetectionio/tests/test_ignorehighlighter.py
+++ b/changedetectionio/tests/test_ignorehighlighter.py
@@ -0,0 +1,57 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from .util import live_server_setup, wait_for_all_checks
+from changedetectionio import html_tools
+from . util import  extract_UUID_from_client
+
+def set_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text<br>
+     <p>Which is across multiple lines</p>
+     <br>
+     So let's see what happens.  <br>
+     <p>oh yeah 456</p>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_highlight_ignore(client, live_server):
+    live_server_setup(live_server)
+    set_original_ignore_response()
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Give the thread time to pick it up
+    wait_for_all_checks(client)
+    uuid = extract_UUID_from_client(client)
+    # use the highlighter endpoint
+    res = client.post(
+        url_for("highlight_submit_ignore_url", uuid=uuid),
+        data={"mode": 'digit-regex', 'selection': 'oh yeah 123'},
+        follow_redirects=True
+    )
+
+    res = client.get(url_for("edit_page", uuid=uuid))
+
+    # should be a regex now
+    assert b'/oh\ yeah\ \d+/' in res.data
+
+    # Should return a link
+    assert b'href' in res.data
+
+    # And it should register in the preview page
+    res = client.get(url_for("preview_page", uuid=uuid))
+    assert b'<div class="ignored">oh yeah 456' in res.data
--- a/changedetectionio/tests/test_import.py
+++ b/changedetectionio/tests/test_import.py
@@ -112,6 +112,7 @@ def test_import_distillio(client, live_server):
    # did the tags work?
    res = client.get( url_for("index"))

+    # check tags
    assert b"nice stuff" in res.data
    assert b"nerd-news" in res.data

--- a/changedetectionio/tests/test_notification.py
+++ b/changedetectionio/tests/test_notification.py
@@ -3,7 +3,8 @@ import os
 import time
 import re
 from flask import url_for
-from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
+    set_longer_modified_response
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -24,9 +25,6 @@ def test_check_notification(client, live_server):
    #live_server_setup(live_server)
    set_original_response()

-    # Give the endpoint time to spin up
-    time.sleep(1)
-
    # Re 360 - new install should have defaults set
    res = client.get(url_for("settings_page"))
    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
@@ -100,6 +98,7 @@ def test_check_notification(client, live_server):
                                                   "Diff Added: {{diff_added}}\n"
                                                   "Diff Removed: {{diff_removed}}\n"
                                                   "Diff Full: {{diff_full}}\n"
+                                                   "Diff as Patch: {{diff_patch}}\n"
                                                   ":-)",
                              "notification_screenshot": True,
                              "notification_format": "Text"}
@@ -142,8 +141,7 @@ def test_check_notification(client, live_server):

    # Did we see the URL that had a change, in the notification?
    # Diff was correctly executed
-    assert test_url in notification_submission
-    assert ':-)' in notification_submission
+
    assert "Diff Full: Some initial text" in notification_submission
    assert "Diff: (changed) Which is across multiple lines" in notification_submission
    assert "(into) which has this one new line" in notification_submission
@@ -156,7 +154,8 @@ def test_check_notification(client, live_server):
    assert "preview/" in notification_submission
    assert ":-)" in notification_submission
    assert "New ChangeDetection.io Notification - {}".format(test_url) in notification_submission
-
+    assert test_url in notification_submission
+    assert ':-)' in notification_submission
    # Check the attachment was added, and that it is a JPEG from the original PNG
    notification_submission_object = json.loads(notification_submission)
    # We keep PNG screenshots for now
@@ -275,7 +274,7 @@ def test_notification_validation(client, live_server):


 def test_notification_custom_endpoint_and_jinja2(client, live_server):
-    time.sleep(1)
+    #live_server_setup(live_server)

    # test_endpoint - that sends the contents of a file
    # test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt)
@@ -286,12 +285,14 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):

    res = client.post(
        url_for("settings_page"),
-        data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
-              "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }',
-              # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-              "application-notification_urls": test_notification_url,
+        data={
+              "application-fetch_backend": "html_requests",
              "application-minutes_between_check": 180,
-              "application-fetch_backend": "html_requests"
+              "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }',
+              "application-notification_format": default_notification_format,
+              "application-notification_urls": test_notification_url,
+              # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+              "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
              },
        follow_redirects=True
    )
@@ -316,9 +317,8 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(2)

-
    with open("test-datastore/notification.txt", 'r') as f:
-        x=f.read()
+        x = f.read()
        j = json.loads(x)
        assert j['url'].startswith('http://localhost')
        assert j['secret'] == 444
@@ -329,5 +329,9 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
        notification_url = f.read()
        assert 'xxx=http' in notification_url

-    os.unlink("test-datastore/notification-url.txt")
+    # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
+    assert os.path.isfile("test-datastore/notification-content-type.txt")
+    with open("test-datastore/notification-content-type.txt", 'r') as f:
+        assert 'application/json' in f.read()

+    os.unlink("test-datastore/notification-url.txt")
--- a/changedetectionio/tests/unit/test_notification_diff.py
+++ b/changedetectionio/tests/unit/test_notification_diff.py
@@ -36,7 +36,7 @@ class TestDiffBuilder(unittest.TestCase):
        output = output.split("\n")
        self.assertIn('(removed) for having learned computerese,', output)
        self.assertIn('(removed) I continue to examine bits, bytes and words', output)
-        
+
        #diff_removed
        with open(base_dir + "/test-content/before.txt", 'r') as f:
            previous_version_file_contents = f.read()
@@ -49,7 +49,7 @@ class TestDiffBuilder(unittest.TestCase):
        self.assertIn('(into) xok', output)
        self.assertIn('(into) next-x-ok', output)
        self.assertNotIn('(added) and something new', output)
-        
+
        #diff_removed
        with open(base_dir + "/test-content/after-2.txt", 'r') as f:
            newest_version_file_contents = f.read()
@@ -57,9 +57,25 @@ class TestDiffBuilder(unittest.TestCase):
        output = output.split("\n")
        self.assertIn('(removed) for having learned computerese,', output)
        self.assertIn('(removed) I continue to examine bits, bytes and words', output)
-        
+
+    def test_expected_diff_patch_output(self):
+        base_dir = os.path.dirname(__file__)
+        with open(base_dir + "/test-content/before.txt", 'r') as f:
+            before = f.read()
+        with open(base_dir + "/test-content/after.txt", 'r') as f:
+            after = f.read()
+
+        output = diff.render_diff(previous_version_file_contents=before,
+                                  newest_version_file_contents=after,
+                                  patch_format=True)
+        output = output.split("\n")
+
+        self.assertIn('-ok', output)
+        self.assertIn('+xok', output)
+        self.assertIn('+next-x-ok', output)
+        self.assertIn('+and something new', output)

        # @todo test blocks of changed, blocks of added, blocks of removed

 if __name__ == '__main__':
-    unittest.main()
+    unittest.main()
--- a/changedetectionio/tests/util.py
+++ b/changedetectionio/tests/util.py
@@ -38,7 +38,25 @@ def set_modified_response():
        f.write(test_return_data)

    return None
+def set_longer_modified_response():
+    test_return_data = """<html>
+    <head><title>modified head title</title></head>
+    <body>
+     Some initial text<br>
+     <p>which has this one new line</p>
+     <br>
+     So let's see what happens.  <br>
+     So let's see what happens.  <br>
+      So let's see what happens.  <br>
+     So let's see what happens.  <br>           
+     </body>
+     </html>
+    """

+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
 def set_more_modified_response():
    test_return_data = """<html>
    <head><title>modified head title</title></head>
@@ -187,6 +205,10 @@ def live_server_setup(live_server):
        with open("test-datastore/notification-url.txt", "w") as f:
            f.write(request.url)

+        if request.content_type:
+            with open("test-datastore/notification-content-type.txt", "w") as f:
+                f.write(request.content_type)
+
        print("\n>> Test notification endpoint was hit.\n", data)
        return "Text was set"

--- a/changedetectionio/update_worker.py
+++ b/changedetectionio/update_worker.py
@@ -32,15 +32,17 @@ class update_worker(threading.Thread):

        watch_history = watch.history
        dates = list(watch_history.keys())
+        # Add text that was triggered
+        snapshot_contents = watch.get_history_snapshot(dates[-1])

        # HTML needs linebreak, but MarkDown and Text can use a linefeed
        if n_object['notification_format'] == 'HTML':
            line_feed_sep = "<br>"
+            # Snapshot will be plaintext on the disk, convert to some kind of HTML
+            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
        else:
            line_feed_sep = "\n"

-        # Add text that was triggered
-        snapshot_contents = watch.get_history_snapshot(dates[-1])
        trigger_text = watch.get('trigger_text', [])
        triggered_text = ''

@@ -56,6 +58,7 @@ class update_worker(threading.Thread):
            'diff': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep),
            'diff_added': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_removed=False, line_feed_sep=line_feed_sep),
            'diff_full': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_equal=True, line_feed_sep=line_feed_sep),
+            'diff_patch': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), line_feed_sep=line_feed_sep, patch_format=True),
            'diff_removed': diff.render_diff(watch.get_history_snapshot(dates[-2]), watch.get_history_snapshot(dates[-1]), include_added=False, line_feed_sep=line_feed_sep),
            'screenshot': watch.get_screenshot() if watch.get('notification_screenshot') else None,
            'triggered_text': triggered_text,
@@ -65,15 +68,48 @@ class update_worker(threading.Thread):
        logging.info (">> SENDING NOTIFICATION")
        self.notification_q.put(n_object)

+    # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
+    def _check_cascading_vars(self, var_name, watch):
+
+        from changedetectionio.notification import (
+            default_notification_format_for_watch,
+            default_notification_body,
+            default_notification_title
+        )
+
+
+        # Would be better if this was some kind of Object where Watch can reference the parent datastore etc
+        v = watch.get(var_name)
+        if v and not watch.get('notification_muted'):
+            if var_name == 'notification_format' and v == default_notification_format_for_watch:
+                return self.datastore.data['settings']['application'].get('notification_format')
+
+            return v
+
+        tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
+        if tags:
+            for tag_uuid, tag in tags.items():
+                v = tag.get(var_name)
+                if v and not tag.get('notification_muted'):
+                    return v
+
+        if self.datastore.data['settings']['application'].get(var_name):
+            return self.datastore.data['settings']['application'].get(var_name)
+
+        # Otherwise could be defaults
+        if var_name == 'notification_format':
+            return default_notification_format_for_watch
+        if var_name == 'notification_body':
+            return default_notification_body
+        if var_name == 'notification_title':
+            return default_notification_title
+
+        return None

    def send_content_changed_notification(self, watch_uuid):

-        from changedetectionio.notification import (
-            default_notification_format_for_watch
-        )
-
        n_object = {}
-        watch = self.datastore.data['watching'].get(watch_uuid, False)
+        watch = self.datastore.data['watching'].get(watch_uuid)
        if not watch:
            return

@@ -87,57 +123,20 @@ class update_worker(threading.Thread):
            )

        # Should be a better parent getter in the model object
+
        # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
-        n_object['notification_urls'] = watch.get('notification_urls')
-
-        n_object['notification_title'] = watch['notification_title'] if watch['notification_title'] else \
-            self.datastore.data['settings']['application']['notification_title']
-
-        n_object['notification_body'] = watch['notification_body'] if watch['notification_body'] else \
-            self.datastore.data['settings']['application']['notification_body']
-
-        n_object['notification_format'] = watch['notification_format'] if watch['notification_format'] != default_notification_format_for_watch else \
-            self.datastore.data['settings']['application']['notification_format']
+        n_object['notification_urls'] = self._check_cascading_vars('notification_urls', watch)
+        n_object['notification_title'] = self._check_cascading_vars('notification_title', watch)
+        n_object['notification_body'] = self._check_cascading_vars('notification_body', watch)
+        n_object['notification_format'] = self._check_cascading_vars('notification_format', watch)

        # (Individual watch) Only prepare to notify if the rules above matched
-        sent = False
-        if 'notification_urls' in n_object and n_object['notification_urls']:
-            sent = True
+        queued = False
+        if n_object and n_object.get('notification_urls'):
+            queued = True
            self.queue_notification_for_watch(n_object, watch)

-        # (Group tags) try by group tag
-        if not sent:
-            # Else, Try by tag, and use system default vars for format, body etc as fallback
-            tags = self.datastore.get_all_tags_for_watch(uuid=watch_uuid)
-            for tag_uuid, tag in tags.items():
-                n_object = {}
-                n_object['notification_urls'] = tag.get('notification_urls')
-
-                n_object['notification_title'] = tag.get('notification_title') if tag.get('notification_title') else \
-                    self.datastore.data['settings']['application']['notification_title']
-
-                n_object['notification_body'] = tag.get('notification_body') if tag.get('notification_body') else \
-                    self.datastore.data['settings']['application']['notification_body']
-
-                n_object['notification_format'] = tag.get('notification_format') if tag.get('notification_format') != default_notification_format_for_watch else \
-                    self.datastore.data['settings']['application']['notification_format']
-
-                if 'notification_urls' in n_object and n_object.get('notification_urls') and not tag.get('notification_muted'):
-                    sent = True
-                    self.queue_notification_for_watch(n_object, watch)
-
-        # (Group tags) try by global
-        if not sent:
-            # leave this as is, but repeat in a loop for each tag also
-            n_object['notification_urls'] = self.datastore.data['settings']['application'].get('notification_urls')
-            n_object['notification_title'] = self.datastore.data['settings']['application'].get('notification_title')
-            n_object['notification_body'] = self.datastore.data['settings']['application'].get('notification_body')
-            n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')
-            if n_object.get('notification_urls') and n_object.get('notification_body') and n_object.get('notification_title'):
-                sent = True
-                self.queue_notification_for_watch(n_object, watch)
-
-        return sent
+        return queued


    def send_filter_failure_notification(self, watch_uuid):
@@ -381,6 +380,9 @@ class update_worker(threading.Thread):
                        if not self.datastore.data['watching'][uuid].get('ignore_status_codes'):
                            update_obj['consecutive_filter_failures'] = 0

+                        # Everything ran OK, clean off any previous error
+                        update_obj['last_error'] = False
+
                        self.cleanup_error_artifacts(uuid)

                    #
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -47,6 +47,9 @@ services:
  #
  #        Hides the `Referer` header so that monitored websites can't see the changedetection.io hostname.
  #      - HIDE_REFERER=true
+  #        
+  #        Default number of parallel/concurrent fetchers
+  #      - FETCH_WORKERS=10

      # Comment out ports: when using behind a reverse proxy , enable networks: etc.
      ports:
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,8 @@ flask~=2.0
 inscriptis~=2.2
 pytz
 timeago~=1.0
-validators
+validators~=0.21
+

 # Set these versions together to avoid a RequestsDependencyWarning
 # >= 2.26 also adds Brotli support if brotli is installed
@@ -32,7 +33,7 @@ dnspython<2.3.0
 # jq not available on Windows so must be installed manually

 # Notification library
-apprise~=1.3.0
+apprise~=1.5.0

 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
@@ -71,3 +72,6 @@ pillow
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup
 pytest ~=7.2
 pytest-flask ~=1.2
+
+# Pin jsonschema version to prevent build errors on armv6 while rpds-py wheels aren't available (1708)
+jsonschema==4.17.3
Author	SHA1	Message	Date
dgtlmoon	460c724e51	0.45.2	2023-09-22 09:45:55 +02:00
dgtlmoon	dcf4bf37ed	Code/Test - Improve testing for creating backups	2023-09-22 09:21:07 +02:00
dgtlmoon	e3cf22fc27	UI - Re-order notification field settings	2023-09-14 14:34:44 +02:00
dgtlmoon	d497db639e	UI - Notifications - Tidyup - Hide the notification tokens but show with a button/link	2023-09-14 14:16:08 +02:00
dgtlmoon	7355ac8d21	UI - Notifications - Tweak discord help text	2023-09-14 13:55:48 +02:00
dgtlmoon	2f2d0ea0f2	RSS feeds - Fixing broken links from RSS index in some environments, refactor code (#152 , #148 , #1684 , #1798 )	2023-09-14 13:19:45 +02:00
dgtlmoon	a958e1fe20	UI - "recheck all" button should ignore blank/empty "tag" setting when set	2023-09-12 15:13:21 +02:00
dgtlmoon	5dc3b00ec6	Update README.md	2023-09-11 23:27:10 +02:00
dgtlmoon	8ac4757cd9	UI - Fix spelling error	2023-09-11 13:18:05 +02:00
dgtlmoon	2180bb256d	UI - Make tgram:// and discord:// examples in notification settings link to how-to pages (#1785 )	2023-09-11 10:22:35 +02:00
dgtlmoon	212f15ad5f	Catch possible crash scenario for listing watches - `date_created` was missing on add (#1787 )	2023-09-10 13:44:24 +02:00
dgtlmoon	22b2068208	Ability to select "No proxy" for a watch when you have proxy's configured	2023-09-08 14:14:47 +02:00
dgtlmoon	4916043055	Updating notification library - Adds support for Pushy, PushDeer, PushMe and Matrix attachment support (screenshots)	2023-09-08 12:40:23 +02:00
dgtlmoon	7bf13bad30	Update README.md	2023-09-07 16:46:21 +02:00
dgtlmoon	0aa2276afb	UI - Fixing update for sort by "date created" or "#" in watch overview table ( #1775 )	2023-09-07 10:36:34 +02:00
Tiago Ilieve	3b875e5a6a	Add 'diff_patch' notification body token - This will allow the diff to be generated in the "unified patch format." (#1765 )	2023-09-07 08:55:06 +02:00
Constantin Hong	8ec50294d2	README/docs: Clarifying xpath version changedetection.io uses (#1773 )	2023-09-07 08:49:26 +02:00
dgtlmoon	e3c9255d9e	0.45.1	2023-09-06 12:27:56 +02:00
dgtlmoon	3b03bdcb82	UI - Fixing open/broken HTML which was causing some buttons to not display	2023-09-06 12:27:27 +02:00
dgtlmoon	e25792bcec	0.45	2023-09-06 09:46:27 +02:00
dgtlmoon	bf4168a2aa	Adding Oxylabs proxy recommendation to proxy config page (#1756 )	2023-09-06 09:43:23 +02:00
dgtlmoon	9d37eaa57b	Fix - Link in the RSS feed was showing the path twice (when used in reverse proxy)	2023-09-05 17:28:13 +02:00
dgtlmoon	40d01acde9	Fix - Regular Expression text in `ignore` and `trigger` were not processing correctly, also refactored for lower CPU usage (#1747 )	2023-09-05 13:07:17 +02:00
Ikko Eltociear Ashimine	d34832de73	Fix typo in README.md (#1759 )	2023-09-04 16:40:12 +02:00
dgtlmoon	ed4bafae63	UI - "Test notification" button in "Group Tag" settings page was broken due to missing variable #1753	2023-08-31 13:29:38 +02:00
dgtlmoon	3a5bceadfa	UI - Clicking 'ignore text' when highlighting text should clear the preview text button/area. #1754	2023-08-31 13:24:19 +02:00
dgtlmoon	6abdf2d332	Update documentation - How to set number of concurrent fetchers	2023-08-30 18:02:10 +02:00
dgtlmoon	dee23709a9	0.44.2	2023-08-28 19:01:59 +02:00
dgtlmoon	52df3b10e7	UI - Ability to highlight text and have it offered as a ignore-text option, really nice easy way to set ignores on changing text (#1746 )	2023-08-24 14:29:48 +02:00
dgtlmoon	087d21c61e	Update README.md	2023-08-22 11:36:15 +02:00
dgtlmoon	171faf465c	Enable ARMv8 builds (for RaspberryPi and other portable devices) (#1733 )	2023-08-13 23:33:49 +02:00
dgtlmoon	a3d8bd0b1a	Updating in app links	2023-08-13 18:35:58 +02:00
dgtlmoon	6ef8a1c18f	Updating URL validation library, ability to block access to simple (no dot) hostnames like "localhost" with BLOCK_SIMPLEHOSTS setting (#1732 )	2023-08-13 18:27:55 +02:00
Marcelo Alencar	126f0fbf87	Re-enable ARMv6 builds (for Raspberry and other portable devices) (#1724 )	2023-08-07 15:48:33 +02:00
dgtlmoon	cfa712c88c	0.44.1	2023-08-02 08:55:07 +02:00
dgtlmoon	6a6ba40b6a	Re-enable ARMv7 builds (for Raspberry and other portable devices)	2023-08-01 17:10:24 +02:00
dgtlmoon	e7f726c057	UI - Fixing darkmode switch icon	2023-07-24 14:06:40 +02:00
dgtlmoon	df0cc7b585	0.44	2023-07-17 18:03:42 +02:00
dgtlmoon	76cd98b521	Updating AppRise notification library, Improved pover, ntfy support, whatsapp updates, Pagertree support, Voip.ms support, Misskey support, plus many fixes and improvements.	2023-07-17 17:32:12 +02:00
dgtlmoon	f84ba0fb31	API - Updating API description for handling a single watch	2023-07-17 17:19:41 +02:00
dgtlmoon	c35cbd33d6	Removing docker build for RaspberryPi (armv6/armv7) for now due to packaging problems	2023-07-17 17:10:29 +02:00
dgtlmoon	661f7fe32c	Proxy scan improvements - handle custom proxies, dont restart when a scan is already running (#1689 )	2023-07-11 16:48:50 +02:00
dgtlmoon	7cb7eebbc5	Browser Steps - When cleaning up old screenshots, check the file exists	2023-07-11 10:44:54 +02:00
dgtlmoon	aaceb4ebad	Scan/Recheck proxies - Report filter not found as "OK" but with warning	2023-07-11 10:44:21 +02:00
dgtlmoon	56cf6e5ea5	Bug fix - Previously encountered fetch errors were sometimes not being cleared (#1687 )	2023-07-11 09:23:41 +02:00
dgtlmoon	1987e109e8	New feature - Helper button to trigger a scan/access test of all proxies for a particular watch (#1685 )	2023-07-10 16:08:45 +02:00
dgtlmoon	20d65cdd26	0.43.2	2023-06-30 22:57:05 +02:00
dgtlmoon	37ff5f6d37	Bug - SMTP mailto:// Notification content-type (HTML/Text) fix and add CI tests (#1660 )	2023-06-30 21:35:35 +02:00
dgtlmoon	2f777ea3bb	Fix - Watches werent falling back to global default formats correctly when required (#1656 )	2023-06-28 00:03:02 +02:00
dgtlmoon	e709201955	0.43.1	2023-06-27 18:28:18 +02:00
dgtlmoon	572f71299f	Bug fix - Notification settings were not cascading from global -> tags -> watch correctly in some cases (#1654 )	2023-06-27 18:27:33 +02:00
dgtlmoon	5f150c4f03	Bug - Fix watch clone (#1647 )	2023-06-27 17:05:32 +02:00