Ability to block access to simplehosts

.dockerignore

@@ -1,2 +1,18 @@
 .git
 .github
+changedetectionio/processors/__pycache__
+changedetectionio/api/__pycache__
+changedetectionio/model/__pycache__
+changedetectionio/blueprint/price_data_follower/__pycache__
+changedetectionio/blueprint/tags/__pycache__
+changedetectionio/blueprint/__pycache__
+changedetectionio/blueprint/browser_steps/__pycache__
+changedetectionio/fetchers/__pycache__
+changedetectionio/tests/visualselector/__pycache__
+changedetectionio/tests/restock/__pycache__
+changedetectionio/tests/__pycache__
+changedetectionio/tests/fetchers/__pycache__
+changedetectionio/tests/unit/__pycache__
+changedetectionio/tests/proxy_list/__pycache__
+changedetectionio/__pycache__
+

.github/workflows/test-container-build.yml

@@ -62,7 +62,7 @@ jobs:
           with:
             context: ./
             file: ./Dockerfile
-            platforms: linux/arm/v7,linux/arm/v6,linux/amd64,linux/arm64,
+            platforms: linux/arm/v6,linux/arm/v7,linux/amd64,linux/arm64,
             cache-from: type=local,src=/tmp/.buildx-cache
             cache-to: type=local,dest=/tmp/.buildx-cache
 

.github/workflows/test-only.yml

@@ -36,10 +36,16 @@ jobs:
         run: |         
           # Build a changedetection.io container and start testing inside
           docker build . -t test-changedetectionio
+          # Debug info
+          docker run test-changedetectionio  bash -c 'pip list'
+
+      - name: Spin up ancillary SMTP+Echo message test server
+        run: |
+          # Debug SMTP server/echo message back server
+          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py' 
 
       - name: Test built container with pytest
         run: |
-          
           # Unit tests
           docker run test-changedetectionio  bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'
           
@@ -63,6 +69,11 @@ jobs:
           # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it
           docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 
+      - name: Test SMTP notification mime types
+        run: |
+          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
+          docker run --rm  --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
+
       - name: Test with puppeteer fetcher and disk cache
         run: |
           docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'

MANIFEST.in

@@ -13,3 +13,6 @@ include changedetection.py
 global-exclude *.pyc
 global-exclude node_modules
 global-exclude venv
+
+global-exclude test-datastore
+global-exclude changedetection.io*dist-info

changedetectionio/__init__.py

@@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.43.1'
+__version__ = '0.44.1'
 
 datastore = None
 
@@ -1439,6 +1439,10 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.blueprint.tags as tags
     app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags')
 
+    import changedetectionio.blueprint.check_proxies as check_proxies
+    app.register_blueprint(check_proxies.construct_blueprint(datastore=datastore), url_prefix='/check_proxy')
+
+
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
     threading.Thread(target=notification_runner).start()

changedetectionio/api/api_v1.py

@@ -1,3 +1,6 @@
+import os
+from distutils.util import strtobool
+
 from flask_expects_json import expects_json
 from changedetectionio import queuedWatchMetaData
 from flask_restful import abort, Resource
@@ -33,7 +36,7 @@ class Watch(Resource):
     @auth.check_token
     def get(self, uuid):
         """
-        @api {get} /api/v1/watch/:uuid Get a single watch data
+        @api {get} /api/v1/watch/:uuid Single watch - get data, recheck, pause, mute.
         @apiDescription Retrieve watch information and set muted/paused status
         @apiExample {curl} Example usage:
             curl http://localhost:4000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091  -H"x-api-key:813031b16330fe25e3780cf0325daa45"
@@ -209,7 +212,9 @@ class CreateWatch(Resource):
         json_data = request.get_json()
         url = json_data['url'].strip()
 
-        if not validators.url(json_data['url'].strip()):
+        # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
+        if not validators.url(url, simple_host=allow_simplehost):
             return "Invalid or unsupported URL", 400
 
         if json_data.get('proxy'):

changedetectionio/blueprint/check_proxies/__init__.py

@@ -0,0 +1,116 @@
+from concurrent.futures import ThreadPoolExecutor
+
+from functools import wraps
+
+from flask import Blueprint
+from flask_login import login_required
+
+from changedetectionio.processors import text_json_diff
+from changedetectionio.store import ChangeDetectionStore
+
+
+STATUS_CHECKING = 0
+STATUS_FAILED = 1
+STATUS_OK = 2
+THREADPOOL_MAX_WORKERS = 3
+_DEFAULT_POOL = ThreadPoolExecutor(max_workers=THREADPOOL_MAX_WORKERS)
+
+
+# Maybe use fetch-time if its >5 to show some expected load time?
+def threadpool(f, executor=None):
+    @wraps(f)
+    def wrap(*args, **kwargs):
+        return (executor or _DEFAULT_POOL).submit(f, *args, **kwargs)
+
+    return wrap
+
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+    check_proxies_blueprint = Blueprint('check_proxies', __name__)
+    checks_in_progress = {}
+
+    @threadpool
+    def long_task(uuid, preferred_proxy):
+        import time
+        from changedetectionio import content_fetcher
+
+        status = {'status': '', 'length': 0, 'text': ''}
+        from jinja2 import Environment, BaseLoader
+
+        contents = ''
+        now = time.time()
+        try:
+            update_handler = text_json_diff.perform_site_check(datastore=datastore)
+            changed_detected, update_obj, contents = update_handler.run(uuid, preferred_proxy=preferred_proxy, skip_when_checksum_same=False)
+        # title, size is len contents not len xfer
+        except content_fetcher.Non200ErrorCodeReceived as e:
+            if e.status_code == 404:
+                status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but 404 (page not found)"})
+            elif e.status_code == 403 or e.status_code == 401:
+                status.update({'status': 'ERROR', 'length': len(contents), 'text': f"{e.status_code} - Access denied"})
+            else:
+                status.update({'status': 'ERROR', 'length': len(contents), 'text': f"Status code: {e.status_code}"})
+        except text_json_diff.FilterNotFoundInResponse:
+            status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but CSS/xPath filter not found (page changed layout?)"})
+        except content_fetcher.EmptyReply as e:
+            if e.status_code == 403 or e.status_code == 401:
+                status.update({'status': 'ERROR OTHER', 'length': len(contents), 'text': f"Got empty reply with code {e.status_code} - Access denied"})
+            else:
+                status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': f"Empty reply with code {e.status_code}, needs chrome?"})
+
+        except Exception as e:
+            status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': 'Error: '+str(e)})
+        else:
+            status.update({'status': 'OK', 'length': len(contents), 'text': ''})
+
+        if status.get('text'):
+            status['text'] = Environment(loader=BaseLoader()).from_string('{{text|e}}').render({'text': status['text']})
+
+        status['time'] = "{:.2f}s".format(time.time() - now)
+
+        return status
+
+    def _recalc_check_status(uuid):
+
+        results = {}
+        for k, v in checks_in_progress.get(uuid, {}).items():
+            try:
+                r_1 = v.result(timeout=0.05)
+            except Exception as e:
+                # If timeout error?
+                results[k] = {'status': 'RUNNING'}
+
+            else:
+                results[k] = r_1
+
+        return results
+
+    @login_required
+    @check_proxies_blueprint.route("/<string:uuid>/status", methods=['GET'])
+    def get_recheck_status(uuid):
+        results = _recalc_check_status(uuid=uuid)
+        return results
+
+    @login_required
+    @check_proxies_blueprint.route("/<string:uuid>/start", methods=['GET'])
+    def start_check(uuid):
+
+        if not datastore.proxy_list:
+            return
+
+        if checks_in_progress.get(uuid):
+            state = _recalc_check_status(uuid=uuid)
+            for proxy_key, v in state.items():
+                if v.get('status') == 'RUNNING':
+                    return state
+        else:
+            checks_in_progress[uuid] = {}
+
+        for k, v in datastore.proxy_list.items():
+            if not checks_in_progress[uuid].get(k):
+                checks_in_progress[uuid][k] = long_task(uuid=uuid, preferred_proxy=k)
+
+        results = _recalc_check_status(uuid=uuid)
+        return results
+
+    return check_proxies_blueprint

changedetectionio/content_fetcher.py

@@ -201,6 +201,7 @@ class Fetcher():
             dest = os.path.join(self.browser_steps_screenshot_path, 'step_*.jpeg')
             files = glob.glob(dest)
             for f in files:
+                if os.path.isfile(f):
                     os.unlink(f)
 
 

changedetectionio/diff.py

@@ -54,4 +54,5 @@ def render_diff(previous_version_file_contents, newest_version_file_contents, in
 
     # Recursively join lists
     f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
-    return f(rendered_diff)
+    p= f(rendered_diff)
+    return p

changedetectionio/forms.py

@@ -1,5 +1,6 @@
 import os
 import re
+from distutils.util import strtobool
 
 from wtforms import (
     BooleanField,
@@ -257,9 +258,10 @@ class validateURL(object):
 
     def __call__(self, form, field):
         import validators
-
+        # If hosts that only contain alphanumerics are allowed ("localhost" for example)
+        allow_simplehost = not strtobool(os.getenv('BLOCK_SIMPLEHOSTS', 'False'))
         try:
-            validators.url(field.data.strip())
+            validators.url(field.data.strip(), simple_host=allow_simplehost)
         except validators.ValidationFailure:
             message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
             raise ValidationError(message)

changedetectionio/notification.py

@@ -93,6 +93,12 @@ def process_notification(n_object, datastore):
         valid_notification_formats[default_notification_format],
     )
 
+    # If we arrived with 'System default' then look it up
+    if n_format == default_notification_format_for_watch and datastore.data['settings']['application'].get('notification_format') != default_notification_format_for_watch:
+        # Initially text or whatever
+        n_format = datastore.data['settings']['application'].get('notification_format', valid_notification_formats[default_notification_format])
+
+    
     # https://github.com/caronc/apprise/wiki/Development_LogCapture
     # Anything higher than or equal to WARNING (which covers things like Connection errors)
     # raise it as an exception
@@ -145,9 +151,12 @@ def process_notification(n_object, datastore):
                     # Apprise will default to HTML, so we need to override it
                     # So that whats' generated in n_body is in line with what is going to be sent.
                     # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-                    if not 'format=' in url and (n_format == 'text' or n_format == 'markdown'):
+                    if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
                         prefix = '?' if not '?' in url else '&'
+                        # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
+                        n_format = n_format.tolower()
                         url = "{}{}format={}".format(url, prefix, n_format)
+                    # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
 
                 apobj.add(url)
 

changedetectionio/processors/__init__.py

@@ -9,7 +9,7 @@ class difference_detection_processor():
         super().__init__(*args, **kwargs)
 
     @abstractmethod
-    def run(self, uuid, skip_when_checksum_same=True):
+    def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
         update_obj = {'last_notification_error': False, 'last_error': False}
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()

changedetectionio/processors/text_json_diff.py

@@ -50,7 +50,7 @@ class perform_site_check(difference_detection_processor):
 
         return regex
 
-    def run(self, uuid, skip_when_checksum_same=True):
+    def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
@@ -105,7 +105,11 @@ class perform_site_check(difference_detection_processor):
             # If the klass doesnt exist, just use a default
             klass = getattr(content_fetcher, "html_requests")
 
+        if preferred_proxy:
+            proxy_id = preferred_proxy
+        else:
             proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
+
         proxy_url = None
         if proxy_id:
             proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')

changedetectionio/static/js/recheck-proxy.js

@@ -0,0 +1,87 @@
+$(function () {
+    /* add container before each proxy location to show status */
+
+    var option_li = $('.fetch-backend-proxy li').filter(function() {
+        return $("input",this)[0].value.length >0;
+    });
+
+    //var option_li = $('.fetch-backend-proxy li');
+    var isActive = false;
+    $(option_li).prepend('<div class="proxy-status"></div>');
+    $(option_li).append('<div class="proxy-timing"></div><div class="proxy-check-details"></div>');
+
+    function set_proxy_check_status(proxy_key, state) {
+        // select input by value name
+        const proxy_li = $('input[value="' + proxy_key + '" ]').parent();
+        if (state['status'] === 'RUNNING') {
+            $('.proxy-status', proxy_li).html('<span class="spinner"></span>');
+        }
+        if (state['status'] === 'OK') {
+            $('.proxy-status', proxy_li).html('<span style="color: green; font-weight: bold" >OK</span>');
+            $('.proxy-check-details', proxy_li).html(state['text']);
+        }
+        if (state['status'] === 'ERROR' || state['status'] === 'ERROR OTHER') {
+            $('.proxy-status', proxy_li).html('<span style="color: red; font-weight: bold" >X</span>');
+            $('.proxy-check-details', proxy_li).html(state['text']);
+        }
+        $('.proxy-timing', proxy_li).html(state['time']);
+    }
+
+
+    function pollServer() {
+        if (isActive) {
+            window.setTimeout(function () {
+                $.ajax({
+                    url: proxy_recheck_status_url,
+                    success: function (data) {
+                        var all_done = true;
+                        $.each(data, function (proxy_key, state) {
+                            set_proxy_check_status(proxy_key, state);
+                            if (state['status'] === 'RUNNING') {
+                                all_done = false;
+                            }
+                        });
+
+                        if (all_done) {
+                            console.log("Shutting down poller, all done.")
+                            isActive = false;
+                        } else {
+                            pollServer();
+                        }
+                    },
+                    error: function () {
+                        //ERROR HANDLING
+                        pollServer();
+                    }
+                });
+            }, 2000);
+        }
+    }
+
+    $('#check-all-proxies').click(function (e) {
+        e.preventDefault()
+        $('body').addClass('proxy-check-active');
+        $('.proxy-check-details').html('');
+        $('.proxy-status').html('<span class="spinner"></span>').fadeIn();
+        $('.proxy-timing').html('');
+
+        // Request start, needs CSRF?
+        $.ajax({
+            type: "GET",
+            url: recheck_proxy_start_url,
+        }).done(function (data) {
+            $.each(data, function (proxy_key, state) {
+                set_proxy_check_status(proxy_key, state['status'])
+            });
+            isActive = true;
+            pollServer();
+
+        }).fail(function (data) {
+            console.log(data);
+            alert('There was an error communicating with the server.');
+        });
+
+    });
+
+});
+

changedetectionio/static/styles/scss/parts/_darkmode.scss

@@ -0,0 +1,25 @@
+
+#toggle-light-mode {
+  width: 3rem;
+  /* default */
+  .icon-dark {
+    display: none;
+  }
+
+}
+
+html[data-darkmode="true"] {
+  #toggle-light-mode {
+    .icon-light {
+      display: none;
+    }
+
+    .icon-dark {
+      display: block;
+    }
+  }
+}
+
+
+
+

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -7,6 +7,7 @@ ul#requests-extra_proxies {
     }
 
   }
+
   /* each proxy entry is a `table` */
   table {
     tr {
@@ -15,3 +16,30 @@ ul#requests-extra_proxies {
   }
 }
 
+#request {
+  /* Auto proxy scan/checker */
+  label[for=proxy] {
+    display: inline-block;
+  }
+}
+
+body.proxy-check-active {
+  #request {
+    .proxy-status {
+      width: 2em;
+    }
+
+    .proxy-check-details {
+      font-size: 80%;
+      color: #555;
+      display: block;
+      padding-left: 4em;
+    }
+
+    .proxy-timing {
+      font-size: 80%;
+      padding-left: 1rem;
+      color: var(--color-link);
+    }
+  }
+}

changedetectionio/static/styles/scss/styles.scss

@@ -8,6 +8,7 @@
 @import "parts/_pagination";
 @import "parts/_spinners";
 @import "parts/_variables";
+@import "parts/_darkmode";
 
 body {
   color: var(--color-text);
@@ -54,22 +55,6 @@ a.github-link {
   }
 }
 
-#toggle-light-mode {
-  width: 3rem;
-  .icon-dark {
-    display: none;
-  }
-
-  &.dark {
-    .icon-light {
-      display: none;
-    }
-
-    .icon-dark {
-      display: block;
-    }
-  }
-}
 
 #toggle-search {
   width: 2rem;

changedetectionio/static/styles/styles.css

@@ -95,6 +95,25 @@ ul#requests-extra_proxies {
   ul#requests-extra_proxies table tr {
     display: inline; }
 
+#request {
+  /* Auto proxy scan/checker */ }
+  #request label[for=proxy] {
+    display: inline-block; }
+
+body.proxy-check-active #request .proxy-status {
+  width: 2em; }
+
+body.proxy-check-active #request .proxy-check-details {
+  font-size: 80%;
+  color: #555;
+  display: block;
+  padding-left: 4em; }
+
+body.proxy-check-active #request .proxy-timing {
+  font-size: 80%;
+  padding-left: 1rem;
+  color: var(--color-link); }
+
 .pagination-page-info {
   color: #fff;
   font-size: 0.85rem;
@@ -283,10 +302,6 @@ html[data-darkmode="true"] {
   --color-icon-github-hover: var(--color-grey-700);
   --color-watch-table-error: var(--color-light-red);
   --color-watch-table-row-text: var(--color-grey-800); }
-  html[data-darkmode="true"] #toggle-light-mode .icon-light {
-    display: none; }
-  html[data-darkmode="true"] #toggle-light-mode .icon-dark {
-    display: block; }
   html[data-darkmode="true"] .icon-spread {
     filter: hue-rotate(-10deg) brightness(1.5); }
   html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
@@ -301,6 +316,18 @@ html[data-darkmode="true"] {
     html[data-darkmode="true"] .watch-table .unviewed.error {
       color: var(--color-watch-table-error); }
 
+#toggle-light-mode {
+  width: 3rem;
+  /* default */ }
+  #toggle-light-mode .icon-dark {
+    display: none; }
+
+html[data-darkmode="true"] #toggle-light-mode .icon-light {
+  display: none; }
+
+html[data-darkmode="true"] #toggle-light-mode .icon-dark {
+  display: block; }
+
 body {
   color: var(--color-text);
   background: var(--color-background-page); }
@@ -335,11 +362,6 @@ a.github-link {
   a.github-link:hover {
     color: var(--color-icon-github-hover); }
 
-#toggle-light-mode {
-  width: 3rem; }
-  #toggle-light-mode .icon-dark {
-    display: none; }
-
 #toggle-search {
   width: 2rem; }
 

changedetectionio/templates/_helpers.jinja

@@ -1,7 +1,6 @@
 {% macro render_field(field) %}
-  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
   <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
-
+  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
   {% if field.errors %}
     <ul class=errors>
     {% for error in field.errors %}
@@ -25,18 +24,6 @@
   </div>
 {% endmacro %}
 
-{% macro render_field(field) %}
-  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
-  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
-  {% if field.errors %}
-    <ul class=errors>
-    {% for error in field.errors %}
-      <li>{{ error }}</li>
-    {% endfor %}
-    </ul>
-  {% endif %}
-  </div>
-{% endmacro %}
 
 {% macro render_simple_field(field) %}
   <span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>

changedetectionio/templates/edit.html

@@ -4,18 +4,19 @@
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script>
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
-    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
-    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
-    const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
-
-{% if emailprefix %}
-    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
-{% endif %}
 
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
     const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
+{% if emailprefix %}
+    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
+{% endif %}
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+    const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
+    const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
+    const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
+    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
+    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
 
 </script>
 
@@ -27,6 +28,8 @@
 <script src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
 {% endif %}
 
+<script src="{{url_for('static_content', group='js', filename='recheck-proxy.js')}}" defer></script>
+
 <div class="edit-form monospaced-textarea">
 
     <div class="tabs collapsable">
@@ -111,7 +114,8 @@
                     </div>
                 {% if form.proxy %}
                     <div class="pure-control-group inline-radio">
-                        {{ render_field(form.proxy, class="fetch-backend-proxy") }}
+                          <div>{{ form.proxy.label }} <a href="" id="check-all-proxies" class="pure-button button-secondary button-xsmall" >Check/Scan all</a></div>
+                          <div>{{ form.proxy(class="fetch-backend-proxy") }}</div>
                         <span class="pure-form-message-inline">
                         Choose a proxy for this watch
                         </span>

changedetectionio/tests/smtp/smtp-test-server.py

@@ -0,0 +1,42 @@
+#!/usr/bin/python3
+import smtpd
+import asyncore
+
+# Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
+
+last_received_message = b"Nothing"
+
+
+class CustomSMTPServer(smtpd.SMTPServer):
+
+    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
+        global last_received_message
+        last_received_message = data
+        print('Receiving message from:', peer)
+        print('Message addressed from:', mailfrom)
+        print('Message addressed to  :', rcpttos)
+        print('Message length        :', len(data))
+        print(data.decode('utf8'))
+        return
+
+
+# Just print out the last message received on plain TCP socket server
+class EchoServer(asyncore.dispatcher):
+
+    def __init__(self, host, port):
+        asyncore.dispatcher.__init__(self)
+        self.create_socket()
+        self.set_reuse_addr()
+        self.bind((host, port))
+        self.listen(5)
+
+    def handle_accepted(self, sock, addr):
+        global last_received_message
+        print('Incoming connection from %s' % repr(addr))
+        sock.send(last_received_message)
+        last_received_message = b''
+
+
+server = CustomSMTPServer(('0.0.0.0', 11025), None)  # SMTP mail goes here
+server2 = EchoServer('0.0.0.0', 11080)  # Echo back last message received
+asyncore.loop()

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -0,0 +1,165 @@
+import json
+import os
+import time
+import re
+from flask import url_for
+from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
+    wait_for_all_checks, \
+    set_longer_modified_response
+from changedetectionio.tests.util import extract_UUID_from_client
+import logging
+import base64
+
+# NOTE - RELIES ON mailserver as hostname running, see github build recipes
+smtp_test_server = 'mailserver'
+
+from changedetectionio.notification import (
+    default_notification_body,
+    default_notification_format,
+    default_notification_title,
+    valid_notification_formats,
+)
+
+def test_setup(live_server):
+    live_server_setup(live_server)
+
+def get_last_message_from_smtp_server():
+    import socket
+    global smtp_test_server
+    port = 11080  # socket server port number
+
+    client_socket = socket.socket()  # instantiate
+    client_socket.connect((smtp_test_server, port))  # connect to the server
+
+    data = client_socket.recv(50024).decode()  # receive response
+    client_socket.close()  # close the connection
+    return data
+
+
+# Requires running the test SMTP server
+
+def test_check_notification_email_formats_default_HTML(client, live_server):
+    # live_server_setup(live_server)
+    set_original_response()
+
+    global smtp_test_server
+    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_body": "fallback-body<br> " + default_notification_body,
+              "application-notification_format": 'HTML',
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Add a watch and trigger a HTTP POST
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": 'nice one'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    wait_for_all_checks(client)
+    set_longer_modified_response()
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    time.sleep(3)
+
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
+
+    # The email should have two bodies, and the text/html part should be <br>
+    assert 'Content-Type: text/plain' in msg
+    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+    assert 'Content-Type: text/html' in msg
+    assert '(added) So let\'s see what happens.<br>' in msg  # the html part
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+
+def test_check_notification_email_formats_default_Text_override_HTML(client, live_server):
+    # live_server_setup(live_server)
+
+    # HTML problems? see this
+    # https://github.com/caronc/apprise/issues/633
+
+    set_original_response()
+    global smtp_test_server
+    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
+
+    #####################
+    # Set this up for when we remove the notification from the watch, it should fallback with these details
+    res = client.post(
+        url_for("settings_page"),
+        data={"application-notification_urls": notification_url,
+              "application-notification_title": "fallback-title " + default_notification_title,
+              "application-notification_body": default_notification_body,
+              "application-notification_format": 'Text',
+              "requests-time_between_check-minutes": 180,
+              'application-fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Settings updated." in res.data
+
+    # Add a watch and trigger a HTTP POST
+    test_url = url_for('test_endpoint', _external=True)
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": test_url, "tags": 'nice one'},
+        follow_redirects=True
+    )
+
+    assert b"Watch added" in res.data
+
+    wait_for_all_checks(client)
+    set_longer_modified_response()
+    client.get(url_for("form_watch_checknow"), follow_redirects=True)
+    wait_for_all_checks(client)
+
+    time.sleep(3)
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
+    #    with open('/tmp/m.txt', 'w') as f:
+    #        f.write(msg)
+
+    # The email should not have two bodies, should be TEXT only
+
+    assert 'Content-Type: text/plain' in msg
+    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+
+    set_original_response()
+    # Now override as HTML format
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={
+            "url": test_url,
+            "notification_format": 'HTML',
+            'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+    wait_for_all_checks(client)
+
+    time.sleep(3)
+    msg = get_last_message_from_smtp_server()
+    assert len(msg) >= 1
+
+    # The email should have two bodies, and the text/html part should be <br>
+    assert 'Content-Type: text/plain' in msg
+    assert '(removed) So let\'s see what happens.\n' in msg  # The plaintext part with \n
+    assert 'Content-Type: text/html' in msg
+    assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
+
+    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data

changedetectionio/tests/test_notification.py

@@ -3,7 +3,8 @@ import os
 import time
 import re
 from flask import url_for
-from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks
+from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
+    set_longer_modified_response
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -272,7 +273,7 @@ def test_notification_validation(client, live_server):
 
 
 def test_notification_custom_endpoint_and_jinja2(client, live_server):
-    time.sleep(1)
+    #live_server_setup(live_server)
 
     # test_endpoint - that sends the contents of a file
     # test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt)
@@ -283,12 +284,14 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
 
     res = client.post(
         url_for("settings_page"),
-        data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
+        data={
-              "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }',
+              "application-fetch_backend": "html_requests",
-              # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-              "application-notification_urls": test_notification_url,
               "application-minutes_between_check": 180,
-              "application-fetch_backend": "html_requests"
+              "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }',
+              "application-notification_format": default_notification_format,
+              "application-notification_urls": test_notification_url,
+              # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
+              "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
               },
         follow_redirects=True
     )
@@ -313,9 +316,8 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(2)
 
-
     with open("test-datastore/notification.txt", 'r') as f:
-        x=f.read()
+        x = f.read()
         j = json.loads(x)
         assert j['url'].startswith('http://localhost')
         assert j['secret'] == 444
@@ -326,5 +328,9 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         notification_url = f.read()
         assert 'xxx=http' in notification_url
 
-    os.unlink("test-datastore/notification-url.txt")
+    # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
+    assert os.path.isfile("test-datastore/notification-content-type.txt")
+    with open("test-datastore/notification-content-type.txt", 'r') as f:
+        assert 'application/json' in f.read()
 
+    os.unlink("test-datastore/notification-url.txt")

changedetectionio/tests/util.py

@@ -38,7 +38,25 @@ def set_modified_response():
         f.write(test_return_data)
 
     return None
+def set_longer_modified_response():
+    test_return_data = """<html>
+    <head><title>modified head title</title></head>
+    <body>
+     Some initial text<br>
+     <p>which has this one new line</p>
+     <br>
+     So let's see what happens.  <br>
+     So let's see what happens.  <br>
+      So let's see what happens.  <br>
+     So let's see what happens.  <br>           
+     </body>
+     </html>
+    """
 
+    with open("test-datastore/endpoint-content.txt", "w") as f:
+        f.write(test_return_data)
+
+    return None
 def set_more_modified_response():
     test_return_data = """<html>
     <head><title>modified head title</title></head>
@@ -187,6 +205,10 @@ def live_server_setup(live_server):
         with open("test-datastore/notification-url.txt", "w") as f:
             f.write(request.url)
 
+        if request.content_type:
+            with open("test-datastore/notification-content-type.txt", "w") as f:
+                f.write(request.content_type)
+
         print("\n>> Test notification endpoint was hit.\n", data)
         return "Text was set"
 

changedetectionio/update_worker.py

@@ -32,15 +32,17 @@ class update_worker(threading.Thread):
 
         watch_history = watch.history
         dates = list(watch_history.keys())
+        # Add text that was triggered
+        snapshot_contents = watch.get_history_snapshot(dates[-1])
 
         # HTML needs linebreak, but MarkDown and Text can use a linefeed
         if n_object['notification_format'] == 'HTML':
             line_feed_sep = "<br>"
+            # Snapshot will be plaintext on the disk, convert to some kind of HTML
+            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
         else:
             line_feed_sep = "\n"
 
-        # Add text that was triggered
-        snapshot_contents = watch.get_history_snapshot(dates[-1])
         trigger_text = watch.get('trigger_text', [])
         triggered_text = ''
 
@@ -71,13 +73,16 @@ class update_worker(threading.Thread):
         from changedetectionio.notification import (
             default_notification_format_for_watch,
             default_notification_body,
-            default_notification_title,
+            default_notification_title
         )
 
 
         # Would be better if this was some kind of Object where Watch can reference the parent datastore etc
         v = watch.get(var_name)
         if v and not watch.get('notification_muted'):
+            if var_name == 'notification_format' and v == default_notification_format_for_watch:
+                return self.datastore.data['settings']['application'].get('notification_format')
+
             return v
 
         tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
@@ -374,6 +379,9 @@ class update_worker(threading.Thread):
                         if not self.datastore.data['watching'][uuid].get('ignore_status_codes'):
                             update_obj['consecutive_filter_failures'] = 0
 
+                        # Everything ran OK, clean off any previous error
+                        update_obj['last_error'] = False
+
                         self.cleanup_error_artifacts(uuid)
 
                     #

requirements.txt

@@ -10,7 +10,8 @@ flask~=2.0
 inscriptis~=2.2
 pytz
 timeago~=1.0
-validators
+validators~=0.21
+
 
 # Set these versions together to avoid a RequestsDependencyWarning
 # >= 2.26 also adds Brotli support if brotli is installed
@@ -32,7 +33,7 @@ dnspython<2.3.0
 # jq not available on Windows so must be installed manually
 
 # Notification library
-apprise~=1.3.0
+apprise~=1.4.5
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt
@@ -71,3 +72,6 @@ pillow
 # Include pytest, so if theres a support issue we can ask them to run these tests on their setup
 pytest ~=7.2
 pytest-flask ~=1.2
+
+# Pin jsonschema version to prevent build errors on armv6 while rpds-py wheels aren't available (1708)
+jsonschema==4.17.3