Watches werent falling back to global default formats correctly

.dockerignore

@@ -1,18 +1,2 @@
 .git
 .github
-changedetectionio/processors/__pycache__
-changedetectionio/api/__pycache__
-changedetectionio/model/__pycache__
-changedetectionio/blueprint/price_data_follower/__pycache__
-changedetectionio/blueprint/tags/__pycache__
-changedetectionio/blueprint/__pycache__
-changedetectionio/blueprint/browser_steps/__pycache__
-changedetectionio/fetchers/__pycache__
-changedetectionio/tests/visualselector/__pycache__
-changedetectionio/tests/restock/__pycache__
-changedetectionio/tests/__pycache__
-changedetectionio/tests/fetchers/__pycache__
-changedetectionio/tests/unit/__pycache__
-changedetectionio/tests/proxy_list/__pycache__
-changedetectionio/__pycache__
-

.github/workflows/test-only.yml

@@ -37,11 +37,6 @@ jobs:
           # Build a changedetection.io container and start testing inside
           docker build . -t test-changedetectionio
 
-      - name: Spin up ancillary SMTP+Echo message test server
-        run: |
-          # Debug SMTP server/echo message back server
-          docker run --network changedet-network -d -p 11025:11025 -p 11080:11080  --hostname mailserver test-changedetectionio  bash -c 'python changedetectionio/tests/smtp/smtp-test-server.py' 
-
       - name: Test built container with pytest
         run: |
           
@@ -63,16 +58,11 @@ jobs:
           # Settings headers playwright tests - Call back in from Browserless, check headers
           docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
           docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
-          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'          
+          docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0  --live-server-port=5004 tests/test_request.py'
           
           # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it
           docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'
 
-      - name: Test SMTP notification mime types
-        run: |
-          # SMTP content types - needs the 'Debug SMTP server/echo message back server' container from above
-          docker run --rm  --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'
-
       - name: Test with puppeteer fetcher and disk cache
         run: |
           docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio  bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py'

MANIFEST.in

@@ -13,6 +13,3 @@ include changedetection.py
 global-exclude *.pyc
 global-exclude node_modules
 global-exclude venv
-
-global-exclude test-datastore
-global-exclude changedetection.io*dist-info

changedetectionio/__init__.py

@@ -38,7 +38,7 @@ from flask_paginate import Pagination, get_page_parameter
 from changedetectionio import html_tools
 from changedetectionio.api import api_v1
 
-__version__ = '0.43.2'
+__version__ = '0.43.1'
 
 datastore = None
 
@@ -1439,10 +1439,6 @@ def changedetection_app(config=None, datastore_o=None):
     import changedetectionio.blueprint.tags as tags
     app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags')
 
-    import changedetectionio.blueprint.check_proxies as check_proxies
-    app.register_blueprint(check_proxies.construct_blueprint(datastore=datastore), url_prefix='/check_proxy')
-
-
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
     threading.Thread(target=notification_runner).start()

changedetectionio/blueprint/check_proxies/__init__.py

@@ -1,105 +0,0 @@
-from concurrent.futures import ThreadPoolExecutor
-
-from functools import wraps
-
-from flask import Blueprint
-from flask_login import login_required
-
-from changedetectionio.processors import text_json_diff
-from changedetectionio.store import ChangeDetectionStore
-
-
-STATUS_CHECKING = 0
-STATUS_FAILED = 1
-STATUS_OK = 2
-THREADPOOL_MAX_WORKERS = 3
-_DEFAULT_POOL = ThreadPoolExecutor(max_workers=THREADPOOL_MAX_WORKERS)
-
-
-# Maybe use fetch-time if its >5 to show some expected load time?
-def threadpool(f, executor=None):
-    @wraps(f)
-    def wrap(*args, **kwargs):
-        return (executor or _DEFAULT_POOL).submit(f, *args, **kwargs)
-
-    return wrap
-
-
-def construct_blueprint(datastore: ChangeDetectionStore):
-    check_proxies_blueprint = Blueprint('check_proxies', __name__)
-    checks_in_progress = {}
-
-    @threadpool
-    def long_task(uuid, preferred_proxy):
-        import time
-        from changedetectionio import content_fetcher
-
-        status = {'status': '', 'length': 0, 'text': ''}
-        from jinja2 import Environment, BaseLoader
-
-        contents = ''
-        now = time.time()
-        try:
-            update_handler = text_json_diff.perform_site_check(datastore=datastore)
-            changed_detected, update_obj, contents = update_handler.run(uuid, preferred_proxy=preferred_proxy, skip_when_checksum_same=False)
-        # title, size is len contents not len xfer
-        except content_fetcher.Non200ErrorCodeReceived as e:
-            if e.status_code == 404:
-                status.update({'status': 'OK', 'length': len(contents), 'text': f"OK but 404 (page not found)"})
-            elif e.status_code == 403:
-                status.update({'status': 'ERROR', 'length': len(contents), 'text': f"403 - Access denied"})
-            else:
-                status.update({'status': 'ERROR', 'length': len(contents), 'text': f"Status code: {e.status_code}"})
-        except content_fetcher.EmptyReply as e:
-            status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': "Empty reply, needs chrome?"})
-        except Exception as e:
-            status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': 'Error: '+str(e)})
-        else:
-            status.update({'status': 'OK', 'length': len(contents), 'text': ''})
-
-        if status.get('text'):
-            status['text'] = Environment(loader=BaseLoader()).from_string('{{text|e}}').render({'text': status['text']})
-
-        status['time'] = "{:.2f}s".format(time.time() - now)
-
-        return status
-
-    def _recalc_check_status(uuid):
-
-        results = {}
-        for k, v in checks_in_progress.get(uuid, {}).items():
-            try:
-                r_1 = v.result(timeout=0.05)
-            except Exception as e:
-                # If timeout error?
-                results[k] = {'status': 'RUNNING'}
-
-            else:
-                results[k] = r_1
-
-        return results
-
-    @login_required
-    @check_proxies_blueprint.route("/<string:uuid>/status", methods=['GET'])
-    def get_recheck_status(uuid):
-        results = _recalc_check_status(uuid=uuid)
-        return results
-
-    @login_required
-    @check_proxies_blueprint.route("/<string:uuid>/start", methods=['GET'])
-    def start_check(uuid):
-
-        if not datastore.proxy_list:
-            return
-
-        # @todo - Cancel any existing runs
-        checks_in_progress[uuid] = {}
-
-        for k, v in datastore.proxy_list.items():
-            if not checks_in_progress[uuid].get(k):
-                checks_in_progress[uuid][k] = long_task(uuid=uuid, preferred_proxy=k)
-
-        results = _recalc_check_status(uuid=uuid)
-        return results
-
-    return check_proxies_blueprint

changedetectionio/diff.py

@@ -54,5 +54,4 @@ def render_diff(previous_version_file_contents, newest_version_file_contents, in
 
     # Recursively join lists
     f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L])
-    p= f(rendered_diff)
+    return f(rendered_diff)
-    return p

changedetectionio/notification.py

@@ -151,12 +151,9 @@ def process_notification(n_object, datastore):
                     # Apprise will default to HTML, so we need to override it
                     # So that whats' generated in n_body is in line with what is going to be sent.
                     # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321
-                    if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'):
+                    if not 'format=' in url and (n_format == 'text' or n_format == 'markdown'):
                         prefix = '?' if not '?' in url else '&'
-                        # Apprise format is lowercase text https://github.com/caronc/apprise/issues/633
-                        n_format = n_format.tolower()
                         url = "{}{}format={}".format(url, prefix, n_format)
-                    # If n_format == HTML, then apprise email should default to text/html and we should be sending HTML only
 
                 apobj.add(url)
 

changedetectionio/processors/__init__.py

@@ -9,7 +9,7 @@ class difference_detection_processor():
         super().__init__(*args, **kwargs)
 
     @abstractmethod
-    def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
+    def run(self, uuid, skip_when_checksum_same=True):
         update_obj = {'last_notification_error': False, 'last_error': False}
         some_data = 'xxxxx'
         update_obj["previous_md5"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()

changedetectionio/processors/text_json_diff.py

@@ -50,7 +50,7 @@ class perform_site_check(difference_detection_processor):
 
         return regex
 
-    def run(self, uuid, skip_when_checksum_same=True, preferred_proxy=None):
+    def run(self, uuid, skip_when_checksum_same=True):
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
@@ -105,11 +105,7 @@ class perform_site_check(difference_detection_processor):
             # If the klass doesnt exist, just use a default
             klass = getattr(content_fetcher, "html_requests")
 
-        if preferred_proxy:
+        proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
-            proxy_id = preferred_proxy
-        else:
-            proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=uuid)
-
         proxy_url = None
         if proxy_id:
             proxy_url = self.datastore.proxy_list.get(proxy_id).get('url')

changedetectionio/static/js/recheck-proxy.js

@@ -1,87 +0,0 @@
-$(function () {
-    /* add container before each proxy location to show status */
-
-    var option_li = $('.fetch-backend-proxy li').filter(function() {
-        return $("input",this)[0].value.length >0;
-    });
-
-    //var option_li = $('.fetch-backend-proxy li');
-    var isActive = false;
-    $(option_li).prepend('<div class="proxy-status"></div>');
-    $(option_li).append('<div class="proxy-timing"></div><div class="proxy-check-details"></div>');
-
-    function set_proxy_check_status(proxy_key, state) {
-        // select input by value name
-        const proxy_li = $("input[value=" + proxy_key + "]").parent();
-        if (state['status'] === 'RUNNING') {
-            $('.proxy-status', proxy_li).html('<span class="spinner"></span>');
-        }
-        if (state['status'] === 'OK') {
-            $('.proxy-status', proxy_li).html('<span style="color: green; font-weight: bold" >OK</span>');
-            $('.proxy-check-details', proxy_li).html(state['text']);
-        }
-        if (state['status'] === 'ERROR' || state['status'] === 'ERROR OTHER') {
-            $('.proxy-status', proxy_li).html('<span style="color: red; font-weight: bold" >X</span>');
-            $('.proxy-check-details', proxy_li).html(state['text']);
-        }
-        $('.proxy-timing', proxy_li).html(state['time']);
-    }
-
-
-    function pollServer() {
-        if (isActive) {
-            window.setTimeout(function () {
-                $.ajax({
-                    url: proxy_recheck_status_url,
-                    success: function (data) {
-                        var all_done = true;
-                        $.each(data, function (proxy_key, state) {
-                            set_proxy_check_status(proxy_key, state);
-                            if (state['status'] === 'RUNNING') {
-                                all_done = false;
-                            }
-                        });
-
-                        if (all_done) {
-                            console.log("Shutting down poller, all done.")
-                            isActive = false;
-                        } else {
-                            pollServer();
-                        }
-                    },
-                    error: function () {
-                        //ERROR HANDLING
-                        pollServer();
-                    }
-                });
-            }, 2000);
-        }
-    }
-
-    $('#check-all-proxies').click(function (e) {
-        e.preventDefault()
-        $('body').addClass('proxy-check-active');
-        $('.proxy-check-details').html('');
-        $('.proxy-status').html('<span class="spinner"></span>').fadeIn();
-        $('.proxy-timing').html('');
-
-        // Request start, needs CSRF?
-        $.ajax({
-            type: "GET",
-            url: recheck_proxy_start_url,
-        }).done(function (data) {
-            $.each(data, function (proxy_key, state) {
-                set_proxy_check_status(proxy_key, state['status'])
-            });
-            isActive = true;
-            pollServer();
-
-        }).fail(function (data) {
-            console.log(data);
-            alert('There was an error communicating with the server.');
-        });
-
-    });
-
-});
-

changedetectionio/static/styles/scss/parts/_extra_proxies.scss

@@ -7,7 +7,6 @@ ul#requests-extra_proxies {
     }
 
   }
-
   /* each proxy entry is a `table` */
   table {
     tr {
@@ -16,30 +15,3 @@ ul#requests-extra_proxies {
   }
 }
 
-#request {
-  /* Auto proxy scan/checker */
-  label[for=proxy] {
-    display: inline-block;
-  }
-}
-
-body.proxy-check-active {
-  #request {
-    .proxy-status {
-      width: 2em;
-    }
-
-    .proxy-check-details {
-      font-size: 80%;
-      color: #555;
-      display: block;
-      padding-left: 4em;
-    }
-
-    .proxy-timing {
-      font-size: 80%;
-      padding-left: 1rem;
-      color: var(--color-link);
-    }
-  }
-}

changedetectionio/static/styles/styles.css

@@ -95,25 +95,6 @@ ul#requests-extra_proxies {
   ul#requests-extra_proxies table tr {
     display: inline; }
 
-#request {
-  /* Auto proxy scan/checker */ }
-  #request label[for=proxy] {
-    display: inline-block; }
-
-body.proxy-check-active #request .proxy-status {
-  width: 2em; }
-
-body.proxy-check-active #request .proxy-check-details {
-  font-size: 80%;
-  color: #555;
-  display: block;
-  padding-left: 4em; }
-
-body.proxy-check-active #request .proxy-timing {
-  font-size: 80%;
-  padding-left: 1rem;
-  color: var(--color-link); }
-
 .pagination-page-info {
   color: #fff;
   font-size: 0.85rem;
@@ -302,6 +283,10 @@ html[data-darkmode="true"] {
   --color-icon-github-hover: var(--color-grey-700);
   --color-watch-table-error: var(--color-light-red);
   --color-watch-table-row-text: var(--color-grey-800); }
+  html[data-darkmode="true"] #toggle-light-mode .icon-light {
+    display: none; }
+  html[data-darkmode="true"] #toggle-light-mode .icon-dark {
+    display: block; }
   html[data-darkmode="true"] .icon-spread {
     filter: hue-rotate(-10deg) brightness(1.5); }
   html[data-darkmode="true"] .watch-table .title-col a[target="_blank"]::after,
@@ -354,10 +339,6 @@ a.github-link {
   width: 3rem; }
   #toggle-light-mode .icon-dark {
     display: none; }
-  #toggle-light-mode.dark .icon-light {
-    display: none; }
-  #toggle-light-mode.dark .icon-dark {
-    display: block; }
 
 #toggle-search {
   width: 2rem; }

changedetectionio/templates/_helpers.jinja

@@ -1,6 +1,7 @@
 {% macro render_field(field) %}
-  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
   <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
+
   {% if field.errors %}
     <ul class=errors>
     {% for error in field.errors %}
@@ -24,6 +25,18 @@
   </div>
 {% endmacro %}
 
+{% macro render_field(field) %}
+  <div {% if field.errors %} class="error" {% endif %}>{{ field.label }}</div>
+  <div {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
+  {% if field.errors %}
+    <ul class=errors>
+    {% for error in field.errors %}
+      <li>{{ error }}</li>
+    {% endfor %}
+    </ul>
+  {% endif %}
+  </div>
+{% endmacro %}
 
 {% macro render_simple_field(field) %}
   <span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>

changedetectionio/templates/edit.html

@@ -4,19 +4,18 @@
 {% from '_common_fields.jinja' import render_common_settings_form %}
 <script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
 <script>
+    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
+    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
+    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
+    const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
+
+{% if emailprefix %}
+    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
+{% endif %}
 
     const browser_steps_config=JSON.parse('{{ browser_steps_config|tojson }}');
     const browser_steps_start_url="{{url_for('browser_steps.browsersteps_start_session', uuid=uuid)}}";
     const browser_steps_sync_url="{{url_for('browser_steps.browsersteps_ui_update', uuid=uuid)}}";
-{% if emailprefix %}
-    const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}');
-{% endif %}
-    const notification_base_url="{{url_for('ajax_callback_send_notification_test')}}";
-    const playwright_enabled={% if playwright_enabled %} true {% else %} false {% endif %};
-    const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}";
-    const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}";
-    const screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid)}}";
-    const watch_visual_selector_data_url="{{url_for('static_content', group='visual_selector_data', filename=uuid)}}";
 
 </script>
 
@@ -28,8 +27,6 @@
 <script src="{{url_for('static_content', group='js', filename='browser-steps.js')}}" defer></script>
 {% endif %}
 
-<script src="{{url_for('static_content', group='js', filename='recheck-proxy.js')}}" defer></script>
-
 <div class="edit-form monospaced-textarea">
 
     <div class="tabs collapsable">
@@ -114,8 +111,7 @@
                     </div>
                 {% if form.proxy %}
                     <div class="pure-control-group inline-radio">
-                          <div>{{ form.proxy.label }} <a href="" id="check-all-proxies" class="pure-button button-secondary button-xsmall" >Check/Scan all</a></div>
+                        {{ render_field(form.proxy, class="fetch-backend-proxy") }}
-                          <div>{{ form.proxy(class="fetch-backend-proxy") }}</div>
                         <span class="pure-form-message-inline">
                         Choose a proxy for this watch
                         </span>

changedetectionio/tests/smtp/smtp-test-server.py

@@ -1,42 +0,0 @@
-#!/usr/bin/python3
-import smtpd
-import asyncore
-
-# Accept a SMTP message and offer a way to retrieve the last message via TCP Socket
-
-last_received_message = b"Nothing"
-
-
-class CustomSMTPServer(smtpd.SMTPServer):
-
-    def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
-        global last_received_message
-        last_received_message = data
-        print('Receiving message from:', peer)
-        print('Message addressed from:', mailfrom)
-        print('Message addressed to  :', rcpttos)
-        print('Message length        :', len(data))
-        print(data.decode('utf8'))
-        return
-
-
-# Just print out the last message received on plain TCP socket server
-class EchoServer(asyncore.dispatcher):
-
-    def __init__(self, host, port):
-        asyncore.dispatcher.__init__(self)
-        self.create_socket()
-        self.set_reuse_addr()
-        self.bind((host, port))
-        self.listen(5)
-
-    def handle_accepted(self, sock, addr):
-        global last_received_message
-        print('Incoming connection from %s' % repr(addr))
-        sock.send(last_received_message)
-        last_received_message = b''
-
-
-server = CustomSMTPServer(('0.0.0.0', 11025), None)  # SMTP mail goes here
-server2 = EchoServer('0.0.0.0', 11080)  # Echo back last message received
-asyncore.loop()

changedetectionio/tests/smtp/test_notification_smtp.py

@@ -1,165 +0,0 @@
-import json
-import os
-import time
-import re
-from flask import url_for
-from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \
-    wait_for_all_checks, \
-    set_longer_modified_response
-from changedetectionio.tests.util import extract_UUID_from_client
-import logging
-import base64
-
-# NOTE - RELIES ON mailserver as hostname running, see github build recipes
-smtp_test_server = 'mailserver'
-
-from changedetectionio.notification import (
-    default_notification_body,
-    default_notification_format,
-    default_notification_title,
-    valid_notification_formats,
-)
-
-def test_setup(live_server):
-    live_server_setup(live_server)
-
-def get_last_message_from_smtp_server():
-    import socket
-    global smtp_test_server
-    port = 11080  # socket server port number
-
-    client_socket = socket.socket()  # instantiate
-    client_socket.connect((smtp_test_server, port))  # connect to the server
-
-    data = client_socket.recv(50024).decode()  # receive response
-    client_socket.close()  # close the connection
-    return data
-
-
-# Requires running the test SMTP server
-
-def test_check_notification_email_formats_default_HTML(client, live_server):
-    # live_server_setup(live_server)
-    set_original_response()
-
-    global smtp_test_server
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": "fallback-body<br> " + default_notification_body,
-              "application-notification_format": 'HTML',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-    set_longer_modified_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
-
-    # The email should have two bodies, and the text/html part should be <br>
-    assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
-    assert 'Content-Type: text/html' in msg
-    assert '(added) So let\'s see what happens.<br>' in msg  # the html part
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
-
-
-def test_check_notification_email_formats_default_Text_override_HTML(client, live_server):
-    # live_server_setup(live_server)
-
-    # HTML problems? see this
-    # https://github.com/caronc/apprise/issues/633
-
-    set_original_response()
-    global smtp_test_server
-    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'
-
-    #####################
-    # Set this up for when we remove the notification from the watch, it should fallback with these details
-    res = client.post(
-        url_for("settings_page"),
-        data={"application-notification_urls": notification_url,
-              "application-notification_title": "fallback-title " + default_notification_title,
-              "application-notification_body": default_notification_body,
-              "application-notification_format": 'Text',
-              "requests-time_between_check-minutes": 180,
-              'application-fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
-
-    # Add a watch and trigger a HTTP POST
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("form_quick_watch_add"),
-        data={"url": test_url, "tags": 'nice one'},
-        follow_redirects=True
-    )
-
-    assert b"Watch added" in res.data
-
-    wait_for_all_checks(client)
-    set_longer_modified_response()
-    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
-    #    with open('/tmp/m.txt', 'w') as f:
-    #        f.write(msg)
-
-    # The email should not have two bodies, should be TEXT only
-
-    assert 'Content-Type: text/plain' in msg
-    assert '(added) So let\'s see what happens.\n' in msg  # The plaintext part with \n
-
-    set_original_response()
-    # Now override as HTML format
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-            "url": test_url,
-            "notification_format": 'HTML',
-            'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-    wait_for_all_checks(client)
-
-    time.sleep(3)
-    msg = get_last_message_from_smtp_server()
-    assert len(msg) >= 1
-
-    # The email should have two bodies, and the text/html part should be <br>
-    assert 'Content-Type: text/plain' in msg
-    assert '(removed) So let\'s see what happens.\n' in msg  # The plaintext part with \n
-    assert 'Content-Type: text/html' in msg
-    assert '(removed) So let\'s see what happens.<br>' in msg  # the html part
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data

changedetectionio/tests/test_notification.py

@@ -3,8 +3,7 @@ import os
 import time
 import re
 from flask import url_for
-from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \
+from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks
-    set_longer_modified_response
 from . util import  extract_UUID_from_client
 import logging
 import base64
@@ -273,7 +272,7 @@ def test_notification_validation(client, live_server):
 
 
 def test_notification_custom_endpoint_and_jinja2(client, live_server):
-    #live_server_setup(live_server)
+    time.sleep(1)
 
     # test_endpoint - that sends the contents of a file
     # test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt)
@@ -284,14 +283,12 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
 
     res = client.post(
         url_for("settings_page"),
-        data={
+        data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
-              "application-fetch_backend": "html_requests",
-              "application-minutes_between_check": 180,
               "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }',
-              "application-notification_format": default_notification_format,
-              "application-notification_urls": test_notification_url,
               # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
-              "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
+              "application-notification_urls": test_notification_url,
+              "application-minutes_between_check": 180,
+              "application-fetch_backend": "html_requests"
               },
         follow_redirects=True
     )
@@ -316,8 +313,9 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
     client.get(url_for("form_watch_checknow"), follow_redirects=True)
     time.sleep(2)
 
+
     with open("test-datastore/notification.txt", 'r') as f:
-        x = f.read()
+        x=f.read()
         j = json.loads(x)
         assert j['url'].startswith('http://localhost')
         assert j['secret'] == 444
@@ -328,9 +326,5 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server):
         notification_url = f.read()
         assert 'xxx=http' in notification_url
 
-    # Should always be automatically detected as JSON content type even when we set it as 'Text' (default)
-    assert os.path.isfile("test-datastore/notification-content-type.txt")
-    with open("test-datastore/notification-content-type.txt", 'r') as f:
-        assert 'application/json' in f.read()
-
     os.unlink("test-datastore/notification-url.txt")
+

changedetectionio/tests/util.py

@@ -38,25 +38,7 @@ def set_modified_response():
         f.write(test_return_data)
 
     return None
-def set_longer_modified_response():
-    test_return_data = """<html>
-    <head><title>modified head title</title></head>
-    <body>
-     Some initial text<br>
-     <p>which has this one new line</p>
-     <br>
-     So let's see what happens.  <br>
-     So let's see what happens.  <br>
-      So let's see what happens.  <br>
-     So let's see what happens.  <br>           
-     </body>
-     </html>
-    """
 
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-    return None
 def set_more_modified_response():
     test_return_data = """<html>
     <head><title>modified head title</title></head>
@@ -205,10 +187,6 @@ def live_server_setup(live_server):
         with open("test-datastore/notification-url.txt", "w") as f:
             f.write(request.url)
 
-        if request.content_type:
-            with open("test-datastore/notification-content-type.txt", "w") as f:
-                f.write(request.content_type)
-
         print("\n>> Test notification endpoint was hit.\n", data)
         return "Text was set"
 

changedetectionio/update_worker.py

@@ -32,17 +32,15 @@ class update_worker(threading.Thread):
 
         watch_history = watch.history
         dates = list(watch_history.keys())
-        # Add text that was triggered
-        snapshot_contents = watch.get_history_snapshot(dates[-1])
 
         # HTML needs linebreak, but MarkDown and Text can use a linefeed
         if n_object['notification_format'] == 'HTML':
             line_feed_sep = "<br>"
-            # Snapshot will be plaintext on the disk, convert to some kind of HTML
-            snapshot_contents = snapshot_contents.replace('\n', line_feed_sep)
         else:
             line_feed_sep = "\n"
 
+        # Add text that was triggered
+        snapshot_contents = watch.get_history_snapshot(dates[-1])
         trigger_text = watch.get('trigger_text', [])
         triggered_text = ''
 
@@ -80,9 +78,6 @@ class update_worker(threading.Thread):
         # Would be better if this was some kind of Object where Watch can reference the parent datastore etc
         v = watch.get(var_name)
         if v and not watch.get('notification_muted'):
-            if var_name == 'notification_format' and v == default_notification_format_for_watch:
-                return self.datastore.data['settings']['application'].get('notification_format')
-
             return v
 
         tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
@@ -379,9 +374,6 @@ class update_worker(threading.Thread):
                         if not self.datastore.data['watching'][uuid].get('ignore_status_codes'):
                             update_obj['consecutive_filter_failures'] = 0
 
-                        # Everything ran OK, clean off any previous error
-                        update_obj['last_error'] = False
-
                         self.cleanup_error_artifacts(uuid)
 
                     #