diff --git a/.github/workflows/test-only.yml b/.github/workflows/test-only.yml index 0881b8d7..f70bd9c5 100644 --- a/.github/workflows/test-only.yml +++ b/.github/workflows/test-only.yml @@ -31,6 +31,8 @@ jobs: # Selenium+browserless docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size="2g" selenium/standalone-chrome-debug:3.141.59 docker run --network changedet-network -d --hostname browserless -e "FUNCTION_BUILT_INS=[\"fs\",\"crypto\"]" -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" browserless/chrome:1.53-chrome-stable + # Debug SMTP server/echo message back server + docker run --network changedet-network --hostname mailserver -v "`pwd`/changedetectionio/tests/smtp/smtp-test-server.py:/smtp-test-server.py" python:3.10-bullseye bash -c 'python /smtp-test-server.py' --rm -p 11025:11025 -p 11080:11080 -d - name: Build changedetection.io container for testing run: | @@ -58,11 +60,15 @@ jobs: # Settings headers playwright tests - Call back in from Browserless, check headers docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "WEBDRIVER_URL=http://selenium:4444/wd/hub" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' - docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' + docker run --name "changedet" --hostname changedet --rm -e "FLASK_SERVER_NAME=changedet" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000?dumpio=true" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py' # restock detection via playwright - added name=changedet here so that playwright/browserless can connect to it docker run --rm --name "changedet" -e "FLASK_SERVER_NAME=changedet" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py' + # SMTP content types + docker run --rm -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py' + + - name: Test with puppeteer fetcher and disk cache run: | docker run --rm -e "PUPPETEER_DISK_CACHE=/tmp/data/" -e "USE_EXPERIMENTAL_PUPPETEER_FETCH=yes" -e "PLAYWRIGHT_DRIVER_URL=ws://browserless:3000" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py && pytest tests/visualselector/test_fetch_data.py' diff --git a/changedetectionio/diff.py b/changedetectionio/diff.py index c3d8b0cd..9cb4c9fe 100644 --- a/changedetectionio/diff.py +++ b/changedetectionio/diff.py @@ -54,4 +54,5 @@ def render_diff(previous_version_file_contents, newest_version_file_contents, in # Recursively join lists f = lambda L: line_feed_sep.join([f(x) if type(x) is list else x for x in L]) - return f(rendered_diff) + p= f(rendered_diff) + return p diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index ca5ea21d..dc8406a6 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -151,7 +151,7 @@ def process_notification(n_object, datastore): # Apprise will default to HTML, so we need to override it # So that whats' generated in n_body is in line with what is going to be sent. # https://github.com/caronc/apprise/issues/633#issuecomment-1191449321 - if not 'format=' in url and (n_format == 'text' or n_format == 'markdown'): + if not 'format=' in url and (n_format == 'Text' or n_format == 'Markdown'): prefix = '?' if not '?' in url else '&' url = "{}{}format={}".format(url, prefix, n_format) diff --git a/changedetectionio/tests/smtp/smtp-test-server.py b/changedetectionio/tests/smtp/smtp-test-server.py new file mode 100755 index 00000000..23378b98 --- /dev/null +++ b/changedetectionio/tests/smtp/smtp-test-server.py @@ -0,0 +1,41 @@ +#!/usr/bin/python3 +import smtpd +import asyncore + +# Accept a SMTP message and offer a way to retrieve the last message via TCP Socket + +last_received_message = b"Nothing" + + +class CustomSMTPServer(smtpd.SMTPServer): + + def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): + global last_received_message + last_received_message = data + print('Receiving message from:', peer) + print('Message addressed from:', mailfrom) + print('Message addressed to :', rcpttos) + print('Message length :', len(data)) + return + + +# Just print out the last message received on plain TCP socket server +class EchoServer(asyncore.dispatcher): + + def __init__(self, host, port): + asyncore.dispatcher.__init__(self) + self.create_socket() + self.set_reuse_addr() + self.bind((host, port)) + self.listen(5) + + def handle_accepted(self, sock, addr): + global last_received_message + print('Incoming connection from %s' % repr(addr)) + sock.send(last_received_message) + last_received_message = b'' + + +server = CustomSMTPServer(('0.0.0.0', 11025), None) # SMTP mail goes here +server2 = EchoServer('0.0.0.0', 11080) # Echo back last message received +asyncore.loop() diff --git a/changedetectionio/tests/smtp/test_notification_smtp.py b/changedetectionio/tests/smtp/test_notification_smtp.py new file mode 100644 index 00000000..aad25385 --- /dev/null +++ b/changedetectionio/tests/smtp/test_notification_smtp.py @@ -0,0 +1,148 @@ +import json +import os +import time +import re +from flask import url_for +from changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \ + set_longer_modified_response +from changedetectionio.tests.util import extract_UUID_from_client +import logging +import base64 + +from changedetectionio.notification import ( + default_notification_body, + default_notification_format, + default_notification_title, + valid_notification_formats, +) + + +def get_last_message_from_smtp_server(): + import socket + host = 'mailserver' # as both code is running on same pc + port = 11080 # socket server port number + + client_socket = socket.socket() # instantiate + client_socket.connect((host, port)) # connect to the server + + data = client_socket.recv(50024).decode() # receive response + client_socket.close() # close the connection + return data + + +# Requires running the test SMTP server + +def test_check_notification_email_formats_default_HTML(client, live_server): + live_server_setup(live_server) + set_original_response() + + notification_url = 'mailto://changedetection@mailserver:11025/?to=fff@home.com' + + ##################### + # Set this up for when we remove the notification from the watch, it should fallback with these details + res = client.post( + url_for("settings_page"), + data={"application-notification_urls": notification_url, + "application-notification_title": "fallback-title " + default_notification_title, + "application-notification_body": "fallback-body
" + default_notification_body, + "application-notification_format": 'HTML', + "requests-time_between_check-minutes": 180, + 'application-fetch_backend': "html_requests"}, + follow_redirects=True + ) + assert b"Settings updated." in res.data + + # Add a watch and trigger a HTTP POST + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("form_quick_watch_add"), + data={"url": test_url, "tags": 'nice one'}, + follow_redirects=True + ) + + assert b"Watch added" in res.data + + wait_for_all_checks(client) + set_longer_modified_response() + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + time.sleep(3) + + msg = get_last_message_from_smtp_server() + assert len(msg) >= 1 + + # The email should have two bodies, and the text/html part should be
+ assert 'Content-Type: text/plain' in msg + assert '(added) So let\'s see what happens.\n' in msg # The plaintext part with \n + assert 'Content-Type: text/html' in msg + assert '(added) So let\'s see what happens.
' in msg # the html part + + +def test_check_notification_email_formats_default_Text_override_HTML(client, live_server): + live_server_setup(live_server) + set_original_response() + + notification_url = 'mailto://changedetection@mailserver:11025/?to=fff@home.com' + + ##################### + # Set this up for when we remove the notification from the watch, it should fallback with these details + res = client.post( + url_for("settings_page"), + data={"application-notification_urls": notification_url, + "application-notification_title": "fallback-title " + default_notification_title, + "application-notification_body": default_notification_body, + "application-notification_format": 'Text', + "requests-time_between_check-minutes": 180, + 'application-fetch_backend': "html_requests"}, + follow_redirects=True + ) + assert b"Settings updated." in res.data + + # Add a watch and trigger a HTTP POST + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("form_quick_watch_add"), + data={"url": test_url, "tags": 'nice one'}, + follow_redirects=True + ) + + assert b"Watch added" in res.data + + wait_for_all_checks(client) + set_longer_modified_response() + client.get(url_for("form_watch_checknow"), follow_redirects=True) + wait_for_all_checks(client) + + time.sleep(3) + msg = get_last_message_from_smtp_server() + assert len(msg) >= 1 + + # The email should not have two bodies, should be TEXT only + + assert 'Content-Type: text/plain' in msg + assert '(added) So let\'s see what happens.\n' in msg # The plaintext part with \n + assert 'Content-Type: text/html' not in msg + + set_original_response() + # Now override as HTML format + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "notification_format": 'HTML', + 'fetch_backend': "html_requests"}, + follow_redirects=True + ) + assert b"Updated watch." in res.data + wait_for_all_checks(client) + + time.sleep(3) + msg = get_last_message_from_smtp_server() + assert len(msg) >= 1 + + # The email should have two bodies, and the text/html part should be
+ assert 'Content-Type: text/plain' in msg + assert '(removed) So let\'s see what happens.\n' in msg # The plaintext part with \n + assert 'Content-Type: text/html' in msg + assert '(removed) So let\'s see what happens.
' in msg # the html part diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index cc5e6588..5f7b0582 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -3,7 +3,8 @@ import os import time import re from flask import url_for -from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks +from .util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, wait_for_all_checks, \ + set_longer_modified_response from . util import extract_UUID_from_client import logging import base64 @@ -272,7 +273,7 @@ def test_notification_validation(client, live_server): def test_notification_custom_endpoint_and_jinja2(client, live_server): - time.sleep(1) + #live_server_setup(live_server) # test_endpoint - that sends the contents of a file # test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt) @@ -283,12 +284,14 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server): res = client.post( url_for("settings_page"), - data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}", - "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }', - # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation - "application-notification_urls": test_notification_url, + data={ + "application-fetch_backend": "html_requests", "application-minutes_between_check": 180, - "application-fetch_backend": "html_requests" + "application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }', + "application-notification_format": default_notification_format, + "application-notification_urls": test_notification_url, + # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation + "application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}", }, follow_redirects=True ) @@ -313,9 +316,8 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server): client.get(url_for("form_watch_checknow"), follow_redirects=True) time.sleep(2) - with open("test-datastore/notification.txt", 'r') as f: - x=f.read() + x = f.read() j = json.loads(x) assert j['url'].startswith('http://localhost') assert j['secret'] == 444 @@ -326,5 +328,9 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server): notification_url = f.read() assert 'xxx=http' in notification_url - os.unlink("test-datastore/notification-url.txt") + # Should always be automatically detected as JSON content type even when we set it as 'Text' (default) + assert os.path.isfile("test-datastore/notification-content-type.txt") + with open("test-datastore/notification-content-type.txt", 'r') as f: + assert 'application/json' in f.read() + os.unlink("test-datastore/notification-url.txt") diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py index 13e3fff9..50671f3d 100644 --- a/changedetectionio/tests/util.py +++ b/changedetectionio/tests/util.py @@ -38,7 +38,25 @@ def set_modified_response(): f.write(test_return_data) return None +def set_longer_modified_response(): + test_return_data = """ + modified head title + + Some initial text
+

which has this one new line

+
+ So let's see what happens.
+ So let's see what happens.
+ So let's see what happens.
+ So let's see what happens.
+ + + """ + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(test_return_data) + + return None def set_more_modified_response(): test_return_data = """ modified head title @@ -187,6 +205,9 @@ def live_server_setup(live_server): with open("test-datastore/notification-url.txt", "w") as f: f.write(request.url) + with open("test-datastore/notification-content-type.txt", "w") as f: + f.write(request.content_type) + print("\n>> Test notification endpoint was hit.\n", data) return "Text was set" diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 7c2c5792..377711ba 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -32,15 +32,17 @@ class update_worker(threading.Thread): watch_history = watch.history dates = list(watch_history.keys()) + # Add text that was triggered + snapshot_contents = watch.get_history_snapshot(dates[-1]) # HTML needs linebreak, but MarkDown and Text can use a linefeed if n_object['notification_format'] == 'HTML': line_feed_sep = "
" + # Snapshot will be plaintext on the disk, convert to some kind of HTML + snapshot_contents = snapshot_contents.replace('\n', line_feed_sep) else: line_feed_sep = "\n" - # Add text that was triggered - snapshot_contents = watch.get_history_snapshot(dates[-1]) trigger_text = watch.get('trigger_text', []) triggered_text = '' @@ -78,6 +80,9 @@ class update_worker(threading.Thread): # Would be better if this was some kind of Object where Watch can reference the parent datastore etc v = watch.get(var_name) if v and not watch.get('notification_muted'): + if var_name == 'notification_format' and v == default_notification_format_for_watch: + return self.datastore.data['settings']['application'].get('notification_format') + return v tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))