mirror of
https://github.com/dgtlmoon/changedetection.io.git
synced 2026-02-08 07:16:03 +00:00
3d14df6a11
Multi-language / Translations Support (#3696) - Complete internationalization system implemented - Support for 7 languages: Czech (cs), German (de), French (fr), Italian (it), Korean (ko), Chinese Simplified (zh), Chinese Traditional (zh_TW) - Language selector with localized flags and theming - Flash message translations - Multiple translation fixes and improvements across all languages - Language setting preserved across redirects Pluggable Content Fetchers (#3653) - New architecture for extensible content fetcher system - Allows custom fetcher implementations Image / Screenshot Comparison Processor (#3680) - New processor for visual change detection (disabled for this release) - Supporting CSS/JS infrastructure added UI Improvements Design & Layout - Auto-generated tag color schemes - Simplified login form styling - Removed hard-coded CSS, moved to SCSS variables - Tag UI cleanup and improvements - Automatic tab wrapper functionality - Menu refactoring for better organization - Cleanup of offset settings - Hide sticky tabs on narrow viewports - Improved responsive layout (#3702) User Experience - Modal alerts/confirmations on delete/clear operations (#3693, #3598, #3382) - Auto-add https:// to URLs in quickwatch form if not present - Better redirect handling on login (#3699) - 'Recheck all' now returns to correct group/tag (#3673) - Language set redirect keeps hash fragment - More friendly human-readable text throughout UI Performance & Reliability Scheduler & Processing - Soft delays instead of blocking time.sleep() calls (#3710) - More resilient handling of same UUID being processed (#3700) - Better Puppeteer timeout handling - Improved Puppeteer shutdown/cleanup (#3692) - Requests cleanup now properly async History & Rendering - Faster server-side "difference" rendering on History page (#3442) - Show ignored/triggered rows in history - API: Retry watch data if watch dict changed (more reliable) API Improvements - Watch get endpoint: retry mechanism for changed watch data - WatchHistoryDiff API endpoint includes extra format args (#3703) Testing Improvements - Replace time.sleep with wait_for_notification_endpoint_output (#3716) - Test for mode switching (#3701) - Test for #3720 added (#3725) - Extract-text difference test fixes - Improved dev workflow Bug Fixes - Notification error text output (#3672, #3669, #3280) - HTML validation fixes (#3704) - Template discovery path fixes - Notification debug log now uses system locale for dates/times - Puppeteer spelling mistake in log output - Recalculation on anchor change - Queue bubble update disabled temporarily Dependency Updates - beautifulsoup4 updated (#3724) - psutil 7.1.0 → 7.2.1 (#3723) - python-engineio ~=4.12.3 → ~=4.13.0 (#3707) - python-socketio ~=5.14.3 → ~=5.16.0 (#3706) - flask-socketio ~=5.5.1 → ~=5.6.0 (#3691) - brotli ~=1.1 → ~=1.2 (#3687) - lxml updated (#3590) - pytest ~=7.2 → ~=9.0 (#3676) - jsonschema ~=4.0 → ~=4.25 (#3618) - pluggy ~=1.5 → ~=1.6 (#3616) - cryptography 44.0.1 → 46.0.3 (security) (#3589) Documentation - README updated with viewport size setup information Development Infrastructure - Dev container only built on dev branch - Improved dev workflow tooling
419 lines
16 KiB
Python
419 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
import os
|
|
|
|
import time
|
|
from flask import url_for
|
|
from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \
|
|
extract_UUID_from_client, delete_all_watches
|
|
|
|
sleep_time_for_fetch_thread = 3
|
|
|
|
|
|
# Basic test to check inscriptus is not adding return line chars, basically works etc
|
|
def test_inscriptus():
|
|
from inscriptis import get_text
|
|
html_content = "<html><body>test!<br>ok man</body></html>"
|
|
stripped_text_from_html = get_text(html_content)
|
|
assert stripped_text_from_html == 'test!\nok man'
|
|
|
|
|
|
|
|
def test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path):
|
|
set_original_response(datastore_path=datastore_path)
|
|
|
|
uuid = client.application.config.get('DATASTORE').add_watch(url=url_for('test_endpoint', _external=True))
|
|
|
|
# Do this a few times.. ensures we dont accidently set the status
|
|
for n in range(3):
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
|
|
# Give the thread time to pick it up
|
|
wait_for_all_checks(client)
|
|
|
|
# It should report nothing found (no new 'has-unread-changes' class)
|
|
res = client.get(url_for("watchlist.index"))
|
|
assert b'has-unread-changes' not in res.data
|
|
assert b'test-endpoint' in res.data
|
|
|
|
# Default no password set, this stuff should be always available.
|
|
|
|
assert b"SETTINGS" in res.data
|
|
assert b"BACKUP" in res.data
|
|
assert b"IMPORT" in res.data
|
|
|
|
#####################
|
|
|
|
# Check HTML conversion detected and workd
|
|
res = client.get(
|
|
url_for("ui.ui_preview.preview_page", uuid="first"),
|
|
follow_redirects=True
|
|
)
|
|
# Check this class does not appear (that we didnt see the actual source)
|
|
assert b'foobar-detection' not in res.data
|
|
|
|
# Make a change
|
|
set_modified_response(datastore_path=datastore_path)
|
|
|
|
# Force recheck
|
|
res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
assert b'Queued 1 watch for rechecking.' in res.data
|
|
|
|
wait_for_all_checks(client)
|
|
|
|
uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
|
|
|
|
# Check the 'get latest snapshot works'
|
|
res = client.get(url_for("ui.ui_edit.watch_get_latest_html", uuid=uuid))
|
|
assert b'which has this one new line' in res.data
|
|
|
|
# Now something should be ready, indicated by having a 'has-unread-changes' class
|
|
res = client.get(url_for("watchlist.index"))
|
|
assert b'has-unread-changes' in res.data
|
|
|
|
# #75, and it should be in the RSS feed
|
|
rss_token = extract_rss_token_from_UI(client)
|
|
res = client.get(url_for("rss.feed", token=rss_token, _external=True))
|
|
expected_url = url_for('test_endpoint', _external=True)
|
|
assert b'<rss' in res.data
|
|
|
|
# re #16 should have the diff in here too
|
|
assert b'which has this one new line' in res.data
|
|
assert b'CDATA' in res.data
|
|
|
|
#
|
|
# Following the 'diff' link, it should no longer display as 'has-unread-changes' even after we recheck it a few times
|
|
res = client.get(url_for("ui.ui_diff.diff_history_page", uuid=uuid))
|
|
assert b'selected=""' in res.data, "Confirm diff history page loaded"
|
|
|
|
assert b'Which is across multiple lines' in res.data
|
|
# The linefeed should have been added ( @BR@ was replaced with a linefeed because this is htmlcolor kinda display )
|
|
assert b'Which is across multiple lines</span>\n' in res.data
|
|
|
|
# Check the [preview] pulls the right one
|
|
res = client.get(
|
|
url_for("ui.ui_preview.preview_page", uuid="first"),
|
|
follow_redirects=True
|
|
)
|
|
assert b'which has this one new line' in res.data
|
|
assert b'Which is across multiple lines' not in res.data
|
|
|
|
wait_for_all_checks(client)
|
|
|
|
|
|
# Do this a few times.. ensures we don't accidently set the status
|
|
for n in range(2):
|
|
res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
# Give the thread time to pick it up
|
|
wait_for_all_checks(client)
|
|
|
|
# It should report nothing found (no new 'has-unread-changes' class)
|
|
res = client.get(url_for("watchlist.index"))
|
|
|
|
assert b'has-unread-changes' not in res.data
|
|
assert b'class="has-unread-changes' not in res.data
|
|
assert b'head title' in res.data # Should be ON by default
|
|
assert b'test-endpoint' in res.data
|
|
|
|
# Recheck it but only with a title change, content wasnt changed
|
|
set_original_response(datastore_path=datastore_path, extra_title=" and more")
|
|
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
wait_for_all_checks(client)
|
|
res = client.get(url_for("watchlist.index"))
|
|
assert b'head title and more' in res.data
|
|
|
|
# Be sure the last_viewed is going to be greater than the last snapshot
|
|
time.sleep(1)
|
|
|
|
# hit the mark all viewed link
|
|
res = client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
|
|
|
|
assert b'class="has-unread-changes' not in res.data
|
|
assert b'has-unread-changes' not in res.data
|
|
|
|
# #2458 "clear history" should make the Watch object update its status correctly when the first snapshot lands again
|
|
client.get(url_for("ui.clear_watch_history", uuid=uuid))
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
wait_for_all_checks(client)
|
|
res = client.get(url_for("watchlist.index"))
|
|
assert b'preview/' in res.data
|
|
|
|
#
|
|
# Cleanup everything
|
|
delete_all_watches(client)
|
|
|
|
def test_title_scraper(client, live_server, measure_memory_usage, datastore_path):
|
|
|
|
set_original_response(datastore_path=datastore_path)
|
|
uuid = client.application.config.get('DATASTORE').add_watch(url=url_for('test_endpoint', _external=True))
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
wait_for_all_checks()
|
|
|
|
# It should report nothing found (no new 'has-unread-changes' class)
|
|
res = client.get(url_for("watchlist.index"))
|
|
|
|
assert b'head title' in res.data # Should be ON by default
|
|
|
|
# Recheck it but only with a title change, content wasnt changed
|
|
set_original_response(datastore_path=datastore_path, extra_title=" and more")
|
|
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
wait_for_all_checks(client)
|
|
res = client.get(url_for("watchlist.index"))
|
|
assert b'head title and more' in res.data
|
|
|
|
# disable <title> pickup
|
|
res = client.post(
|
|
url_for("settings.settings_page"),
|
|
data={"application-ui-use_page_title_in_list": "",
|
|
"requests-time_between_check-minutes": 180,
|
|
'application-fetch_backend': "html_requests"},
|
|
follow_redirects=True
|
|
)
|
|
|
|
set_original_response(datastore_path=datastore_path, extra_title=" SHOULD NOT APPEAR")
|
|
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
wait_for_all_checks(client)
|
|
res = client.get(url_for("watchlist.index"))
|
|
assert b'SHOULD NOT APPEAR' not in res.data
|
|
|
|
delete_all_watches(client)
|
|
|
|
def test_title_scraper_html_only(client, live_server, measure_memory_usage, datastore_path):
|
|
|
|
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
|
|
f.write('"My text document\nWhere I talk about <title>\nwhich should not get registered\n</title>')
|
|
|
|
test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
|
|
|
|
uuid = client.application.config.get('DATASTORE').add_watch(test_url)
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
wait_for_all_checks()
|
|
|
|
# It should report nothing found (no new 'has-unread-changes' class)
|
|
res = client.get(url_for("watchlist.index"))
|
|
|
|
assert b'which should not get registered' not in res.data # Should be ON by default
|
|
assert not live_server.app.config['DATASTORE'].data['watching'][uuid].get('title')
|
|
|
|
|
|
|
|
|
|
# Server says its plaintext, we should always treat it as plaintext, and then if they have a filter, try to apply that
|
|
def test_requests_timeout(client, live_server, measure_memory_usage, datastore_path):
|
|
delay = 2
|
|
test_url = url_for('test_endpoint', delay=delay, _external=True)
|
|
|
|
res = client.post(
|
|
url_for("settings.settings_page"),
|
|
data={"application-ui-use_page_title_in_list": "",
|
|
"requests-time_between_check-minutes": 180,
|
|
"requests-timeout": delay - 1,
|
|
'application-fetch_backend': "html_requests"},
|
|
follow_redirects=True
|
|
)
|
|
|
|
# Add our URL to the import page
|
|
uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
wait_for_all_checks(client)
|
|
|
|
# requests takes >2 sec but we timeout at 1 second
|
|
res = client.get(url_for("watchlist.index"))
|
|
assert b'Read timed out. (read timeout=1)' in res.data
|
|
|
|
##### Now set a longer timeout
|
|
res = client.post(
|
|
url_for("settings.settings_page"),
|
|
data={"application-ui-use_page_title_in_list": "",
|
|
"requests-time_between_check-minutes": 180,
|
|
"requests-timeout": delay + 1, # timeout should be a second more than the reply time
|
|
'application-fetch_backend': "html_requests"},
|
|
follow_redirects=True
|
|
)
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
|
|
wait_for_all_checks(client)
|
|
|
|
res = client.get(url_for("watchlist.index"))
|
|
assert b'Read timed out' not in res.data
|
|
|
|
def test_non_text_mime_or_downloads(client, live_server, measure_memory_usage, datastore_path):
|
|
"""
|
|
|
|
https://github.com/dgtlmoon/changedetection.io/issues/3434
|
|
I noticed that a watched website can be monitored fine as long as the server sends content-type: text/plain; charset=utf-8,
|
|
but once the server sends content-type: application/octet-stream (which is usually done to force the browser to show the Download dialog),
|
|
changedetection somehow ignores all line breaks and treats the document file as if everything is on one line.
|
|
|
|
WHAT THIS DOES - makes the system rely on 'magic' to determine what is it
|
|
|
|
:param client:
|
|
:param live_server:
|
|
:param measure_memory_usage:
|
|
:return:
|
|
"""
|
|
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
|
|
f.write("""some random text that should be split by line
|
|
and not parsed with html_to_text
|
|
this way we know that it correctly parsed as plain text
|
|
\r\n
|
|
ok\r\n
|
|
got it\r\n
|
|
""")
|
|
|
|
test_url = url_for('test_endpoint', content_type="application/octet-stream", _external=True)
|
|
|
|
# Add our URL to the import page
|
|
uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
|
|
wait_for_all_checks(client)
|
|
|
|
### check the front end
|
|
res = client.get(
|
|
url_for("ui.ui_preview.preview_page", uuid="first"),
|
|
follow_redirects=True
|
|
)
|
|
assert b"some random text that should be split by line\n" in res.data
|
|
####
|
|
|
|
# Check the snapshot by API that it has linefeeds too
|
|
watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
|
|
api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
|
|
res = client.get(
|
|
url_for("watchhistory", uuid=watch_uuid),
|
|
headers={'x-api-key': api_key},
|
|
)
|
|
|
|
# Fetch a snapshot by timestamp, check the right one was found
|
|
res = client.get(
|
|
url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(res.json.keys())[-1]),
|
|
headers={'x-api-key': api_key},
|
|
)
|
|
assert b"some random text that should be split by line\n" in res.data
|
|
|
|
|
|
delete_all_watches(client)
|
|
|
|
|
|
def test_standard_text_plain(client, live_server, measure_memory_usage, datastore_path):
|
|
"""
|
|
|
|
https://github.com/dgtlmoon/changedetection.io/issues/3434
|
|
I noticed that a watched website can be monitored fine as long as the server sends content-type: text/plain; charset=utf-8,
|
|
but once the server sends content-type: application/octet-stream (which is usually done to force the browser to show the Download dialog),
|
|
changedetection somehow ignores all line breaks and treats the document file as if everything is on one line.
|
|
|
|
The real bug here can be that it will try to process plain-text as HTML, losing <etc>
|
|
|
|
:param client:
|
|
:param live_server:
|
|
:param measure_memory_usage:
|
|
:return:
|
|
"""
|
|
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
|
|
f.write("""some random text that should be split by line
|
|
and not parsed with html_to_text
|
|
<title>Even this title should stay because we are just plain text</title>
|
|
this way we know that it correctly parsed as plain text
|
|
\r\n
|
|
ok\r\n
|
|
got it\r\n
|
|
""")
|
|
|
|
test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
|
|
|
|
# Add our URL to the import page
|
|
uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
|
|
wait_for_all_checks(client)
|
|
|
|
### check the front end
|
|
res = client.get(
|
|
url_for("ui.ui_preview.preview_page", uuid="first"),
|
|
follow_redirects=True
|
|
)
|
|
|
|
assert b"some random text that should be split by line\n" in res.data
|
|
####
|
|
|
|
# Check the snapshot by API that it has linefeeds too
|
|
watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
|
|
api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
|
|
res = client.get(
|
|
url_for("watchhistory", uuid=watch_uuid),
|
|
headers={'x-api-key': api_key},
|
|
)
|
|
|
|
# Fetch a snapshot by timestamp, check the right one was found
|
|
res = client.get(
|
|
url_for("watchsinglehistory", uuid=watch_uuid, timestamp=list(res.json.keys())[-1]),
|
|
headers={'x-api-key': api_key},
|
|
)
|
|
assert b"some random text that should be split by line\n" in res.data
|
|
assert b"<title>Even this title should stay because we are just plain text</title>" in res.data
|
|
|
|
delete_all_watches(client)
|
|
|
|
# Server says its plaintext, we should always treat it as plaintext
|
|
def test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage, datastore_path):
|
|
|
|
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
|
|
f.write("""<?xml version="1.0" encoding="utf-8"?>
|
|
<resources xmlns:tools="http://schemas.android.com/tools">
|
|
<!--Activity and fragment titles-->
|
|
<string name="feed_update_receiver_name">Abonnementen bijwerken</string>
|
|
</resources>
|
|
""")
|
|
|
|
test_url = url_for('test_endpoint', content_type="text/plain", _external=True)
|
|
|
|
# Add our URL to the import page
|
|
uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
|
|
wait_for_all_checks(client)
|
|
|
|
res = client.get(
|
|
url_for("ui.ui_preview.preview_page", uuid="first"),
|
|
follow_redirects=True
|
|
)
|
|
|
|
assert b'<string name="feed_update_receiver_name"' in res.data
|
|
|
|
delete_all_watches(client)
|
|
|
|
# Server says its plaintext, we should always treat it as plaintext, and then if they have a filter, try to apply that
|
|
def test_plaintext_even_if_xml_content_and_can_apply_filters(client, live_server, measure_memory_usage, datastore_path):
|
|
|
|
|
|
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
|
|
f.write("""<?xml version="1.0" encoding="utf-8"?>
|
|
<resources xmlns:tools="http://schemas.android.com/tools">
|
|
<!--Activity and fragment titles-->
|
|
<string name="feed_update_receiver_name">Abonnementen bijwerken</string>
|
|
<foobar>ok man</foobar>
|
|
</resources>
|
|
""")
|
|
|
|
test_url=url_for('test_endpoint', content_type="text/plain", _external=True)
|
|
uuid = client.application.config.get('DATASTORE').add_watch(url=test_url, extras={"include_filters": ['//string']})
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
wait_for_all_checks(client)
|
|
|
|
res = client.get(
|
|
url_for("ui.ui_preview.preview_page", uuid="first"),
|
|
follow_redirects=True
|
|
)
|
|
|
|
# Check that the string element with the correct name attribute is present
|
|
# Note: namespace declarations may be included when extracting elements, which is correct XML behavior
|
|
assert b'feed_update_receiver_name' in res.data
|
|
assert b'Abonnementen bijwerken' in res.data
|
|
assert b'<foobar' not in res.data
|
|
|
|
res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True)
|