mirror of
https://github.com/dgtlmoon/changedetection.io.git
synced 2026-01-21 22:50:25 +00:00
3d14df6a11
Multi-language / Translations Support (#3696) - Complete internationalization system implemented - Support for 7 languages: Czech (cs), German (de), French (fr), Italian (it), Korean (ko), Chinese Simplified (zh), Chinese Traditional (zh_TW) - Language selector with localized flags and theming - Flash message translations - Multiple translation fixes and improvements across all languages - Language setting preserved across redirects Pluggable Content Fetchers (#3653) - New architecture for extensible content fetcher system - Allows custom fetcher implementations Image / Screenshot Comparison Processor (#3680) - New processor for visual change detection (disabled for this release) - Supporting CSS/JS infrastructure added UI Improvements Design & Layout - Auto-generated tag color schemes - Simplified login form styling - Removed hard-coded CSS, moved to SCSS variables - Tag UI cleanup and improvements - Automatic tab wrapper functionality - Menu refactoring for better organization - Cleanup of offset settings - Hide sticky tabs on narrow viewports - Improved responsive layout (#3702) User Experience - Modal alerts/confirmations on delete/clear operations (#3693, #3598, #3382) - Auto-add https:// to URLs in quickwatch form if not present - Better redirect handling on login (#3699) - 'Recheck all' now returns to correct group/tag (#3673) - Language set redirect keeps hash fragment - More friendly human-readable text throughout UI Performance & Reliability Scheduler & Processing - Soft delays instead of blocking time.sleep() calls (#3710) - More resilient handling of same UUID being processed (#3700) - Better Puppeteer timeout handling - Improved Puppeteer shutdown/cleanup (#3692) - Requests cleanup now properly async History & Rendering - Faster server-side "difference" rendering on History page (#3442) - Show ignored/triggered rows in history - API: Retry watch data if watch dict changed (more reliable) API Improvements - Watch get endpoint: retry mechanism for changed watch data - WatchHistoryDiff API endpoint includes extra format args (#3703) Testing Improvements - Replace time.sleep with wait_for_notification_endpoint_output (#3716) - Test for mode switching (#3701) - Test for #3720 added (#3725) - Extract-text difference test fixes - Improved dev workflow Bug Fixes - Notification error text output (#3672, #3669, #3280) - HTML validation fixes (#3704) - Template discovery path fixes - Notification debug log now uses system locale for dates/times - Puppeteer spelling mistake in log output - Recalculation on anchor change - Queue bubble update disabled temporarily Dependency Updates - beautifulsoup4 updated (#3724) - psutil 7.1.0 → 7.2.1 (#3723) - python-engineio ~=4.12.3 → ~=4.13.0 (#3707) - python-socketio ~=5.14.3 → ~=5.16.0 (#3706) - flask-socketio ~=5.5.1 → ~=5.6.0 (#3691) - brotli ~=1.1 → ~=1.2 (#3687) - lxml updated (#3590) - pytest ~=7.2 → ~=9.0 (#3676) - jsonschema ~=4.0 → ~=4.25 (#3618) - pluggy ~=1.5 → ~=1.6 (#3616) - cryptography 44.0.1 → 46.0.3 (security) (#3589) Documentation - README updated with viewport size setup information Development Infrastructure - Dev container only built on dev branch - Improved dev workflow tooling
341 lines
12 KiB
Python
341 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
import json
|
|
import time
|
|
import os
|
|
|
|
from flask import url_for
|
|
from .util import live_server_setup, wait_for_all_checks, delete_all_watches
|
|
from ..model import CONDITIONS_MATCH_LOGIC_DEFAULT
|
|
|
|
|
|
def set_original_response(datastore_path, number="50"):
|
|
test_return_data = f"""<html>
|
|
<body>
|
|
<h1>Test Page for Conditions</h1>
|
|
<p>This page contains a number that will be tested with conditions.</p>
|
|
<div class="number-container">Current value: {number}</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
|
|
f.write(test_return_data)
|
|
|
|
def set_number_in_range_response(datastore_path, number="75"):
|
|
test_return_data = f"""<html>
|
|
<body>
|
|
<h1>Test Page for Conditions</h1>
|
|
<p>This page contains a number that will be tested with conditions.</p>
|
|
<div class="number-container">Current value: {number}</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
|
|
f.write(test_return_data)
|
|
|
|
def set_number_out_of_range_response(datastore_path, number="150"):
|
|
test_return_data = f"""<html>
|
|
<body>
|
|
<h1>Test Page for Conditions</h1>
|
|
<p>This page contains a number that will be tested with conditions.</p>
|
|
<div class="number-container">Current value: {number}</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
|
|
f.write(test_return_data)
|
|
|
|
|
|
# def test_setup(client, live_server, measure_memory_usage, datastore_path):
|
|
"""Test that both text and number conditions work together with AND logic."""
|
|
# live_server_setup(live_server) # Setup on conftest per function
|
|
|
|
def test_conditions_with_text_and_number(client, live_server, measure_memory_usage, datastore_path):
|
|
"""Test that both text and number conditions work together with AND logic."""
|
|
|
|
set_original_response(datastore_path=datastore_path, number="50")
|
|
|
|
|
|
test_url = url_for('test_endpoint', _external=True)
|
|
|
|
# Add our URL to the import page
|
|
uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
wait_for_all_checks(client)
|
|
|
|
# Configure the watch with two conditions connected with AND:
|
|
# 1. The page filtered text must contain "5" (first digit of value)
|
|
# 2. The extracted number should be >= 20 and <= 100
|
|
res = client.post(
|
|
url_for("ui.ui_edit.edit_page", uuid="first"),
|
|
data={
|
|
"url": test_url,
|
|
"fetch_backend": "html_requests",
|
|
"include_filters": ".number-container",
|
|
"title": "Number AND Text Condition Test",
|
|
"conditions_match_logic": CONDITIONS_MATCH_LOGIC_DEFAULT, # ALL = AND logic
|
|
"conditions-0-operator": "in",
|
|
"conditions-0-field": "page_filtered_text",
|
|
"conditions-0-value": "5",
|
|
|
|
"conditions-1-operator": ">=",
|
|
"conditions-1-field": "extracted_number",
|
|
"conditions-1-value": "20",
|
|
|
|
"conditions-2-operator": "<=",
|
|
"conditions-2-field": "extracted_number",
|
|
"conditions-2-value": "100",
|
|
|
|
# So that 'operations' from pluggy discovery are tested
|
|
"conditions-3-operator": "length_min",
|
|
"conditions-3-field": "page_filtered_text",
|
|
"conditions-3-value": "1",
|
|
|
|
# So that 'operations' from pluggy discovery are tested
|
|
"conditions-4-operator": "length_max",
|
|
"conditions-4-field": "page_filtered_text",
|
|
"conditions-4-value": "100",
|
|
|
|
# So that 'operations' from pluggy discovery are tested
|
|
"conditions-5-operator": "contains_regex",
|
|
"conditions-5-field": "page_filtered_text",
|
|
"conditions-5-value": "\d",
|
|
"time_between_check_use_default": "y",
|
|
},
|
|
follow_redirects=True
|
|
)
|
|
assert b"Updated watch." in res.data
|
|
|
|
wait_for_all_checks(client)
|
|
client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
|
|
time.sleep(0.2)
|
|
|
|
wait_for_all_checks(client)
|
|
|
|
# Case 1
|
|
set_number_in_range_response(datastore_path=datastore_path, number="70.5")
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
wait_for_all_checks(client)
|
|
|
|
time.sleep(2)
|
|
# 75 is > 20 and < 100 and contains "5"
|
|
res = client.get(url_for("watchlist.index"))
|
|
assert b'has-unread-changes' in res.data
|
|
|
|
|
|
# Case 2: Change with one condition violated
|
|
# Number out of range (150) but contains '5'
|
|
client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
|
|
time.sleep(0.2)
|
|
|
|
set_number_out_of_range_response(datastore_path=datastore_path, number="150.5")
|
|
|
|
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
wait_for_all_checks(client)
|
|
|
|
# Should NOT be marked as having changes since not all conditions are met
|
|
res = client.get(url_for("watchlist.index"))
|
|
assert b'has-unread-changes' not in res.data
|
|
|
|
delete_all_watches(client)
|
|
|
|
# The 'validate' button next to each rule row
|
|
def test_condition_validate_rule_row(client, live_server, measure_memory_usage, datastore_path):
|
|
|
|
set_original_response(datastore_path=datastore_path, number="50")
|
|
|
|
test_url = url_for('test_endpoint', _external=True)
|
|
|
|
# Add our URL to the import page
|
|
uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
wait_for_all_checks(client)
|
|
|
|
uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
|
|
|
|
# the front end submits the current form state which should override the watch in a temporary copy
|
|
res = client.post(
|
|
url_for("conditions.verify_condition_single_rule", watch_uuid=uuid), # Base URL
|
|
query_string={"rule": json.dumps({"field": "extracted_number", "operator": "==", "value": "50"})},
|
|
data={'include_filter': ""},
|
|
follow_redirects=True
|
|
)
|
|
assert res.status_code == 200
|
|
assert b'success' in res.data
|
|
|
|
# Now a number that does not equal what is found in the last fetch
|
|
res = client.post(
|
|
url_for("conditions.verify_condition_single_rule", watch_uuid=uuid), # Base URL
|
|
query_string={"rule": json.dumps({"field": "extracted_number", "operator": "==", "value": "111111"})},
|
|
data={'include_filter': ""},
|
|
follow_redirects=True
|
|
)
|
|
assert res.status_code == 200
|
|
assert b'false' in res.data
|
|
|
|
# Now custom filter that exists
|
|
res = client.post(
|
|
url_for("conditions.verify_condition_single_rule", watch_uuid=uuid), # Base URL
|
|
query_string={"rule": json.dumps({"field": "extracted_number", "operator": "==", "value": "50"})},
|
|
data={'include_filter': ".number-container"},
|
|
follow_redirects=True
|
|
)
|
|
assert res.status_code == 200
|
|
assert b'success' in res.data
|
|
|
|
# Now custom filter that DOES NOT exists
|
|
res = client.post(
|
|
url_for("conditions.verify_condition_single_rule", watch_uuid=uuid), # Base URL
|
|
query_string={"rule": json.dumps({"field": "extracted_number", "operator": "==", "value": "50"})},
|
|
data={'include_filters': ".NOT-container"},
|
|
follow_redirects=True
|
|
)
|
|
assert res.status_code == 200
|
|
assert b'false' in res.data
|
|
# cleanup for the next
|
|
client.get(
|
|
url_for("ui.form_delete", uuid="all"),
|
|
follow_redirects=True
|
|
)
|
|
|
|
|
|
|
|
# If there was only a change in the whitespacing, then we shouldnt have a change detected
|
|
def test_wordcount_conditions_plugin(client, live_server, measure_memory_usage, datastore_path):
|
|
|
|
|
|
test_return_data = """<html>
|
|
<body>
|
|
Some initial text<br>
|
|
<p>Which is across multiple lines</p>
|
|
<br>
|
|
So let's see what happens. <br>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
|
|
f.write(test_return_data)
|
|
|
|
# Add our URL to the import page
|
|
test_url = url_for('test_endpoint', _external=True)
|
|
uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)
|
|
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
|
|
# Give the thread time to pick it up
|
|
wait_for_all_checks(client)
|
|
|
|
# Check it saved
|
|
res = client.get(
|
|
url_for("ui.ui_edit.edit_page", uuid="first"),
|
|
)
|
|
|
|
# Assert the word count is counted correctly
|
|
assert b'<td>13</td>' in res.data
|
|
|
|
# cleanup for the next
|
|
client.get(
|
|
url_for("ui.form_delete", uuid="all"),
|
|
follow_redirects=True
|
|
)
|
|
|
|
# If there was only a change in the whitespacing, then we shouldnt have a change detected
|
|
def test_lev_conditions_plugin(client, live_server, measure_memory_usage, datastore_path):
|
|
# This should break..
|
|
|
|
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
|
|
f.write("""<html>
|
|
<body>
|
|
Some initial text<br>
|
|
<p>Which is across multiple lines</p>
|
|
<br>
|
|
So let's see what happens. <br>
|
|
</body>
|
|
</html>
|
|
""")
|
|
|
|
# Add our URL to the import page
|
|
test_url = url_for('test_endpoint', _external=True)
|
|
res = client.post(
|
|
url_for("ui.ui_views.form_quick_watch_add"),
|
|
data={"url": test_url, "tags": '', 'edit_and_watch_submit_button': 'Edit > Watch'},
|
|
follow_redirects=True
|
|
)
|
|
assert b"Watch added in Paused state, saving will unpause" in res.data
|
|
|
|
uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))
|
|
# Give the thread time to pick it up
|
|
wait_for_all_checks(client)
|
|
res = client.post(
|
|
url_for("ui.ui_edit.edit_page", uuid=uuid, unpause_on_save=1),
|
|
data={
|
|
"url": test_url,
|
|
"fetch_backend": "html_requests",
|
|
"conditions_match_logic": CONDITIONS_MATCH_LOGIC_DEFAULT, # ALL = AND logic
|
|
"conditions-0-field": "levenshtein_ratio",
|
|
"conditions-0-operator": "<",
|
|
"conditions-0-value": "0.8", # needs to be more of a diff to trigger a change
|
|
"time_between_check_use_default": "y"
|
|
},
|
|
follow_redirects=True
|
|
)
|
|
|
|
assert b"unpaused" in res.data
|
|
|
|
wait_for_all_checks(client)
|
|
res = client.get(url_for("watchlist.index"))
|
|
assert b'has-unread-changes' not in res.data
|
|
|
|
# Check the content saved initially, even tho a condition was set - this is the first snapshot so shouldnt be affected by conditions
|
|
res = client.get(
|
|
url_for("ui.ui_preview.preview_page", uuid=uuid),
|
|
follow_redirects=True
|
|
)
|
|
assert b'Which is across multiple lines' in res.data
|
|
|
|
|
|
############### Now change it a LITTLE bit...
|
|
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
|
|
f.write("""<html>
|
|
<body>
|
|
Some initial text<br>
|
|
<p>Which is across multiple lines</p>
|
|
<br>
|
|
So let's see what happenxxxxxxxxx. <br>
|
|
</body>
|
|
</html>
|
|
""")
|
|
|
|
res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
assert b'Queued 1 watch for rechecking.' in res.data
|
|
wait_for_all_checks(client)
|
|
|
|
res = client.get(url_for("watchlist.index"))
|
|
assert b'has-unread-changes' not in res.data #because this will be like 0.90 not 0.8 threshold
|
|
|
|
############### Now change it a MORE THAN 50%
|
|
test_return_data = """<html>
|
|
<body>
|
|
Some sxxxx<br>
|
|
<p>Which is across a lines</p>
|
|
<br>
|
|
ok. <br>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
with open(os.path.join(datastore_path, "endpoint-content.txt"), "w") as f:
|
|
f.write(test_return_data)
|
|
res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
|
|
assert b'Queued 1 watch for rechecking.' in res.data
|
|
wait_for_all_checks(client)
|
|
res = client.get(url_for("watchlist.index"))
|
|
assert b'has-unread-changes' in res.data
|
|
# cleanup for the next
|
|
client.get(
|
|
url_for("ui.form_delete", uuid="all"),
|
|
follow_redirects=True
|
|
) |