Compare commits

..

3 Commits

Author SHA1 Message Date
dgtlmoon
3acf9fa60d BUmp docs 2025-08-25 14:39:00 +02:00
dgtlmoon
001d294654 Validate API calls against our YAML 2025-08-25 14:34:51 +02:00
dgtlmoon
994d17fc7a Improvements to schema and adding YAML validation 2025-08-25 13:41:59 +02:00
47 changed files with 161 additions and 883 deletions

View File

@@ -33,6 +33,7 @@ venv/
# Test and development files
test-datastore/
tests/
docs/
*.md
!README.md

View File

@@ -41,7 +41,7 @@ jobs:
steps:
- uses: actions/checkout@v5
- name: Set up Python 3.11
uses: actions/setup-python@v6
uses: actions/setup-python@v5
with:
python-version: 3.11

View File

@@ -9,7 +9,7 @@ jobs:
steps:
- uses: actions/checkout@v5
- name: Set up Python
uses: actions/setup-python@v6
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install pypa/build
@@ -39,7 +39,7 @@ jobs:
name: python-package-distributions
path: dist/
- name: Set up Python 3.11
uses: actions/setup-python@v6
uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Test that the basic pip built package runs without error

View File

@@ -48,7 +48,7 @@ jobs:
steps:
- uses: actions/checkout@v5
- name: Set up Python 3.11
uses: actions/setup-python@v6
uses: actions/setup-python@v5
with:
python-version: 3.11

View File

@@ -24,7 +24,7 @@ jobs:
# Mainly just for link/flake8
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@v6
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}

View File

@@ -5,6 +5,7 @@ ARG PYTHON_VERSION=3.11
FROM python:${PYTHON_VERSION}-slim-bookworm AS builder
# See `cryptography` pin comment in requirements.txt
ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1
RUN apt-get update && apt-get install -y --no-install-recommends \
g++ \
@@ -16,7 +17,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libxslt-dev \
make \
patch \
pkg-config \
zlib1g-dev
RUN mkdir /install
@@ -26,14 +26,6 @@ COPY requirements.txt /requirements.txt
# Use cache mounts and multiple wheel sources for faster ARM builds
ENV PIP_CACHE_DIR=/tmp/pip-cache
# Help Rust find OpenSSL for cryptography package compilation on ARM
ENV PKG_CONFIG_PATH="/usr/lib/pkgconfig:/usr/lib/arm-linux-gnueabihf/pkgconfig:/usr/lib/aarch64-linux-gnu/pkgconfig"
ENV PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1
ENV OPENSSL_DIR="/usr"
ENV OPENSSL_LIB_DIR="/usr/lib/arm-linux-gnueabihf"
ENV OPENSSL_INCLUDE_DIR="/usr/include/openssl"
# Additional environment variables for cryptography Rust build
ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
RUN --mount=type=cache,target=/tmp/pip-cache \
pip install \
--extra-index-url https://www.piwheels.org/simple \
@@ -84,11 +76,6 @@ EXPOSE 5000
# The actual flask app module
COPY changedetectionio /app/changedetectionio
# Also for OpenAPI validation wrapper - needs the YML
RUN [ ! -d "/app/docs" ] && mkdir /app/docs
COPY docs/api-spec.yaml /app/docs/api-spec.yaml
# Starting wrapper
COPY changedetection.py /app/changedetection.py

View File

@@ -1,7 +1,7 @@
recursive-include changedetectionio/api *
recursive-include changedetectionio/blueprint *
recursive-include changedetectionio/conditions *
recursive-include changedetectionio/content_fetchers *
recursive-include changedetectionio/conditions *
recursive-include changedetectionio/model *
recursive-include changedetectionio/notification *
recursive-include changedetectionio/processors *
@@ -9,7 +9,6 @@ recursive-include changedetectionio/realtime *
recursive-include changedetectionio/static *
recursive-include changedetectionio/templates *
recursive-include changedetectionio/tests *
recursive-include changedetectionio/widgets *
prune changedetectionio/static/package-lock.json
prune changedetectionio/static/styles/node_modules
prune changedetectionio/static/styles/package-lock.json

View File

@@ -2,7 +2,7 @@
# Read more https://github.com/dgtlmoon/changedetection.io/wiki
__version__ = '0.50.13'
__version__ = '0.50.10'
from changedetectionio.strtobool import strtobool
from json.decoder import JSONDecodeError

View File

@@ -2,7 +2,6 @@ import copy
import yaml
import functools
from flask import request, abort
from loguru import logger
from openapi_core import OpenAPI
from openapi_core.contrib.flask import FlaskOpenAPIRequest
from . import api_schema
@@ -14,7 +13,6 @@ schema = api_schema.build_watch_json_schema(watch_base_config)
schema_create_watch = copy.deepcopy(schema)
schema_create_watch['required'] = ['url']
del schema_create_watch['properties']['last_viewed']
schema_update_watch = copy.deepcopy(schema)
schema_update_watch['additionalProperties'] = False
@@ -32,13 +30,17 @@ schema_create_notification_urls['required'] = ['notification_urls']
schema_delete_notification_urls = copy.deepcopy(schema_notification_urls)
schema_delete_notification_urls['required'] = ['notification_urls']
@functools.cache
# Load OpenAPI spec for validation
_openapi_spec = None
def get_openapi_spec():
import os
spec_path = os.path.join(os.path.dirname(__file__), '../../docs/api-spec.yaml')
with open(spec_path, 'r') as f:
spec_dict = yaml.safe_load(f)
_openapi_spec = OpenAPI.from_dict(spec_dict)
global _openapi_spec
if _openapi_spec is None:
import os
spec_path = os.path.join(os.path.dirname(__file__), '../../docs/api-spec.yaml')
with open(spec_path, 'r') as f:
spec_dict = yaml.safe_load(f)
_openapi_spec = OpenAPI.from_dict(spec_dict)
return _openapi_spec
def validate_openapi_request(operation_id):
@@ -47,25 +49,16 @@ def validate_openapi_request(operation_id):
@functools.wraps(f)
def wrapper(*args, **kwargs):
try:
# Skip OpenAPI validation for GET requests since they don't have request bodies
if request.method.upper() != 'GET':
spec = get_openapi_spec()
openapi_request = FlaskOpenAPIRequest(request)
result = spec.unmarshal_request(openapi_request)
if result.errors:
from werkzeug.exceptions import BadRequest
error_details = []
for error in result.errors:
error_details.append(str(error))
raise BadRequest(f"OpenAPI validation failed: {error_details}")
except BadRequest:
# Re-raise BadRequest exceptions (validation failures)
raise
spec = get_openapi_spec()
openapi_request = FlaskOpenAPIRequest(request)
result = spec.unmarshal_request(openapi_request, operation_id)
if result.errors:
abort(400, message=f"OpenAPI validation failed: {result.errors}")
return f(*args, **kwargs)
except Exception as e:
# If OpenAPI spec loading fails, log but don't break existing functionality
logger.critical(f"OpenAPI validation warning for {operation_id}: {e}")
abort(500)
return f(*args, **kwargs)
# If OpenAPI validation fails, log but don't break existing functionality
print(f"OpenAPI validation warning for {operation_id}: {e}")
return f(*args, **kwargs)
return wrapper
return decorator
@@ -75,4 +68,3 @@ from .Tags import Tags, Tag
from .Import import Import
from .SystemInfo import SystemInfo
from .Notifications import Notifications

View File

@@ -78,13 +78,6 @@ def build_watch_json_schema(d):
]:
schema['properties'][v]['anyOf'].append({'type': 'string', "maxLength": 5000})
for v in ['last_viewed']:
schema['properties'][v] = {
"type": "integer",
"description": "Unix timestamp in seconds of the last time the watch was viewed.",
"minimum": 0
}
# None or Boolean
schema['properties']['track_ldjson_price_data']['anyOf'].append({'type': 'boolean'})

View File

@@ -310,6 +310,15 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
continue
if process_changedetection_results:
# Extract title if needed
if datastore.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
if not watch['title'] or not len(watch['title']):
try:
update_obj['title'] = html_tools.extract_element(find='title', html_content=update_handler.fetcher.content)
logger.info(f"UUID: {uuid} Extract <title> updated title to '{update_obj['title']}")
except Exception as e:
logger.warning(f"UUID: {uuid} Extract <title> as watch title was enabled, but couldn't find a <title>.")
try:
datastore.update_watch(uuid=uuid, update_obj=update_obj)
@@ -348,14 +357,6 @@ async def async_update_worker(worker_id, q, notification_q, app, datastore):
# Always record attempt count
count = watch.get('check_count', 0) + 1
# Always record page title (used in notifications, and can change even when the content is the same)
try:
page_title = html_tools.extract_title(data=update_handler.fetcher.content)
logger.debug(f"UUID: {uuid} Page <title> is '{page_title}'")
datastore.update_watch(uuid=uuid, update_obj={'page_title': page_title})
except Exception as e:
logger.warning(f"UUID: {uuid} Exception when extracting <title> - {str(e)}")
# Record server header
try:
server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]

View File

@@ -108,13 +108,10 @@ def construct_blueprint(datastore: ChangeDetectionStore):
fe.link(link=diff_link)
# Same logic as watch-overview.html
if datastore.data['settings']['application']['ui'].get('use_page_title_in_list') or watch.get('use_page_title_in_list'):
watch_label = watch.label
else:
watch_label = watch.get('url')
# @todo watch should be a getter - watch.get('title') (internally if URL else..)
fe.title(title=watch_label)
watch_title = watch.get('title') if watch.get('title') else watch.get('url')
fe.title(title=watch_title)
try:
html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]),
@@ -130,7 +127,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
# @todo User could decide if <link> goes to the diff page, or to the watch link
rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n"
content = jinja_render(template_str=rss_template, watch_title=watch_label, html_diff=html_diff, watch_url=watch.link)
content = jinja_render(template_str=rss_template, watch_title=watch_title, html_diff=html_diff, watch_url=watch.link)
# Out of range chars could also break feedgen
if scan_invalid_chars_in_rss(content):

View File

@@ -1,7 +1,7 @@
{% extends 'base.html' %}
{% block content %}
{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, render_ternary_field %}
{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %}
{% from '_common_fields.html' import render_common_settings_form %}
<script>
const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="global-settings")}}";
@@ -75,10 +75,18 @@
<div class="pure-control-group">
{{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }}
</div>
<div class="pure-control-group">
{{ render_field(form.application.form.pager_size) }}
<span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
</div>
<div class="pure-control-group">
{{ render_field(form.application.form.rss_content_format) }}
<span class="pure-form-message-inline">Love RSS? Does your reader support HTML? Set it here</span>
</div>
<div class="pure-control-group">
{{ render_checkbox_field(form.application.form.extract_title_as_title) }}
<span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
</div>
<div class="pure-control-group">
{{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}
<span class="pure-form-message-inline">When a request returns no content, or the HTML does not contain any text, is this considered a change?</span>
@@ -252,13 +260,6 @@ nav
{{ render_checkbox_field(form.application.form.ui.form.favicons_enabled, class="") }}
<span class="pure-form-message-inline">Enable or Disable Favicons next to the watch list</span>
</div>
<div class="pure-control-group">
{{ render_checkbox_field(form.application.form.ui.use_page_title_in_list) }}
</div>
<div class="pure-control-group">
{{ render_field(form.application.form.pager_size) }}
<span class="pure-form-message-inline">Number of items per page in the watch overview list, 0 to disable.</span>
</div>
</div>
<div class="tab-pane-inner" id="proxies">
@@ -323,8 +324,8 @@ nav
<div id="actions">
<div class="pure-control-group">
{{ render_button(form.save_button) }}
<a href="{{url_for('watchlist.index')}}" class="pure-button button-cancel">Back</a>
<a href="{{url_for('ui.clear_all_history')}}" class="pure-button button-error">Clear Snapshot History</a>
<a href="{{url_for('watchlist.index')}}" class="pure-button button-small button-cancel">Back</a>
<a href="{{url_for('ui.clear_all_history')}}" class="pure-button button-small button-error">Clear Snapshot History</a>
</div>
</div>
</form>

View File

@@ -1,6 +1,6 @@
{% extends 'base.html' %}
{% block content %}
{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_ternary_field %}
{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}
{% from '_common_fields.html' import render_common_settings_form %}
<script>
const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', mode="group-settings")}}";
@@ -64,7 +64,7 @@
<div class="tab-pane-inner" id="notifications">
<fieldset>
<div class="pure-control-group inline-radio">
{{ render_ternary_field(form.notification_muted, BooleanField=True) }}
{{ render_checkbox_field(form.notification_muted) }}
</div>
{% if 1 %}
<div class="pure-control-group inline-radio">

View File

@@ -242,7 +242,6 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
'available_timezones': sorted(available_timezones()),
'browser_steps_config': browser_step_ui_config,
'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
'extra_classes': 'checking-now' if worker_handler.is_watch_running(uuid) else '',
'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
'extra_processor_config': form.extra_tab_content(),
'extra_title': f" - Edit - {watch.label}",

View File

@@ -44,16 +44,12 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
# Sort by last_changed and add the uuid which is usually the key..
sorted_watches = []
with_errors = request.args.get('with_errors') == "1"
unread_only = request.args.get('unread') == "1"
errored_count = 0
search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
for uuid, watch in datastore.data['watching'].items():
if with_errors and not watch.get('last_error'):
continue
if unread_only and (watch.viewed or watch.last_changed == 0) :
continue
if active_tag_uuid and not active_tag_uuid in watch['tags']:
continue
if watch.get('last_error'):

View File

@@ -118,8 +118,7 @@ document.addEventListener('DOMContentLoaded', function() {
{%- set checking_now = is_checking_now(watch) -%}
{%- set history_n = watch.history_n -%}
{%- set favicon = watch.get_favicon_filename() -%}
{%- set system_use_url_watchlist = datastore.data['settings']['application']['ui'].get('use_page_title_in_list') -%}
{# Class settings mirrored in changedetectionio/static/js/realtime.js for the frontend #}
{# Mirror in changedetectionio/static/js/realtime.js for the frontend #}
{%- set row_classes = [
loop.cycle('pure-table-odd', 'pure-table-even'),
'processor-' ~ watch['processor'],
@@ -134,8 +133,7 @@ document.addEventListener('DOMContentLoaded', function() {
'checking-now' if checking_now else '',
'notification_muted' if watch.notification_muted else '',
'single-history' if history_n == 1 else '',
'multiple-history' if history_n >= 2 else '',
'use-html-title' if system_use_url_watchlist else 'no-html-title',
'multiple-history' if history_n >= 2 else '',
] -%}
<tr id="{{ watch.uuid }}" data-watch-uuid="{{ watch.uuid }}" class="{{ row_classes | reject('equalto', '') | join(' ') }}">
<td class="inline checkbox-uuid" ><div><input name="uuids" type="checkbox" value="{{ watch.uuid}} " > <span class="counter-i">{{ loop.index+pagination.skip }}</span></div></td>
@@ -157,12 +155,7 @@ document.addEventListener('DOMContentLoaded', function() {
{% endif %}
<div>
<span class="watch-title">
{% if system_use_url_watchlist or watch.get('use_page_title_in_list') %}
{{watch.label}}
{% else %}
{{watch.url}}
{% endif %}
<a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}">&nbsp;</a>
{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}&nbsp;<a class="external" target="_blank" rel="noopener" href="{{ watch.link.replace('source:','') }}">&nbsp;</a>
</span>
<div class="error-text" style="display:none;">{{ watch.compile_error_texts(has_proxies=datastore.proxy_list) }}</div>
{%- if watch['processor'] == 'text_json_diff' -%}
@@ -252,9 +245,6 @@ document.addEventListener('DOMContentLoaded', function() {
<a href="{{url_for('ui.mark_all_viewed', tag=active_tag_uuid) }}" class="pure-button button-tag " id="mark-all-viewed">Mark all viewed in '{{active_tag.title}}'</a>
</li>
{%- endif -%}
<li id="post-list-unread" class="{%- if has_unviewed -%}has-unviewed{%- endif -%}" style="display: none;" >
<a href="{{url_for('watchlist.index', unread=1, tag=request.args.get('tag')) }}" class="pure-button button-tag">Unread</a>
</li>
<li>
<a href="{{ url_for('ui.form_watch_checknow', tag=active_tag_uuid, with_errors=request.args.get('with_errors',0)) }}" class="pure-button button-tag" id="recheck-all">Recheck
all {% if active_tag_uuid %} in '{{active_tag.title}}'{%endif%}</a>

View File

@@ -47,7 +47,6 @@ async () => {
'nicht lieferbar',
'nicht verfügbar',
'nicht vorrätig',
'nicht mehr lieferbar',
'nicht zur verfügung',
'nie znaleziono produktów',
'niet beschikbaar',

View File

@@ -28,8 +28,6 @@ from wtforms.validators import ValidationError
from validators.url import url as url_validator
from changedetectionio.widgets import TernaryNoneBooleanField
# default
# each select <option data-enabled="enabled-0-0"
@@ -550,6 +548,7 @@ class commonSettingsForm(Form):
self.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
self.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})
extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
fetch_backend = RadioField(u'Fetch Method', choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
@@ -617,18 +616,18 @@ class processor_text_json_diff_form(commonSettingsForm):
text_should_not_be_present = StringListField('Block change-detection while text matches', [validators.Optional(), ValidateListRegex()])
webdriver_js_execute_code = TextAreaField('Execute JavaScript before change detection', render_kw={"rows": "5"}, validators=[validators.Optional()])
save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
save_button = SubmitField('Save', render_kw={"class": "pure-button button-small pure-button-primary"})
proxy = RadioField('Proxy')
# filter_failure_notification_send @todo make ternary
filter_failure_notification_send = BooleanField(
'Send a notification when the filter can no longer be found on the page', default=False)
notification_muted = TernaryNoneBooleanField('Notifications', default=None, yes_text="Muted", no_text="On")
notification_muted = BooleanField('Notifications Muted / Off', default=False)
notification_screenshot = BooleanField('Attach screenshot to notification (where possible)', default=False)
conditions_match_logic = RadioField(u'Match', choices=[('ALL', 'Match all of the following'),('ANY', 'Match any of the following')], default='ALL')
conditions = FieldList(FormField(ConditionFormRow), min_entries=1) # Add rule logic here
use_page_title_in_list = TernaryNoneBooleanField('Use page <title> in list', default=None)
def extra_tab_content(self):
return None
@@ -756,7 +755,6 @@ class globalSettingsApplicationUIForm(Form):
open_diff_in_new_tab = BooleanField("Open 'History' page in a new tab", default=True, validators=[validators.Optional()])
socket_io_enabled = BooleanField('Realtime UI Updates Enabled', default=True, validators=[validators.Optional()])
favicons_enabled = BooleanField('Favicons Enabled', default=True, validators=[validators.Optional()])
use_page_title_in_list = BooleanField('Use page <title> in watch overview list') #BooleanField=True
# datastore.data['settings']['application']..
class globalSettingsApplicationForm(commonSettingsForm):
@@ -781,7 +779,7 @@ class globalSettingsApplicationForm(commonSettingsForm):
removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})
render_anchor_tag_content = BooleanField('Render anchor tag content', default=False)
shared_diff_access = BooleanField('Allow anonymous access to watch history page when password is enabled', default=False, validators=[validators.Optional()])
shared_diff_access = BooleanField('Allow access to view diff page when password is enabled', default=False, validators=[validators.Optional()])
rss_hide_muted_watches = BooleanField('Hide muted watches from RSS feed', default=True,
validators=[validators.Optional()])
filter_failure_notification_threshold_attempts = IntegerField('Number of times the filter can be missing before sending a notification',
@@ -803,7 +801,7 @@ class globalSettingsForm(Form):
requests = FormField(globalSettingsRequestForm)
application = FormField(globalSettingsApplicationForm)
save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
save_button = SubmitField('Save', render_kw={"class": "pure-button button-small pure-button-primary"})
class extractDataForm(Form):

View File

@@ -1,7 +1,6 @@
from loguru import logger
from lxml import etree
from typing import List
import html
import json
import re
@@ -10,11 +9,6 @@ TEXT_FILTER_LIST_LINE_SUFFIX = "<br>"
TRANSLATE_WHITESPACE_TABLE = str.maketrans('', '', '\r\n\t ')
PERL_STYLE_REGEX = r'^/(.*?)/([a-z]*)?$'
TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.I | re.S)
META_CS = re.compile(r'<meta[^>]+charset=["\']?\s*([a-z0-9_\-:+.]+)', re.I)
META_CT = re.compile(r'<meta[^>]+http-equiv=["\']?content-type["\']?[^>]*content=["\'][^>]*charset=([a-z0-9_\-:+.]+)', re.I)
# 'price' , 'lowPrice', 'highPrice' are usually under here
# All of those may or may not appear on different websites - I didnt find a way todo case-insensitive searching here
LD_JSON_PRODUCT_OFFER_SELECTORS = ["json:$..offers", "json:$..Offers"]
@@ -516,43 +510,3 @@ def get_triggered_text(content, trigger_text):
i += 1
return triggered_text
def extract_title(data: bytes | str, sniff_bytes: int = 2048, scan_chars: int = 8192) -> str | None:
try:
# Only decode/process the prefix we need for title extraction
match data:
case bytes() if data.startswith((b"\xff\xfe", b"\xfe\xff")):
prefix = data[:scan_chars * 2].decode("utf-16", errors="replace")
case bytes() if data.startswith((b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff")):
prefix = data[:scan_chars * 4].decode("utf-32", errors="replace")
case bytes():
try:
prefix = data[:scan_chars].decode("utf-8")
except UnicodeDecodeError:
try:
head = data[:sniff_bytes].decode("ascii", errors="ignore")
if m := (META_CS.search(head) or META_CT.search(head)):
enc = m.group(1).lower()
else:
enc = "cp1252"
prefix = data[:scan_chars * 2].decode(enc, errors="replace")
except Exception as e:
logger.error(f"Title extraction encoding detection failed: {e}")
return None
case str():
prefix = data[:scan_chars] if len(data) > scan_chars else data
case _:
logger.error(f"Title extraction received unsupported data type: {type(data)}")
return None
# Search only in the prefix
if m := TITLE_RE.search(prefix):
title = html.unescape(" ".join(m.group(1).split())).strip()
# Some safe limit
return title[:2000]
return None
except Exception as e:
logger.error(f"Title extraction failed: {e}")
return None

View File

@@ -39,12 +39,12 @@ class model(dict):
'api_access_token_enabled': True,
'base_url' : None,
'empty_pages_are_a_change': False,
'extract_title_as_title': False,
'fetch_backend': getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
'filter_failure_notification_threshold_attempts': _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT,
'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
'global_subtractive_selectors': [],
'ignore_whitespace': True,
'ignore_status_codes': False, #@todo implement, as ternary.
'notification_body': default_notification_body,
'notification_format': default_notification_format,
'notification_title': default_notification_title,
@@ -57,11 +57,10 @@ class model(dict):
'rss_hide_muted_watches': True,
'schema_version' : 0,
'shared_diff_access': False,
'webdriver_delay': None , # Extra delay in seconds before extracting text
'tags': {}, #@todo use Tag.model initialisers
'timezone': None, # Default IANA timezone name
'webdriver_delay': None , # Extra delay in seconds before extracting text
'ui': {
'use_page_title_in_list': True,
'open_diff_in_new_tab': True,
'socket_io_enabled': True,
'favicons_enabled': True

View File

@@ -169,8 +169,8 @@ class model(watch_base):
@property
def label(self):
# Used for sorting, display, etc
return self.get('title') or self.get('page_title') or self.get('url')
# Used for sorting
return self.get('title') if self.get('title') else self.get('url')
@property
def last_changed(self):

View File

@@ -24,6 +24,7 @@ class watch_base(dict):
'content-type': None,
'date_created': None,
'extract_text': [], # Extract text by regex after filters
'extract_title_as_title': False,
'fetch_backend': 'system', # plaintext, playwright etc
'fetch_time': 0.0,
'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
@@ -34,7 +35,6 @@ class watch_base(dict):
'has_ldjson_price_data': None,
'headers': {}, # Extra headers to send
'ignore_text': [], # List of text to ignore when calculating the comparison checksum
'ignore_status_codes': None,
'in_stock_only': True, # Only trigger change on going to instock from out-of-stock
'include_filters': [],
'last_checked': 0,
@@ -49,7 +49,6 @@ class watch_base(dict):
'notification_screenshot': False, # Include the latest screenshot if available and supported by the apprise URL
'notification_title': None,
'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise)
'page_title': None, # <title> from the page
'paused': False,
'previous_md5': False,
'previous_md5_before_filters': False, # Used for skipping changedetection entirely
@@ -123,13 +122,12 @@ class watch_base(dict):
}
},
},
'title': None, # An arbitrary field that overrides 'page_title'
'title': None,
'track_ldjson_price_data': None,
'trim_text_whitespace': False,
'remove_duplicate_lines': False,
'trigger_text': [], # List of text or regex to wait for until a change is detected
'url': '',
'use_page_title_in_list': None, # None = use system settings
'uuid': str(uuid.uuid4()),
'webdriver_delay': None,
'webdriver_js_execute_code': None, # Run before change-detection

View File

@@ -149,7 +149,7 @@ def create_notification_parameters(n_object, datastore):
uuid = n_object['uuid'] if 'uuid' in n_object else ''
if uuid:
watch_title = datastore.data['watching'][uuid].label
watch_title = datastore.data['watching'][uuid].get('title', '')
tag_list = []
tags = datastore.get_all_tags_for_watch(uuid)
if tags:

View File

@@ -251,7 +251,8 @@ class perform_site_check(difference_detection_processor):
update_obj["last_check_status"] = self.fetcher.get_last_status_code()
# 615 Extract text by regex
extract_text = list(dict.fromkeys(watch.get('extract_text', []) + self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='extract_text')))
extract_text = watch.get('extract_text', [])
extract_text += self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='extract_text')
if len(extract_text) > 0:
regex_matched_output = []
for s_re in extract_text:
@@ -310,7 +311,8 @@ class perform_site_check(difference_detection_processor):
############ Blocking rules, after checksum #################
blocked = False
trigger_text = list(dict.fromkeys(watch.get('trigger_text', []) + self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='trigger_text')))
trigger_text = watch.get('trigger_text', [])
trigger_text += self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='trigger_text')
if len(trigger_text):
# Assume blocked
blocked = True
@@ -324,7 +326,8 @@ class perform_site_check(difference_detection_processor):
if result:
blocked = False
text_should_not_be_present = list(dict.fromkeys(watch.get('text_should_not_be_present', []) + self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='text_should_not_be_present')))
text_should_not_be_present = watch.get('text_should_not_be_present', [])
text_should_not_be_present += self.datastore.get_tag_overrides_for_watch(uuid=watch.get('uuid'), attr='text_should_not_be_present')
if len(text_should_not_be_present):
# If anything matched, then we should block a change from happening
result = html_tools.strip_ignore_text(content=str(stripped_text_from_html),

View File

@@ -153,7 +153,6 @@ $(document).ready(function () {
// Tabs at bottom of list
$('#post-list-mark-views').toggleClass("has-unviewed", general_stats.has_unviewed);
$('#post-list-unread').toggleClass("has-unviewed", general_stats.has_unviewed);
$('#post-list-with-errors').toggleClass("has-error", general_stats.count_errors !== 0)
$('#post-list-with-errors a').text(`With errors (${ general_stats.count_errors })`);

View File

@@ -51,7 +51,6 @@ $(document).ready(function () {
$('#notification_body').val('');
$('#notification_format').val('System default');
$('#notification_urls').val('');
$('#notification_muted_none').prop('checked', true); // in the case of a ternary field
e.preventDefault();
});
$("#notification-token-toggle").click(function (e) {

View File

@@ -24,9 +24,6 @@ body.checking-now {
#post-list-mark-views.has-unviewed {
display: inline-block !important;
}
#post-list-unread.has-unviewed {
display: inline-block !important;
}
}

View File

@@ -1,115 +0,0 @@
// Ternary radio button group component
.ternary-radio-group {
display: flex;
gap: 0;
border: 1px solid var(--color-grey-750);
border-radius: 4px;
overflow: hidden;
width: fit-content;
background: var(--color-background);
.ternary-radio-option {
position: relative;
cursor: pointer;
margin: 0;
display: flex;
align-items: center;
input[type="radio"] {
position: absolute;
opacity: 0;
width: 0;
height: 0;
}
.ternary-radio-label {
padding: 8px 16px;
background: var(--color-grey-900);
border: none;
border-right: 1px solid var(--color-grey-750);
font-size: 13px;
font-weight: 500;
color: var(--color-text);
transition: all 0.2s ease;
cursor: pointer;
display: block;
min-width: 60px;
text-align: center;
}
&:last-child .ternary-radio-label {
border-right: none;
}
input:checked + .ternary-radio-label {
background: var(--color-link);
color: var(--color-text-button);
font-weight: 600;
&.ternary-default {
background: var(--color-grey-600);
color: var(--color-text-button);
}
&:hover {
background: #1a7bc4;
&.ternary-default {
background: var(--color-grey-500);
}
}
}
&:hover .ternary-radio-label {
background: var(--color-grey-800);
}
}
@media (max-width: 480px) {
width: 100%;
.ternary-radio-label {
flex: 1;
min-width: auto;
}
}
}
// Standard radio button styling
input[type="radio"].pure-radio:checked + label,
input[type="radio"].pure-radio:checked {
background: var(--color-link);
color: var(--color-text-button);
}
html[data-darkmode="true"] {
.ternary-radio-group {
.ternary-radio-option {
.ternary-radio-label {
background: var(--color-grey-350);
}
&:hover .ternary-radio-label {
background: var(--color-grey-400);
}
input:checked + .ternary-radio-label {
background: var(--color-link);
color: var(--color-text-button);
&.ternary-default {
background: var(--color-grey-600);
}
&:hover {
background: #1a7bc4;
&.ternary-default {
background: var(--color-grey-500);
}
}
}
}
}
}

View File

@@ -20,7 +20,7 @@
@use "parts/lister_extra";
@use "parts/socket";
@use "parts/visualselector";
@use "parts/widgets";
body {
color: var(--color-text);
@@ -1130,12 +1130,11 @@ ul {
}
#realtime-conn-error {
position: fixed;
position: absolute;
bottom: 0;
left: 0;
left: 30px;
background: var(--color-warning);
padding: 10px;
font-size: 0.8rem;
color: #fff;
opacity: 0.8;
}

File diff suppressed because one or more lines are too long

View File

@@ -262,6 +262,11 @@ class ChangeDetectionStore:
extras = deepcopy(self.data['watching'][uuid])
new_uuid = self.add_watch(url=url, extras=extras)
watch = self.data['watching'][new_uuid]
if self.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
# Because it will be recalculated on the next fetch
self.data['watching'][new_uuid]['title'] = None
return new_uuid
def url_exists(self, url):
@@ -303,6 +308,7 @@ class ChangeDetectionStore:
'browser_steps',
'css_filter',
'extract_text',
'extract_title_as_title',
'headers',
'ignore_text',
'include_filters',
@@ -317,7 +323,6 @@ class ChangeDetectionStore:
'title',
'trigger_text',
'url',
'use_page_title_in_list',
'webdriver_js_execute_code',
]:
if res.get(k):
@@ -968,16 +973,6 @@ class ChangeDetectionStore:
f_d.write(zlib.compress(f_j.read()))
os.unlink(json_path)
def update_20(self):
for uuid, watch in self.data['watching'].items():
if self.data['watching'][uuid].get('extract_title_as_title'):
self.data['watching'][uuid]['use_page_title_in_list'] = self.data['watching'][uuid].get('extract_title_as_title')
del self.data['watching'][uuid]['extract_title_as_title']
if self.data['settings']['application'].get('extract_title_as_title'):
self.data['settings']['application']['ui']['use_page_title_in_list'] = self.data['settings']['application'].get('extract_title_as_title')
def add_notification_url(self, notification_url):
logger.debug(f">>> Adding new notification_url - '{notification_url}'")

View File

@@ -70,7 +70,7 @@
</tr>
<tr>
<td><code>{{ '{{watch_title}}' }}</code></td>
<td>The page title of the watch, uses &lt;title&gt; if not set, falls back to URL</td>
<td>The title of the watch.</td>
</tr>
<tr>
<td><code>{{ '{{watch_tag}}' }}</code></td>

View File

@@ -24,23 +24,6 @@
</div>
{% endmacro %}
{% macro render_ternary_field(field, BooleanField=false) %}
{% if BooleanField %}
{% set _ = field.__setattr__('boolean_mode', true) %}
{% endif %}
<div class="ternary-field {% if field.errors %} error {% endif %}">
<div class="ternary-field-label">{{ field.label }}</div>
<div class="ternary-field-widget">{{ field(**kwargs)|safe }}</div>
{% if field.errors %}
<ul class=errors>
{% for error in field.errors %}
<li>{{ error }}</li>
{% endfor %}
</ul>
{% endif %}
</div>
{% endmacro %}
{% macro render_simple_field(field) %}
<span class="label {% if field.errors %}error{% endif %}">{{ field.label }}</span>

View File

@@ -5,7 +5,6 @@
<meta charset="utf-8" >
<meta name="viewport" content="width=device-width, initial-scale=1.0" >
<meta name="description" content="Self hosted website change detection." >
<meta name="robots" content="noindex">
<title>Change Detection{{extra_title}}</title>
{% if app_rss_token %}
<link rel="alternate" type="application/rss+xml" title="Changedetection.io » Feed{% if active_tag_uuid %}- {{active_tag.title}}{% endif %}" href="{{ url_for('rss.feed', tag=active_tag_uuid , token=app_rss_token)}}" >
@@ -41,7 +40,7 @@
{% endif %}
</head>
<body class="{{extra_classes}}">
<body class="">
<div class="header">
<div class="pure-menu-fixed" style="width: 100%;">
<div class="home-menu pure-menu pure-menu-horizontal" id="nav-menu">
@@ -237,7 +236,7 @@
<script src="{{url_for('static_content', group='js', filename='toggle-theme.js')}}" defer></script>
<div id="checking-now-fixed-tab" style="display: none;"><span class="spinner"></span><span>&nbsp;Checking now</span></div>
<div id="realtime-conn-error" style="display:none">Real-time updates offline</div>
<div id="realtime-conn-error" style="display:none">Offline</div>
</body>
</html>

View File

@@ -1,6 +1,6 @@
{% extends 'base.html' %}
{% block content %}
{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_playwright_type_watches_warning, render_conditions_fieldlist_of_formfields_as_table, render_ternary_field %}
{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_playwright_type_watches_warning, render_conditions_fieldlist_of_formfields_as_table %}
{% from '_common_fields.html' import render_common_settings_form %}
<script src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
<script src="{{url_for('static_content', group='js', filename='vis.js')}}" defer></script>
@@ -72,16 +72,15 @@
<div class="pure-form-message">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></div>
<div class="pure-form-message">Variables are supported in the URL (<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a>).</div>
</div>
<div class="pure-control-group">
{{ render_field(form.tags) }}
<span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
</div>
<div class="pure-control-group inline-radio">
{{ render_field(form.processor) }}
</div>
<div class="pure-control-group">
{{ render_field(form.title, class="m-d", placeholder=watch.label) }}
<span class="pure-form-message-inline">Automatically uses the page title if found, you can also use your own title/description here</span>
{{ render_field(form.title, class="m-d") }}
</div>
<div class="pure-control-group">
{{ render_field(form.tags) }}
<span class="pure-form-message-inline">Organisational tag/group name used in the main listing page</span>
</div>
<div class="pure-control-group time-between-check border-fieldset">
@@ -102,16 +101,15 @@
</div>
<br>
</div>
<div class="pure-control-group">
{{ render_checkbox_field(form.extract_title_as_title) }}
</div>
<div class="pure-control-group">
{{ render_checkbox_field(form.filter_failure_notification_send) }}
<span class="pure-form-message-inline">
Sends a notification when the filter can no longer be seen on the page, good for knowing when the page changed and your filter will not work anymore.
</span>
</div>
<div class="pure-control-group">
{{ render_ternary_field(form.use_page_title_in_list) }}
</div>
</fieldset>
</div>
@@ -264,7 +262,7 @@ Math: {{ 1 + 1 }}") }}
<div class="tab-pane-inner" id="notifications">
<fieldset>
<div class="pure-control-group inline-radio">
{{ render_ternary_field(form.notification_muted, BooleanField=true) }}
{{ render_checkbox_field(form.notification_muted) }}
</div>
{% if watch_needs_selenium_or_playwright %}
<div class="pure-control-group inline-radio">
@@ -471,11 +469,11 @@ Math: {{ 1 + 1 }}") }}
<div class="pure-control-group">
{{ render_button(form.save_button) }}
<a href="{{url_for('ui.form_delete', uuid=uuid)}}"
class="pure-button button-error ">Delete</a>
class="pure-button button-small button-error ">Delete</a>
{% if watch.history_n %}<a href="{{url_for('ui.clear_watch_history', uuid=uuid)}}"
class="pure-button button-error">Clear History</a>{% endif %}
class="pure-button button-small button-error ">Clear History</a>{% endif %}
<a href="{{url_for('ui.form_clone', uuid=uuid)}}"
class="pure-button">Clone &amp; Edit</a>
class="pure-button button-small ">Clone &amp; Edit</a>
</div>
</div>
</form>

View File

@@ -311,7 +311,7 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
"value": "." # contains anything
}
],
"conditions_match_logic": "ALL",
"conditions_match_logic": "ALL"
}
),
headers={'content-type': 'application/json', 'x-api-key': api_key},
@@ -328,7 +328,6 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
)
watch_uuid = list(res.json.keys())[0]
assert not res.json[watch_uuid].get('viewed'), 'A newly created watch can only be unviewed'
# Check in the edit page just to be sure
res = client.get(
@@ -342,12 +341,7 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
res = client.put(
url_for("watch", uuid=watch_uuid),
headers={'x-api-key': api_key, 'content-type': 'application/json'},
data=json.dumps({
"title": "new title",
'time_between_check': {'minutes': 552},
'headers': {'cookie': 'all eaten'},
'last_viewed': int(time.time())
}),
data=json.dumps({"title": "new title", 'time_between_check': {'minutes': 552}, 'headers': {'cookie': 'all eaten'}}),
)
assert res.status_code == 200, "HTTP PUT update was sent OK"
@@ -357,7 +351,6 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
headers={'x-api-key': api_key}
)
assert res.json.get('title') == 'new title'
assert res.json.get('viewed'), 'With the timestamp greater than "changed" a watch can be updated to viewed'
# Check in the edit page just to be sure
res = client.get(
@@ -390,13 +383,13 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage):
def test_api_import(client, live_server, measure_memory_usage):
api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
res = client.post(
url_for("import") + "?tag=import-test",
data='https://website1.com\r\nhttps://website2.com',
headers={'x-api-key': api_key, 'content-type': 'text/plain'},
headers={'x-api-key': api_key},
follow_redirects=True
)

View File

@@ -1,199 +0,0 @@
#!/usr/bin/env python3
"""
OpenAPI validation tests for ChangeDetection.io API
This test file specifically verifies that OpenAPI validation is working correctly
by testing various scenarios that should trigger validation errors.
"""
import time
import json
from flask import url_for
from .util import live_server_setup, wait_for_all_checks
def test_openapi_validation_invalid_content_type_on_create_watch(client, live_server, measure_memory_usage):
"""Test that creating a watch with invalid content-type triggers OpenAPI validation error."""
api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
# Try to create a watch with JSON data but without proper content-type header
res = client.post(
url_for("createwatch"),
data=json.dumps({"url": "https://example.com", "title": "Test Watch"}),
headers={'x-api-key': api_key}, # Missing 'content-type': 'application/json'
follow_redirects=True
)
# Should get 400 error due to OpenAPI validation failure
assert res.status_code == 400, f"Expected 400 but got {res.status_code}"
assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
def test_openapi_validation_missing_required_field_create_watch(client, live_server, measure_memory_usage):
"""Test that creating a watch without required URL field triggers OpenAPI validation error."""
api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
# Try to create a watch without the required 'url' field
res = client.post(
url_for("createwatch"),
data=json.dumps({"title": "Test Watch Without URL"}), # Missing required 'url' field
headers={'x-api-key': api_key, 'content-type': 'application/json'},
follow_redirects=True
)
# Should get 400 error due to missing required field
assert res.status_code == 400, f"Expected 400 but got {res.status_code}"
assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
def test_openapi_validation_invalid_field_in_request_body(client, live_server, measure_memory_usage):
"""Test that including invalid fields triggers OpenAPI validation error."""
api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
# First create a valid watch
res = client.post(
url_for("createwatch"),
data=json.dumps({"url": "https://example.com", "title": "Test Watch"}),
headers={'x-api-key': api_key, 'content-type': 'application/json'},
follow_redirects=True
)
assert res.status_code == 201, "Watch creation should succeed"
# Get the watch list to find the UUID
res = client.get(
url_for("createwatch"),
headers={'x-api-key': api_key}
)
assert res.status_code == 200
watch_uuid = list(res.json.keys())[0]
# Now try to update the watch with an invalid field
res = client.put(
url_for("watch", uuid=watch_uuid),
headers={'x-api-key': api_key, 'content-type': 'application/json'},
data=json.dumps({
"title": "Updated title",
"invalid_field_that_doesnt_exist": "this should cause validation error"
}),
)
# Should get 400 error due to invalid field (this will be caught by internal validation)
# Note: This tests the flow where OpenAPI validation passes but internal validation catches it
assert res.status_code == 400, f"Expected 400 but got {res.status_code}"
assert b"Additional properties are not allowed" in res.data, "Should contain validation error about additional properties"
def test_openapi_validation_import_wrong_content_type(client, live_server, measure_memory_usage):
"""Test that import endpoint with wrong content-type triggers OpenAPI validation error."""
api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
# Try to import URLs with JSON content-type instead of text/plain
res = client.post(
url_for("import") + "?tag=test-import",
data='https://website1.com\nhttps://website2.com',
headers={'x-api-key': api_key, 'content-type': 'application/json'}, # Wrong content-type
follow_redirects=True
)
# Should get 400 error due to content-type mismatch
assert res.status_code == 400, f"Expected 400 but got {res.status_code}"
assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
def test_openapi_validation_import_correct_content_type_succeeds(client, live_server, measure_memory_usage):
"""Test that import endpoint with correct content-type succeeds (positive test)."""
api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
# Import URLs with correct text/plain content-type
res = client.post(
url_for("import") + "?tag=test-import",
data='https://website1.com\nhttps://website2.com',
headers={'x-api-key': api_key, 'content-type': 'text/plain'}, # Correct content-type
follow_redirects=True
)
# Should succeed
assert res.status_code == 200, f"Expected 200 but got {res.status_code}"
assert len(res.json) == 2, "Should import 2 URLs"
def test_openapi_validation_get_requests_bypass_validation(client, live_server, measure_memory_usage):
"""Test that GET requests bypass OpenAPI validation entirely."""
api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
# Disable API token requirement first
res = client.post(
url_for("settings.settings_page"),
data={
"requests-time_between_check-minutes": 180,
"application-fetch_backend": "html_requests",
"application-api_access_token_enabled": ""
},
follow_redirects=True
)
assert b"Settings updated." in res.data
# Make GET request to list watches - should succeed even without API key or content-type
res = client.get(url_for("createwatch")) # No headers needed for GET
assert res.status_code == 200, f"GET requests should succeed without OpenAPI validation, got {res.status_code}"
# Should return JSON with watch list (empty in this case)
assert isinstance(res.json, dict), "Should return JSON dictionary for watch list"
def test_openapi_validation_create_tag_missing_required_title(client, live_server, measure_memory_usage):
"""Test that creating a tag without required title triggers OpenAPI validation error."""
api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
# Try to create a tag without the required 'title' field
res = client.post(
url_for("tag"),
data=json.dumps({"notification_urls": ["mailto:test@example.com"]}), # Missing required 'title' field
headers={'x-api-key': api_key, 'content-type': 'application/json'},
follow_redirects=True
)
# Should get 400 error due to missing required field
assert res.status_code == 400, f"Expected 400 but got {res.status_code}"
assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message"
def test_openapi_validation_watch_update_allows_partial_updates(client, live_server, measure_memory_usage):
"""Test that watch updates allow partial updates without requiring all fields (positive test)."""
api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
# First create a valid watch
res = client.post(
url_for("createwatch"),
data=json.dumps({"url": "https://example.com", "title": "Test Watch"}),
headers={'x-api-key': api_key, 'content-type': 'application/json'},
follow_redirects=True
)
assert res.status_code == 201, "Watch creation should succeed"
# Get the watch list to find the UUID
res = client.get(
url_for("createwatch"),
headers={'x-api-key': api_key}
)
assert res.status_code == 200
watch_uuid = list(res.json.keys())[0]
# Update only the title (partial update) - should succeed
res = client.put(
url_for("watch", uuid=watch_uuid),
headers={'x-api-key': api_key, 'content-type': 'application/json'},
data=json.dumps({"title": "Updated Title Only"}), # Only updating title, not URL
)
# Should succeed because UpdateWatch schema allows partial updates
assert res.status_code == 200, f"Partial updates should succeed, got {res.status_code}"
# Verify the update worked
res = client.get(
url_for("watch", uuid=watch_uuid),
headers={'x-api-key': api_key}
)
assert res.status_code == 200
assert res.json.get('title') == 'Updated Title Only', "Title should be updated"
assert res.json.get('url') == 'https://example.com', "URL should remain unchanged"

View File

@@ -89,7 +89,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
assert b'CDATA' in res.data
assert expected_url.encode('utf-8') in res.data
#
# Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
res = client.get(url_for("ui.ui_views.diff_history_page", uuid=uuid))
assert b'selected=""' in res.data, "Confirm diff history page loaded"
@@ -104,34 +104,26 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
wait_for_all_checks(client)
# Do this a few times.. ensures we don't accidently set the status
# Do this a few times.. ensures we dont accidently set the status
for n in range(2):
res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
wait_for_all_checks(client)
# It should report nothing found (no new 'unviewed' class)
res = client.get(url_for("watchlist.index"))
assert b'unviewed' not in res.data
assert b'class="has-unviewed' not in res.data
assert b'head title' in res.data # Should be ON by default
assert b'head title' not in res.data # Should not be present because this is off by default
assert b'test-endpoint' in res.data
# Recheck it but only with a title change, content wasnt changed
set_original_response(extra_title=" and more")
set_original_response()
client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
wait_for_all_checks(client)
res = client.get(url_for("watchlist.index"))
assert b'head title and more' in res.data
# disable <title> pickup
# Enable auto pickup of <title> in settings
res = client.post(
url_for("settings.settings_page"),
data={"application-ui-use_page_title_in_list": "", "requests-time_between_check-minutes": 180,
data={"application-extract_title_as_title": "1", "requests-time_between_check-minutes": 180,
'application-fetch_backend': "html_requests"},
follow_redirects=True
)
@@ -142,14 +134,16 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
res = client.get(url_for("watchlist.index"))
assert b'unviewed' in res.data
assert b'class="has-unviewed' in res.data
assert b'head title' not in res.data # should now be off
# It should have picked up the <title>
assert b'head title' in res.data
# Be sure the last_viewed is going to be greater than the last snapshot
time.sleep(1)
# hit the mark all viewed link
res = client.get(url_for("ui.mark_all_viewed"), follow_redirects=True)
time.sleep(0.2)
assert b'class="has-unviewed' not in res.data
assert b'unviewed' not in res.data

View File

@@ -6,9 +6,9 @@ from flask import url_for
import logging
import time
def set_original_response(extra_title=''):
test_return_data = f"""<html>
<head><title>head title{extra_title}</title></head>
def set_original_response():
test_return_data = """<html>
<head><title>head title</title></head>
<body>
Some initial text<br>
<p>Which is across multiple lines</p>

View File

@@ -1,3 +0,0 @@
from .ternary_boolean import TernaryNoneBooleanWidget, TernaryNoneBooleanField
__all__ = ['TernaryNoneBooleanWidget', 'TernaryNoneBooleanField']

View File

@@ -1,104 +0,0 @@
from wtforms import Field
from wtforms import widgets
from markupsafe import Markup
class TernaryNoneBooleanWidget:
"""
A widget that renders a horizontal radio button group with either two options (Yes/No)
or three options (Yes/No/Default), depending on the field's boolean_mode setting.
"""
def __call__(self, field, **kwargs):
html = ['<div class="ternary-radio-group pure-form">']
field_id = kwargs.pop('id', field.id)
boolean_mode = getattr(field, 'boolean_mode', False)
# Get custom text or use defaults
yes_text = getattr(field, 'yes_text', 'Yes')
no_text = getattr(field, 'no_text', 'No')
none_text = getattr(field, 'none_text', 'Main settings')
# True option
checked_true = ' checked' if field.data is True else ''
html.append(f'''
<label class="ternary-radio-option">
<input type="radio" name="{field.name}" value="true" id="{field_id}_true"{checked_true} class="pure-radio">
<span class="ternary-radio-label pure-button-primary">{yes_text}</span>
</label>
''')
# False option
checked_false = ' checked' if field.data is False else ''
html.append(f'''
<label class="ternary-radio-option">
<input type="radio" name="{field.name}" value="false" id="{field_id}_false"{checked_false} class="pure-radio">
<span class="ternary-radio-label">{no_text}</span>
</label>
''')
# None option (only show if not in boolean mode)
if not boolean_mode:
checked_none = ' checked' if field.data is None else ''
html.append(f'''
<label class="ternary-radio-option">
<input type="radio" name="{field.name}" value="none" id="{field_id}_none"{checked_none} class="pure-radio">
<span class="ternary-radio-label ternary-default">{none_text}</span>
</label>
''')
html.append('</div>')
return Markup(''.join(html))
class TernaryNoneBooleanField(Field):
"""
A field that can handle True, False, or None values, represented as a horizontal radio group.
When boolean_mode=True, it acts like a BooleanField (only Yes/No options).
When boolean_mode=False (default), it shows Yes/No/Default options.
Custom text can be provided for each option:
- yes_text: Text for True option (default: "Yes")
- no_text: Text for False option (default: "No")
- none_text: Text for None option (default: "Default")
"""
widget = TernaryNoneBooleanWidget()
def __init__(self, label=None, validators=None, false_values=None, boolean_mode=False,
yes_text="Yes", no_text="No", none_text="Main settings", **kwargs):
super(TernaryNoneBooleanField, self).__init__(label, validators, **kwargs)
self.boolean_mode = boolean_mode
self.yes_text = yes_text
self.no_text = no_text
self.none_text = none_text
if false_values is None:
self.false_values = {'false', ''}
else:
self.false_values = false_values
def process_formdata(self, valuelist):
if not valuelist or not valuelist[0]:
# In boolean mode, default to False instead of None
self.data = False if self.boolean_mode else None
elif valuelist[0].lower() == 'true':
self.data = True
elif valuelist[0].lower() == 'false':
self.data = False
elif valuelist[0].lower() == 'none':
# In boolean mode, treat 'none' as False
self.data = False if self.boolean_mode else None
else:
self.data = False if self.boolean_mode else None
def _value(self):
if self.data is True:
return 'true'
elif self.data is False:
return 'false'
else:
# In boolean mode, None should be treated as False
if self.boolean_mode:
return 'false'
else:
return 'none'

View File

@@ -1,135 +0,0 @@
#!/usr/bin/env python3
import sys
import os
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../..'))
from changedetectionio.widgets import TernaryNoneBooleanField
from wtforms import Form
class TestForm(Form):
# Default text
default_field = TernaryNoneBooleanField('Default Field', default=None)
# Custom text with HTML icons
notification_field = TernaryNoneBooleanField(
'Notifications',
default=False,
yes_text='🔕 Muted',
no_text='🔔 Unmuted',
none_text='⚙️ System default'
)
# HTML with styling
styled_field = TernaryNoneBooleanField(
'Status',
default=None,
yes_text='<strong style="color: green;">✅ Active</strong>',
no_text='<strong style="color: red;">❌ Inactive</strong>',
none_text='<em style="color: gray;">🔧 Auto</em>'
)
# Boolean mode with custom text
boolean_field = TernaryNoneBooleanField(
'Boolean Field',
default=True,
boolean_mode=True,
yes_text="Enabled",
no_text="Disabled"
)
# FontAwesome example
fontawesome_field = TernaryNoneBooleanField(
'Notifications with FontAwesome',
default=None,
yes_text='<i class="fa fa-bell-slash"></i> Muted',
no_text='<i class="fa fa-bell"></i> Unmuted',
none_text='<i class="fa fa-cogs"></i> System default'
)
def test_custom_text():
"""Test custom text functionality"""
form = TestForm()
print("=== Testing TernaryNoneBooleanField Custom Text ===")
# Test default field
print("\n--- Default Field ---")
default_field = form.default_field
default_html = default_field.widget(default_field)
print(f"Contains 'Yes': {'Yes' in default_html}")
print(f"Contains 'No': {'No' in default_html}")
print(f"Contains 'Default': {'Default' in default_html}")
assert 'Yes' in default_html and 'No' in default_html and 'Default' in default_html
# Test custom text field
print("\n--- Custom Text Field with Emojis ---")
notification_field = form.notification_field
notification_html = notification_field.widget(notification_field)
print(f"Contains '🔕 Muted': {'🔕 Muted' in notification_html}")
print(f"Contains '🔔 Unmuted': {'🔔 Unmuted' in notification_html}")
print(f"Contains '⚙️ System default': {'⚙️ System default' in notification_html}")
print(f"Does NOT contain 'Yes': {'Yes' not in notification_html}")
print(f"Does NOT contain 'No': {'No' not in notification_html}")
assert '🔕 Muted' in notification_html and '🔔 Unmuted' in notification_html
assert 'Yes' not in notification_html and 'No' not in notification_html
# Test HTML styling
print("\n--- HTML Styled Field ---")
styled_field = form.styled_field
styled_html = styled_field.widget(styled_field)
print(f"Contains HTML tags: {'<strong' in styled_html}")
print(f"Contains color styling: {'color: green' in styled_html}")
print(f"Contains emojis: {'' in styled_html and '' in styled_html}")
assert '<strong' in styled_html and 'color: green' in styled_html
# Test boolean mode with custom text
print("\n--- Boolean Field with Custom Text ---")
boolean_field = form.boolean_field
boolean_html = boolean_field.widget(boolean_field)
print(f"Contains 'Enabled': {'Enabled' in boolean_html}")
print(f"Contains 'Disabled': {'Disabled' in boolean_html}")
print(f"Does NOT contain 'System default': {'System default' not in boolean_html}")
print(f"Does NOT contain 'Default': {'Default' not in boolean_html}")
assert 'Enabled' in boolean_html and 'Disabled' in boolean_html
assert 'System default' not in boolean_html and 'Default' not in boolean_html
# Test FontAwesome field
print("\n--- FontAwesome Icons Field ---")
fontawesome_field = form.fontawesome_field
fontawesome_html = fontawesome_field.widget(fontawesome_field)
print(f"Contains FontAwesome classes: {'fa fa-bell' in fontawesome_html}")
print(f"Contains multiple FA icons: {'fa fa-bell-slash' in fontawesome_html and 'fa fa-cogs' in fontawesome_html}")
assert 'fa fa-bell' in fontawesome_html
print("\n✅ All custom text tests passed!")
print("\n--- Example Usage ---")
print("TernaryNoneBooleanField('Status', yes_text='🟢 Online', no_text='🔴 Offline', none_text='🟡 Auto')")
print("TernaryNoneBooleanField('Notifications', yes_text='<i class=\"fa fa-bell-slash\"></i> Muted', ...)")
def test_data_processing():
"""Test that custom text doesn't affect data processing"""
print("\n=== Testing Data Processing ===")
form = TestForm()
field = form.notification_field
# Test form data processing
field.process_formdata(['true'])
assert field.data is True, "Custom text should not affect data processing"
print("✅ True processing works with custom text")
field.process_formdata(['false'])
assert field.data is False, "Custom text should not affect data processing"
print("✅ False processing works with custom text")
field.process_formdata(['none'])
assert field.data is None, "Custom text should not affect data processing"
print("✅ None processing works with custom text")
print("✅ All data processing tests passed!")
if __name__ == '__main__':
test_custom_text()
test_data_processing()

View File

@@ -84,7 +84,6 @@ services:
# Comment out ports: when using behind a reverse proxy , enable networks: etc.
# Mac users! Use "127.0.0.1:5050:5000" (port 5050) so theres no conflict with Airplay etc. (https://github.com/dgtlmoon/changedetection.io/issues/3401)
ports:
- 127.0.0.1:5000:5000
restart: unless-stopped

View File

@@ -1,4 +1,4 @@
openapi: 3.0.4
openapi: 3.0.3
info:
title: ChangeDetection.io API
description: |
@@ -6,7 +6,7 @@ info:
REST API for managing Page watches, Group tags, and Notifications.
changedetection.io can be driven by its built in simple API, in the examples below you will also find `curl` command line and `python` examples to help you get started faster.
changedetection.io can be driven by its built in simple API, in the examples below you will also find `curl` command line examples to help you.
## Where to find my API key?
@@ -119,9 +119,14 @@ components:
Enter your API key in the "Authorize" button above to automatically populate all code examples.
schemas:
WatchBase:
Watch:
type: object
properties:
uuid:
type: string
format: uuid
description: Unique identifier for the web page change monitor (watch)
readOnly: true
url:
type: string
format: uri
@@ -224,51 +229,28 @@ components:
maxLength: 5000
required: [operation, selector, optional_value]
description: Browser automation steps
Watch:
allOf:
- $ref: '#/components/schemas/WatchBase'
- type: object
properties:
uuid:
type: string
format: uuid
description: Unique identifier for the web page change monitor (watch)
readOnly: true
last_checked:
type: integer
description: Unix timestamp of last check
readOnly: true
last_changed:
type: integer
description: Unix timestamp of last change
readOnly: true
last_error:
type: string
description: Last error message
readOnly: true
last_viewed:
type: integer
description: Unix timestamp in seconds of the last time the watch was viewed. Setting it to a value higher than `last_changed` in the "Update watch" endpoint marks the watch as viewed.
minimum: 0
last_checked:
type: integer
description: Unix timestamp of last check
readOnly: true
last_changed:
type: integer
description: Unix timestamp of last change
readOnly: true
last_error:
type: string
description: Last error message
readOnly: true
required:
- url
CreateWatch:
allOf:
- $ref: '#/components/schemas/WatchBase'
- type: 'object'
- $ref: '#/components/schemas/Watch'
- type: object
required:
- url
UpdateWatch:
allOf:
- $ref: '#/components/schemas/WatchBase'
- type: object
properties:
last_viewed:
type: integer
description: Unix timestamp in seconds of the last time the watch was viewed. Setting it to a value higher than `last_changed` in the "Update watch" endpoint marks the watch as viewed.
minimum: 0
Tag:
type: object
properties:
@@ -289,13 +271,8 @@ components:
notification_muted:
type: boolean
description: Whether notifications are muted for this tag
CreateTag:
allOf:
- $ref: '#/components/schemas/Tag'
- type: object
required:
- title
required:
- title
NotificationUrls:
type: object
@@ -585,7 +562,7 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateWatch'
$ref: '#/components/schemas/Watch'
responses:
'200':
description: Web page change monitor (watch) updated successfully
@@ -599,7 +576,7 @@ paths:
delete:
operationId: deleteWatch
tags: [Watch Management]
tags: [Watch Management]
summary: Delete watch
description: Delete a web page change monitor (watch) and all related history
x-code-samples:
@@ -828,7 +805,7 @@ paths:
'Content-Type': 'application/json'
}
data = {'title': 'Important Sites'}
response = requests.post('http://localhost:5000/api/v1/tag',
response = requests.post('http://localhost:5000/api/v1/tag',
headers=headers, json=data)
print(response.json())
requestBody:
@@ -836,7 +813,7 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/CreateTag'
$ref: '#/components/schemas/Tag'
example:
title: "Important Sites"
responses:
@@ -1316,4 +1293,4 @@ paths:
watch_count: 42
tag_count: 5
uptime: "2 days, 3:45:12"
version: "0.50.10"
version: "0.50.10"

File diff suppressed because one or more lines are too long

View File

@@ -41,16 +41,13 @@ jsonpath-ng~=1.5.3
# Notification library
apprise==1.9.3
# - Needed for apprise/spush, and maybe others? hopefully doesnt trigger a rust compile.
# - Requires extra wheel for rPi, adds build time for arm/v8 which is not in piwheels
# Pinned to 43.0.1 for ARM compatibility (45.x may not have pre-built ARM wheels)
# Also pinned because dependabot wants specific versions
cryptography==44.0.1
# apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
# use any version other than 2.0.x due to https://github.com/eclipse/paho.mqtt.python/issues/814
paho-mqtt!=2.0.*
# Requires extra wheel for rPi
#cryptography~=42.0.8
# Used for CSS filtering
beautifulsoup4>=4.0.0