diff --git a/changedetectionio/api/Import.py b/changedetectionio/api/Import.py index 1c1fddb65..9b0cec4e4 100644 --- a/changedetectionio/api/Import.py +++ b/changedetectionio/api/Import.py @@ -2,7 +2,7 @@ from changedetectionio.strtobool import strtobool from flask_restful import abort, Resource from flask import request from functools import wraps -from . import auth, validate_openapi_request, schema_create_watch +from . import auth, validate_openapi_request from ..validate_url import is_safe_valid_url import json @@ -33,9 +33,25 @@ def convert_query_param_to_type(value, schema_property): Returns: Converted value in the appropriate type + + Supports both OpenAPI 3.1 formats: + - type: [string, 'null'] (array format) + - anyOf: [{type: string}, {type: null}] (anyOf format) """ - # Handle anyOf schemas (extract the first type) - if 'anyOf' in schema_property: + prop_type = schema_property.get('type') + + # Handle OpenAPI 3.1 type arrays: type: [string, 'null'] + if isinstance(prop_type, list): + # Use the first non-null type from the array + for t in prop_type: + if t != 'null': + prop_type = t + break + else: + prop_type = None + + # Handle anyOf schemas (older format) + elif 'anyOf' in schema_property: # Use the first non-null type from anyOf for option in schema_property['anyOf']: if option.get('type') and option.get('type') != 'null': @@ -43,8 +59,6 @@ def convert_query_param_to_type(value, schema_property): break else: prop_type = None - else: - prop_type = schema_property.get('type') # Handle array type (e.g., notification_urls) if prop_type == 'array': @@ -89,7 +103,7 @@ class Import(Resource): @validate_openapi_request('importWatches') def post(self): """Import a list of watched URLs with optional watch configuration.""" - + from . import get_watch_schema_properties # Special parameters that are NOT watch configuration special_params = {'tag', 'tag_uuids', 'dedupe', 'proxy'} @@ -115,7 +129,8 @@ class Import(Resource): tag_uuids = tag_uuids.split(',') # Extract ALL other query parameters as watch configuration - schema_properties = schema_create_watch.get('properties', {}) + # Get schema from OpenAPI spec (replaces old schema_create_watch) + schema_properties = get_watch_schema_properties() for param_name, param_value in request.args.items(): # Skip special parameters if param_name in special_params: diff --git a/changedetectionio/api/Notifications.py b/changedetectionio/api/Notifications.py index 7d2f8db98..8ce4e9774 100644 --- a/changedetectionio/api/Notifications.py +++ b/changedetectionio/api/Notifications.py @@ -1,8 +1,6 @@ -from flask_expects_json import expects_json from flask_restful import Resource, abort from flask import request from . import auth, validate_openapi_request -from . import schema_create_notification_urls, schema_delete_notification_urls class Notifications(Resource): def __init__(self, **kwargs): @@ -22,7 +20,6 @@ class Notifications(Resource): @auth.check_token @validate_openapi_request('addNotifications') - @expects_json(schema_create_notification_urls) def post(self): """Create Notification URLs.""" @@ -50,7 +47,6 @@ class Notifications(Resource): @auth.check_token @validate_openapi_request('replaceNotifications') - @expects_json(schema_create_notification_urls) def put(self): """Replace Notification URLs.""" json_data = request.get_json() @@ -73,7 +69,6 @@ class Notifications(Resource): @auth.check_token @validate_openapi_request('deleteNotifications') - @expects_json(schema_delete_notification_urls) def delete(self): """Delete Notification URLs.""" diff --git a/changedetectionio/api/Tags.py b/changedetectionio/api/Tags.py index 726b185ad..33791e346 100644 --- a/changedetectionio/api/Tags.py +++ b/changedetectionio/api/Tags.py @@ -1,6 +1,5 @@ from changedetectionio import queuedWatchMetaData from changedetectionio import worker_pool -from flask_expects_json import expects_json from flask_restful import abort, Resource from loguru import logger @@ -8,8 +7,7 @@ import threading from flask import request from . import auth -# Import schemas from __init__.py -from . import schema_tag, schema_create_tag, schema_update_tag, validate_openapi_request +from . import validate_openapi_request class Tag(Resource): @@ -69,7 +67,25 @@ class Tag(Resource): tag.commit() return "OK", 200 - return tag + # Filter out Watch-specific runtime fields that don't apply to Tags (yet) + # TODO: Future enhancement - aggregate these values from all Watches that have this tag: + # - check_count: sum of all watches' check_count + # - last_checked: most recent last_checked from all watches + # - last_changed: most recent last_changed from all watches + # - consecutive_filter_failures: count of watches with failures + # - etc. + # These come from watch_base inheritance but currently have no meaningful value for Tags + watch_only_fields = { + 'browser_steps_last_error_step', 'check_count', 'consecutive_filter_failures', + 'content-type', 'fetch_time', 'last_changed', 'last_checked', 'last_error', + 'last_notification_error', 'last_viewed', 'notification_alert_count', + 'page_title', 'previous_md5', 'previous_md5_before_filters', 'remote_server_reply' + } + + # Create clean tag dict without Watch-specific fields + clean_tag = {k: v for k, v in tag.items() if k not in watch_only_fields} + + return clean_tag @auth.check_token @validate_openapi_request('deleteTag') @@ -102,24 +118,46 @@ class Tag(Resource): @auth.check_token @validate_openapi_request('updateTag') - @expects_json(schema_update_tag) def put(self, uuid): """Update tag information.""" tag = self.datastore.data['settings']['application']['tags'].get(uuid) if not tag: abort(404, message='No tag exists with the UUID of {}'.format(uuid)) + # Make a mutable copy of request.json for modification + json_data = dict(request.json) + # Validate notification_urls if provided - if 'notification_urls' in request.json: + if 'notification_urls' in json_data: from wtforms import ValidationError from changedetectionio.api.Notifications import validate_notification_urls try: - notification_urls = request.json.get('notification_urls', []) + notification_urls = json_data.get('notification_urls', []) validate_notification_urls(notification_urls) except ValidationError as e: return str(e), 400 - tag.update(request.json) + # Filter out readOnly fields (extracted from OpenAPI spec Tag schema) + # These are system-managed fields that should never be user-settable + from . import get_readonly_tag_fields + readonly_fields = get_readonly_tag_fields() + + # Tag model inherits from watch_base but has no @property attributes of its own + # So we only need to filter readOnly fields + for field in readonly_fields: + json_data.pop(field, None) + + # Validate remaining fields - reject truly unknown fields + # Get valid fields from Tag schema + from . import get_tag_schema_properties + valid_fields = set(get_tag_schema_properties().keys()) + + # Check for unknown fields + unknown_fields = set(json_data.keys()) - valid_fields + if unknown_fields: + return f"Unknown field(s): {', '.join(sorted(unknown_fields))}", 400 + + tag.update(json_data) tag.commit() return "OK", 200 @@ -127,13 +165,21 @@ class Tag(Resource): @auth.check_token @validate_openapi_request('createTag') - # Only cares for {'title': 'xxxx'} def post(self): """Create a single tag/group.""" json_data = request.get_json() title = json_data.get("title",'').strip() + # Validate that only valid fields are provided + # Get valid fields from Tag schema + from . import get_tag_schema_properties + valid_fields = set(get_tag_schema_properties().keys()) + + # Check for unknown fields + unknown_fields = set(json_data.keys()) - valid_fields + if unknown_fields: + return f"Unknown field(s): {', '.join(sorted(unknown_fields))}", 400 new_uuid = self.datastore.add_tag(title=title) if new_uuid: diff --git a/changedetectionio/api/Watch.py b/changedetectionio/api/Watch.py index b13ffb174..b0e56f140 100644 --- a/changedetectionio/api/Watch.py +++ b/changedetectionio/api/Watch.py @@ -8,13 +8,11 @@ from . import auth from changedetectionio import queuedWatchMetaData, strtobool from changedetectionio import worker_pool from flask import request, make_response, send_from_directory -from flask_expects_json import expects_json from flask_restful import abort, Resource from loguru import logger import copy -# Import schemas from __init__.py -from . import schema, schema_create_watch, schema_update_watch, validate_openapi_request +from . import validate_openapi_request, get_readonly_watch_fields from ..notification import valid_notification_formats from ..notification.handler import newline_re @@ -121,7 +119,6 @@ class Watch(Resource): @auth.check_token @validate_openapi_request('updateWatch') - @expects_json(schema_update_watch) def put(self, uuid): """Update watch information.""" watch = self.datastore.data['watching'].get(uuid) @@ -175,6 +172,35 @@ class Watch(Resource): # Extract and remove processor config fields from json_data processor_config_data = processors.extract_processor_config_from_form_data(json_data) + # Filter out readOnly fields (extracted from OpenAPI spec Watch schema) + # These are system-managed fields that should never be user-settable + readonly_fields = get_readonly_watch_fields() + + # Also filter out @property attributes (computed/derived values from the model) + # These are not stored and should be ignored in PUT requests + from changedetectionio.model.Watch import model as WatchModel + property_fields = WatchModel.get_property_names() + + # Combine both sets of fields to ignore + fields_to_ignore = readonly_fields | property_fields + + # Remove all ignored fields from update data + for field in fields_to_ignore: + json_data.pop(field, None) + + # Validate remaining fields - reject truly unknown fields + # Get valid fields from WatchBase schema + from . import get_watch_schema_properties + valid_fields = set(get_watch_schema_properties().keys()) + + # Also allow last_viewed (explicitly defined in UpdateWatch schema) + valid_fields.add('last_viewed') + + # Check for unknown fields + unknown_fields = set(json_data.keys()) - valid_fields + if unknown_fields: + return f"Unknown field(s): {', '.join(sorted(unknown_fields))}", 400 + # Update watch with regular (non-processor-config) fields watch.update(json_data) watch.commit() @@ -393,7 +419,6 @@ class CreateWatch(Resource): @auth.check_token @validate_openapi_request('createWatch') - @expects_json(schema_create_watch) def post(self): """Create a single watch.""" diff --git a/changedetectionio/api/__init__.py b/changedetectionio/api/__init__.py index c95a37005..e0d9cc95a 100644 --- a/changedetectionio/api/__init__.py +++ b/changedetectionio/api/__init__.py @@ -1,41 +1,6 @@ -import copy import functools from flask import request, abort from loguru import logger -from . import api_schema -from ..model import watch_base - -# Build a JSON Schema atleast partially based on our Watch model -watch_base_config = watch_base() -schema = api_schema.build_watch_json_schema(watch_base_config) - -schema_create_watch = copy.deepcopy(schema) -schema_create_watch['required'] = ['url'] -del schema_create_watch['properties']['last_viewed'] -# Allow processor_config_* fields (handled separately in endpoint) -schema_create_watch['patternProperties'] = { - '^processor_config_': {'type': ['string', 'number', 'boolean', 'object', 'array', 'null']} -} - -schema_update_watch = copy.deepcopy(schema) -schema_update_watch['additionalProperties'] = False -# Allow processor_config_* fields (handled separately in endpoint) -schema_update_watch['patternProperties'] = { - '^processor_config_': {'type': ['string', 'number', 'boolean', 'object', 'array', 'null']} -} - -# Tag schema is also based on watch_base since Tag inherits from it -schema_tag = copy.deepcopy(schema) -schema_create_tag = copy.deepcopy(schema_tag) -schema_create_tag['required'] = ['title'] -schema_update_tag = copy.deepcopy(schema_tag) -schema_update_tag['additionalProperties'] = False - -schema_notification_urls = copy.deepcopy(schema) -schema_create_notification_urls = copy.deepcopy(schema_notification_urls) -schema_create_notification_urls['required'] = ['notification_urls'] -schema_delete_notification_urls = copy.deepcopy(schema_notification_urls) -schema_delete_notification_urls['required'] = ['notification_urls'] @functools.cache def get_openapi_spec(): @@ -54,6 +19,134 @@ def get_openapi_spec(): _openapi_spec = OpenAPI.from_dict(spec_dict) return _openapi_spec +@functools.cache +def get_openapi_schema_dict(): + """ + Get the raw OpenAPI spec dictionary for schema access. + + Used by Import endpoint to validate and convert query parameters. + Returns the YAML dict directly (not the OpenAPI object). + """ + import os + import yaml + + spec_path = os.path.join(os.path.dirname(__file__), '../../docs/api-spec.yaml') + if not os.path.exists(spec_path): + spec_path = os.path.join(os.path.dirname(__file__), '../docs/api-spec.yaml') + + with open(spec_path, 'r', encoding='utf-8') as f: + return yaml.safe_load(f) + +@functools.cache +def _resolve_schema_properties(schema_name): + """ + Generic helper to resolve schema properties, including allOf inheritance. + + Args: + schema_name: Name of the schema (e.g., 'WatchBase', 'Watch', 'Tag') + + Returns: + dict: All properties including inherited ones from $ref schemas + """ + spec_dict = get_openapi_schema_dict() + schema = spec_dict['components']['schemas'].get(schema_name, {}) + + properties = {} + + # Handle allOf (schema inheritance) + if 'allOf' in schema: + for item in schema['allOf']: + # Resolve $ref to parent schema + if '$ref' in item: + ref_path = item['$ref'].split('/')[-1] + ref_schema = spec_dict['components']['schemas'].get(ref_path, {}) + properties.update(ref_schema.get('properties', {})) + # Add schema-specific properties + if 'properties' in item: + properties.update(item['properties']) + else: + # Direct properties (no inheritance) + properties = schema.get('properties', {}) + + return properties + +@functools.cache +def _resolve_readonly_fields(schema_name): + """ + Generic helper to resolve readOnly fields, including allOf inheritance. + + Args: + schema_name: Name of the schema (e.g., 'Watch', 'Tag') + + Returns: + frozenset: All readOnly field names including inherited ones + """ + spec_dict = get_openapi_schema_dict() + schema = spec_dict['components']['schemas'].get(schema_name, {}) + + readonly_fields = set() + + # Handle allOf (schema inheritance) + if 'allOf' in schema: + for item in schema['allOf']: + # Resolve $ref to parent schema + if '$ref' in item: + ref_path = item['$ref'].split('/')[-1] + ref_schema = spec_dict['components']['schemas'].get(ref_path, {}) + if 'properties' in ref_schema: + for field_name, field_def in ref_schema['properties'].items(): + if field_def.get('readOnly') is True: + readonly_fields.add(field_name) + # Check schema-specific properties + if 'properties' in item: + for field_name, field_def in item['properties'].items(): + if field_def.get('readOnly') is True: + readonly_fields.add(field_name) + else: + # Direct properties (no inheritance) + if 'properties' in schema: + for field_name, field_def in schema['properties'].items(): + if field_def.get('readOnly') is True: + readonly_fields.add(field_name) + + return frozenset(readonly_fields) + +@functools.cache +def get_watch_schema_properties(): + """ + Extract watch schema properties from OpenAPI spec for Import endpoint. + + Returns WatchBase properties (all writable Watch fields). + """ + return _resolve_schema_properties('WatchBase') + +@functools.cache +def get_readonly_watch_fields(): + """ + Extract readOnly field names from Watch schema in OpenAPI spec. + + Returns readOnly fields from WatchBase (uuid, date_created) + Watch-specific readOnly fields. + """ + return _resolve_readonly_fields('Watch') + +@functools.cache +def get_tag_schema_properties(): + """ + Extract Tag schema properties from OpenAPI spec. + + Returns WatchBase properties + Tag-specific properties (overrides_watch). + """ + return _resolve_schema_properties('Tag') + +@functools.cache +def get_readonly_tag_fields(): + """ + Extract readOnly field names from Tag schema in OpenAPI spec. + + Returns readOnly fields from WatchBase (uuid, date_created) + Tag-specific readOnly fields. + """ + return _resolve_readonly_fields('Tag') + def validate_openapi_request(operation_id): """Decorator to validate incoming requests against OpenAPI spec.""" def decorator(f): @@ -72,8 +165,16 @@ def validate_openapi_request(operation_id): if result.errors: error_details = [] for error in result.errors: - error_details.append(str(error)) - raise BadRequest(f"OpenAPI validation failed: {error_details}") + # Extract detailed schema errors from __cause__ + if hasattr(error, '__cause__') and hasattr(error.__cause__, 'schema_errors'): + for schema_error in error.__cause__.schema_errors: + field = '.'.join(str(p) for p in schema_error.path) if schema_error.path else 'body' + msg = schema_error.message if hasattr(schema_error, 'message') else str(schema_error) + error_details.append(f"{field}: {msg}") + else: + error_details.append(str(error)) + logger.error(f"API Call - Validation failed: {'; '.join(error_details)}") + raise BadRequest(f"Validation failed: {'; '.join(error_details)}") except BadRequest: # Re-raise BadRequest exceptions (validation failures) raise diff --git a/changedetectionio/api/api_schema.py b/changedetectionio/api/api_schema.py deleted file mode 100644 index cf6bc9445..000000000 --- a/changedetectionio/api/api_schema.py +++ /dev/null @@ -1,162 +0,0 @@ -# Responsible for building the storage dict into a set of rules ("JSON Schema") acceptable via the API -# Probably other ways to solve this when the backend switches to some ORM -from changedetectionio.notification import valid_notification_formats - - -def build_time_between_check_json_schema(): - # Setup time between check schema - schema_properties_time_between_check = { - "type": "object", - "additionalProperties": False, - "properties": {} - } - for p in ['weeks', 'days', 'hours', 'minutes', 'seconds']: - schema_properties_time_between_check['properties'][p] = { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ] - } - - return schema_properties_time_between_check - -def build_watch_json_schema(d): - # Base JSON schema - schema = { - 'type': 'object', - 'properties': {}, - } - - for k, v in d.items(): - # @todo 'integer' is not covered here because its almost always for internal usage - - if isinstance(v, type(None)): - schema['properties'][k] = { - "anyOf": [ - {"type": "null"}, - ] - } - elif isinstance(v, list): - schema['properties'][k] = { - "anyOf": [ - {"type": "array", - # Always is an array of strings, like text or regex or something - "items": { - "type": "string", - "maxLength": 5000 - } - }, - ] - } - elif isinstance(v, bool): - schema['properties'][k] = { - "anyOf": [ - {"type": "boolean"}, - ] - } - elif isinstance(v, str): - schema['properties'][k] = { - "anyOf": [ - {"type": "string", - "maxLength": 5000}, - ] - } - - # Can also be a string (or None by default above) - for v in ['body', - 'notification_body', - 'notification_format', - 'notification_title', - 'proxy', - 'tag', - 'title', - 'webdriver_js_execute_code' - ]: - schema['properties'][v]['anyOf'].append({'type': 'string', "maxLength": 5000}) - - for v in ['last_viewed']: - schema['properties'][v] = { - "type": "integer", - "description": "Unix timestamp in seconds of the last time the watch was viewed.", - "minimum": 0 - } - - # None or Boolean - schema['properties']['track_ldjson_price_data']['anyOf'].append({'type': 'boolean'}) - - schema['properties']['method'] = {"type": "string", - "enum": ["GET", "POST", "DELETE", "PUT"] - } - - schema['properties']['fetch_backend']['anyOf'].append({"type": "string", - "enum": ["html_requests", "html_webdriver"] - }) - - schema['properties']['processor'] = {"anyOf": [ - {"type": "string", "enum": ["restock_diff", "text_json_diff"]}, - {"type": "null"} - ]} - - # All headers must be key/value type dict - schema['properties']['headers'] = { - "type": "object", - "patternProperties": { - # Should always be a string:string type value - ".*": {"type": "string"}, - } - } - - schema['properties']['notification_format'] = {'type': 'string', - 'enum': list(valid_notification_formats.keys()) - } - - # Stuff that shouldn't be available but is just state-storage - for v in ['previous_md5', 'last_error', 'has_ldjson_price_data', 'previous_md5_before_filters', 'uuid']: - del schema['properties'][v] - - schema['properties']['webdriver_delay']['anyOf'].append({'type': 'integer'}) - - schema['properties']['time_between_check'] = build_time_between_check_json_schema() - - schema['properties']['time_between_check_use_default'] = { - "type": "boolean", - "default": True, - "description": "Whether to use global settings for time between checks - defaults to true if not set" - } - - schema['properties']['browser_steps'] = { - "anyOf": [ - { - "type": "array", - "items": { - "type": "object", - "properties": { - "operation": { - "type": ["string", "null"], - "maxLength": 5000 # Allows null and any string up to 5000 chars (including "") - }, - "selector": { - "type": ["string", "null"], - "maxLength": 5000 - }, - "optional_value": { - "type": ["string", "null"], - "maxLength": 5000 - } - }, - "required": ["operation", "selector", "optional_value"], - "additionalProperties": False # No extra keys allowed - } - }, - {"type": "null"}, # Allows null for `browser_steps` - {"type": "array", "maxItems": 0} # Allows empty array [] - ] - } - - # headers ? - return schema - diff --git a/changedetectionio/model/__init__.py b/changedetectionio/model/__init__.py index b010c1b57..c3192bf60 100644 --- a/changedetectionio/model/__init__.py +++ b/changedetectionio/model/__init__.py @@ -26,6 +26,7 @@ class watch_base(dict): - Configuration override chain resolution (Watch → Tag → Global) - Immutability options - Better testing + - USE https://docs.pydantic.dev/latest/integrations/datamodel_code_generator TO BUILD THE MODEL FROM THE API-SPEC!!! CHAIN RESOLUTION ARCHITECTURE: The dream is a 3-level override hierarchy: @@ -173,7 +174,7 @@ class watch_base(dict): 'body': None, 'browser_steps': [], 'browser_steps_last_error_step': None, - 'conditions' : {}, + 'conditions' : [], 'conditions_match_logic': CONDITIONS_MATCH_LOGIC_DEFAULT, 'check_count': 0, 'check_unique_lines': False, # On change-detected, compare against all history if its something new @@ -299,6 +300,42 @@ class watch_base(dict): if self.get('default'): del self['default'] + @classmethod + def get_property_names(cls): + """ + Get all @property attribute names from this model class using introspection. + + This discovers computed/derived properties that are not stored in the datastore. + These properties should be filtered out during PUT/POST requests. + + Returns: + frozenset: Immutable set of @property attribute names from the model class + """ + import functools + + # Create a cached version if it doesn't exist + if not hasattr(cls, '_cached_get_property_names'): + @functools.cache + def _get_props(): + properties = set() + # Use introspection to find all @property attributes + for name in dir(cls): + # Skip private/magic attributes + if name.startswith('_'): + continue + try: + attr = getattr(cls, name) + # Check if it's a property descriptor + if isinstance(attr, property): + properties.add(name) + except (AttributeError, TypeError): + continue + return frozenset(properties) + + cls._cached_get_property_names = _get_props + + return cls._cached_get_property_names() + def __deepcopy__(self, memo): """ Custom deepcopy for all watch_base subclasses (Watch, Tag, etc.). diff --git a/changedetectionio/tests/test_api.py b/changedetectionio/tests/test_api.py index a7277eeb4..8a2e1b776 100644 --- a/changedetectionio/tests/test_api.py +++ b/changedetectionio/tests/test_api.py @@ -328,6 +328,68 @@ def test_api_simple(client, live_server, measure_memory_usage, datastore_path): ) assert len(res.json) == 0, "Watch list should be empty" +def test_roundtrip_API(client, live_server, measure_memory_usage, datastore_path): + """ + Test the full round trip, this way we test the default Model fits back into OpenAPI spec + :param client: + :param live_server: + :param measure_memory_usage: + :param datastore_path: + :return: + """ + api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') + + set_original_response(datastore_path=datastore_path) + test_url = url_for('test_endpoint', _external=True) + + # Create new + res = client.post( + url_for("createwatch"), + data=json.dumps({"url": test_url}), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + follow_redirects=True + ) + + assert res.status_code == 201 + uuid = res.json.get('uuid') + + # Now fetch it and send it back + + res = client.get( + url_for("watch", uuid=uuid), + headers={'x-api-key': api_key} + ) + + watch=res.json + + # Be sure that 'readOnly' values are never updated in the real watch + watch['last_changed'] = 454444444444 + watch['date_created'] = 454444444444 + + # HTTP PUT ( UPDATE an existing watch ) + res = client.put( + url_for("watch", uuid=uuid), + headers={'x-api-key': api_key, 'content-type': 'application/json'}, + data=json.dumps(watch), + ) + if res.status_code != 200: + print(f"\n=== PUT failed with {res.status_code} ===") + print(f"Error: {res.data}") + assert res.status_code == 200, "HTTP PUT update was sent OK" + + res = client.get( + url_for("watch", uuid=uuid), + headers={'x-api-key': api_key} + ) + last_changed = res.json.get('last_changed') + assert last_changed != 454444444444 + assert last_changed != "454444444444" + + date_created = res.json.get('date_created') + assert date_created != 454444444444 + assert date_created != "454444444444" + + def test_access_denied(client, live_server, measure_memory_usage, datastore_path): # `config_api_token_enabled` Should be On by default res = client.get( @@ -401,6 +463,9 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage, datasto follow_redirects=True ) + if res.status_code != 201: + print(f"\n=== POST createwatch failed with {res.status_code} ===") + print(f"Response: {res.data}") assert res.status_code == 201 wait_for_all_checks(client) @@ -464,11 +529,12 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage, datasto ) assert res.status_code == 400, "Should get error 400 when we give a field that doesnt exist" - # Message will come from `flask_expects_json` - # With patternProperties for processor_config_*, the error message format changed slightly - assert (b'Additional properties are not allowed' in res.data or + # Backend validation now rejects unknown fields with a clear error message + assert (b'Unknown field' in res.data or + b'Additional properties are not allowed' in res.data or + b'Unevaluated properties are not allowed' in res.data or b'does not match any of the regexes' in res.data), \ - "Should reject unknown fields with schema validation error" + "Should reject unknown fields with validation error" # Try a XSS URL @@ -553,6 +619,8 @@ def test_api_import(client, live_server, measure_memory_usage, datastore_path): assert res.status_code == 200 uuid = res.json[0] watch = live_server.app.config['DATASTORE'].data['watching'][uuid] + assert isinstance(watch['notification_urls'], list), "notification_urls must be stored as a list" + assert len(watch['notification_urls']) == 2, "notification_urls should have 2 entries" assert 'mailto://test@example.com' in watch['notification_urls'], "notification_urls should contain first email" assert 'mailto://admin@example.com' in watch['notification_urls'], "notification_urls should contain second email" @@ -599,6 +667,34 @@ def test_api_import(client, live_server, measure_memory_usage, datastore_path): assert res.status_code == 400, "Should reject unknown field" assert b"Unknown watch configuration parameter" in res.data, "Error message should mention unknown parameter" + # Test 7: Import with complex nested array (browser_steps) - array of objects + browser_steps = json.dumps([ + {"operation": "wait", "selector": "5", "optional_value": ""}, + {"operation": "click", "selector": "button.submit", "optional_value": ""} + ]) + params = urllib.parse.urlencode({ + 'tag': 'browser-test', + 'browser_steps': browser_steps + }) + + res = client.post( + url_for("import") + "?" + params, + data='https://website8.com', + headers={'x-api-key': api_key}, + follow_redirects=True + ) + + assert res.status_code == 200, "Should accept browser_steps array" + uuid = res.json[0] + watch = live_server.app.config['DATASTORE'].data['watching'][uuid] + assert len(watch['browser_steps']) == 2, "Should have 2 browser steps" + assert watch['browser_steps'][0]['operation'] == 'wait', "First step should be wait" + assert watch['browser_steps'][1]['operation'] == 'click', "Second step should be click" + assert watch['browser_steps'][1]['selector'] == 'button.submit', "Second step selector should be button.submit" + + # Cleanup + delete_all_watches(client) + def test_api_import_small_synchronous(client, live_server, measure_memory_usage, datastore_path): """Test that small imports (< threshold) are processed synchronously""" @@ -837,7 +933,9 @@ def test_api_url_validation(client, live_server, measure_memory_usage, datastore ) assert res.status_code == 400, "Updating watch URL to null should fail" # Accept either OpenAPI validation error or our custom validation error - assert b'URL cannot be null' in res.data or b'OpenAPI validation failed' in res.data or b'validation error' in res.data.lower() + assert (b'URL cannot be null' in res.data or + b'Validation failed' in res.data or + b'validation error' in res.data.lower()) # Test 8: UPDATE to empty string URL should fail res = client.put( @@ -924,3 +1022,140 @@ def test_api_url_validation(client, live_server, measure_memory_usage, datastore headers={'x-api-key': api_key}, ) delete_all_watches(client) + + +def test_api_time_between_check_validation(client, live_server, measure_memory_usage, datastore_path): + """ + Test that time_between_check validation works correctly: + - When time_between_check_use_default is false, at least one time value must be > 0 + - Values must be valid integers + """ + import json + from flask import url_for + + api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') + + # Test 1: time_between_check_use_default=false with NO time_between_check should fail + res = client.post( + url_for("createwatch"), + data=json.dumps({ + "url": "https://example.com", + "time_between_check_use_default": False + }), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 400, "Should fail when time_between_check_use_default=false with no time_between_check" + assert b"At least one time interval" in res.data, "Error message should mention time interval requirement" + + # Test 2: time_between_check_use_default=false with ALL zeros should fail + res = client.post( + url_for("createwatch"), + data=json.dumps({ + "url": "https://example.com", + "time_between_check_use_default": False, + "time_between_check": { + "weeks": 0, + "days": 0, + "hours": 0, + "minutes": 0, + "seconds": 0 + } + }), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 400, "Should fail when all time values are 0" + assert b"At least one time interval" in res.data, "Error message should mention time interval requirement" + + # Test 3: time_between_check_use_default=false with NULL values should fail + res = client.post( + url_for("createwatch"), + data=json.dumps({ + "url": "https://example.com", + "time_between_check_use_default": False, + "time_between_check": { + "weeks": None, + "days": None, + "hours": None, + "minutes": None, + "seconds": None + } + }), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 400, "Should fail when all time values are null" + assert b"At least one time interval" in res.data, "Error message should mention time interval requirement" + + # Test 4: time_between_check_use_default=false with valid hours should succeed + res = client.post( + url_for("createwatch"), + data=json.dumps({ + "url": "https://example.com", + "time_between_check_use_default": False, + "time_between_check": { + "hours": 2 + } + }), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 201, "Should succeed with valid hours value" + uuid1 = res.json.get('uuid') + + # Test 5: time_between_check_use_default=false with valid minutes should succeed + res = client.post( + url_for("createwatch"), + data=json.dumps({ + "url": "https://example2.com", + "time_between_check_use_default": False, + "time_between_check": { + "minutes": 30 + } + }), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 201, "Should succeed with valid minutes value" + uuid2 = res.json.get('uuid') + + # Test 6: time_between_check_use_default=true (or missing) with no time_between_check should succeed (uses defaults) + res = client.post( + url_for("createwatch"), + data=json.dumps({ + "url": "https://example3.com", + "time_between_check_use_default": True + }), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 201, "Should succeed when using default settings" + uuid3 = res.json.get('uuid') + + # Test 7: Default behavior (no time_between_check_use_default field) should use defaults and succeed + res = client.post( + url_for("createwatch"), + data=json.dumps({ + "url": "https://example4.com" + }), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 201, "Should succeed with default behavior (using global settings)" + uuid4 = res.json.get('uuid') + + # Test 8: Verify integer type validation - string should fail (OpenAPI validation) + res = client.post( + url_for("createwatch"), + data=json.dumps({ + "url": "https://example5.com", + "time_between_check_use_default": False, + "time_between_check": { + "hours": "not_a_number" + } + }), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 400, "Should fail when time value is not an integer" + assert b"Validation failed" in res.data or b"not of type" in res.data, "Should mention validation/type error" + + # Cleanup + for uuid in [uuid1, uuid2, uuid3, uuid4]: + client.delete( + url_for("watch", uuid=uuid), + headers={'x-api-key': api_key}, + ) diff --git a/changedetectionio/tests/test_api_notification_urls_validation.py b/changedetectionio/tests/test_api_notification_urls_validation.py index 6d62e09c2..0f56dfd68 100644 --- a/changedetectionio/tests/test_api_notification_urls_validation.py +++ b/changedetectionio/tests/test_api_notification_urls_validation.py @@ -107,7 +107,7 @@ def test_watch_notification_urls_validation(client, live_server, measure_memory_ headers={'content-type': 'application/json', 'x-api-key': api_key} ) assert res.status_code == 400, "Should reject non-list notification_urls" - assert b"OpenAPI validation failed" in res.data or b"Request body validation error" in res.data + assert b"Validation failed" in res.data or b"is not of type" in res.data # Test 6: Verify original URLs are preserved after failed update res = client.get( @@ -159,7 +159,7 @@ def test_tag_notification_urls_validation(client, live_server, measure_memory_us headers={'content-type': 'application/json', 'x-api-key': api_key} ) assert res.status_code == 400, "Should reject non-list notification_urls" - assert b"OpenAPI validation failed" in res.data or b"Request body validation error" in res.data + assert b"Validation failed" in res.data or b"is not of type" in res.data # Test 4: Verify original URLs are preserved after failed update tag = datastore.data['settings']['application']['tags'][tag_uuid] diff --git a/changedetectionio/tests/test_api_openapi.py b/changedetectionio/tests/test_api_openapi.py index e5b22c051..837cab0f8 100644 --- a/changedetectionio/tests/test_api_openapi.py +++ b/changedetectionio/tests/test_api_openapi.py @@ -26,7 +26,7 @@ def test_openapi_validation_invalid_content_type_on_create_watch(client, live_se # Should get 400 error due to OpenAPI validation failure assert res.status_code == 400, f"Expected 400 but got {res.status_code}" - assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message" + assert b"Validation failed" in res.data, "Should contain validation error message" def test_openapi_validation_missing_required_field_create_watch(client, live_server, measure_memory_usage, datastore_path): @@ -43,7 +43,7 @@ def test_openapi_validation_missing_required_field_create_watch(client, live_ser # Should get 400 error due to missing required field assert res.status_code == 400, f"Expected 400 but got {res.status_code}" - assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message" + assert b"Validation failed" in res.data, "Should contain validation error message" def test_openapi_validation_invalid_field_in_request_body(client, live_server, measure_memory_usage, datastore_path): @@ -80,10 +80,9 @@ def test_openapi_validation_invalid_field_in_request_body(client, live_server, m # Should get 400 error due to invalid field (this will be caught by internal validation) # Note: This tests the flow where OpenAPI validation passes but internal validation catches it assert res.status_code == 400, f"Expected 400 but got {res.status_code}" - # With patternProperties for processor_config_*, the error message format changed slightly - assert (b"Additional properties are not allowed" in res.data or - b"does not match any of the regexes" in res.data), \ - "Should contain validation error about additional/invalid properties" + # Backend validation now returns "Unknown field(s):" message + assert b"Unknown field" in res.data, \ + "Should contain validation error about unknown fields" def test_openapi_validation_import_wrong_content_type(client, live_server, measure_memory_usage, datastore_path): @@ -100,7 +99,7 @@ def test_openapi_validation_import_wrong_content_type(client, live_server, measu # Should get 400 error due to content-type mismatch assert res.status_code == 400, f"Expected 400 but got {res.status_code}" - assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message" + assert b"Validation failed" in res.data, "Should contain validation error message" def test_openapi_validation_import_correct_content_type_succeeds(client, live_server, measure_memory_usage, datastore_path): @@ -158,7 +157,7 @@ def test_openapi_validation_create_tag_missing_required_title(client, live_serve # Should get 400 error due to missing required field assert res.status_code == 400, f"Expected 400 but got {res.status_code}" - assert b"OpenAPI validation failed" in res.data, "Should contain OpenAPI validation error message" + assert b"Validation failed" in res.data, "Should contain validation error message" def test_openapi_validation_watch_update_allows_partial_updates(client, live_server, measure_memory_usage, datastore_path): diff --git a/changedetectionio/tests/test_api_tags.py b/changedetectionio/tests/test_api_tags.py index 5a5253939..bf2823efd 100644 --- a/changedetectionio/tests/test_api_tags.py +++ b/changedetectionio/tests/test_api_tags.py @@ -176,4 +176,57 @@ def test_api_tags_listing(client, live_server, measure_memory_usage, datastore_p assert res.status_code == 204 +def test_roundtrip_API(client, live_server, measure_memory_usage, datastore_path): + """ + Test the full round trip, this way we test the default Model fits back into OpenAPI spec + :param client: + :param live_server: + :param measure_memory_usage: + :param datastore_path: + :return: + """ + api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') + set_original_response(datastore_path=datastore_path) + + res = client.post( + url_for("tag"), + data=json.dumps({"title": "My tag title"}), + headers={'content-type': 'application/json', 'x-api-key': api_key} + ) + assert res.status_code == 201 + + uuid = res.json.get('uuid') + + # Now fetch it and send it back + + res = client.get( + url_for("tag", uuid=uuid), + headers={'x-api-key': api_key} + ) + + tag = res.json + + # Only test with date_created (readOnly field that should be filtered out) + # last_changed is Watch-specific and doesn't apply to Tags + tag['date_created'] = 454444444444 + + # HTTP PUT ( UPDATE an existing watch ) + res = client.put( + url_for("tag", uuid=uuid), + headers={'x-api-key': api_key, 'content-type': 'application/json'}, + data=json.dumps(tag), + ) + if res.status_code != 200: + print(f"\n=== PUT failed with {res.status_code} ===") + print(f"Error: {res.data}") + assert res.status_code == 200, "HTTP PUT update was sent OK" + + # Verify readOnly fields like date_created cannot be overridden + res = client.get( + url_for("tag", uuid=uuid), + headers={'x-api-key': api_key} + ) + date_created = res.json.get('date_created') + assert date_created != 454444444444, "ReadOnly date_created should not be updateable" + assert date_created != "454444444444", "ReadOnly date_created should not be updateable" diff --git a/docs/api-spec.yaml b/docs/api-spec.yaml index 0ee11e0b9..daac3b7e7 100644 --- a/docs/api-spec.yaml +++ b/docs/api-spec.yaml @@ -28,7 +28,7 @@ info: For example: `x-api-key: YOUR_API_KEY` - version: 0.1.5 + version: 0.1.6 contact: name: ChangeDetection.io url: https://github.com/dgtlmoon/changedetection.io @@ -126,13 +126,22 @@ components: WatchBase: type: object properties: + uuid: + type: string + format: uuid + description: Unique identifier + readOnly: true + date_created: + type: [integer, 'null'] + description: Unix timestamp of creation + readOnly: true url: type: string format: uri description: URL to monitor for changes maxLength: 5000 title: - type: string + type: [string, 'null'] description: Custom title for the web page change monitor (watch), not to be confused with page_title maxLength: 5000 tag: @@ -156,56 +165,61 @@ components: description: HTTP method to use fetch_backend: type: string - enum: [html_requests, html_webdriver] - description: Backend to use for fetching content + description: | + Backend to use for fetching content. Common values: + - `system` (default) - Use the system-wide default fetcher + - `html_requests` - Fast requests-based fetcher + - `html_webdriver` - Browser-based fetcher (Playwright/Puppeteer) + - `extra_browser_*` - Custom browser configurations (if configured) + - Plugin-provided fetchers (if installed) + pattern: '^(system|html_requests|html_webdriver|extra_browser_.+)$' + default: system headers: type: object additionalProperties: type: string description: HTTP headers to include in requests body: - type: string + type: [string, 'null'] description: HTTP request body maxLength: 5000 proxy: - type: string + type: [string, 'null'] description: Proxy configuration maxLength: 5000 + ignore_status_codes: + type: [boolean, 'null'] + description: Ignore HTTP status code errors (boolean or null) webdriver_delay: - type: integer + type: [integer, 'null'] description: Delay in seconds for webdriver webdriver_js_execute_code: - type: string + type: [string, 'null'] description: JavaScript code to execute maxLength: 5000 time_between_check: type: object properties: weeks: - type: integer + type: [integer, 'null'] minimum: 0 maximum: 52000 - nullable: true days: - type: integer + type: [integer, 'null'] minimum: 0 maximum: 365000 - nullable: true hours: - type: integer + type: [integer, 'null'] minimum: 0 maximum: 8760000 - nullable: true minutes: - type: integer + type: [integer, 'null'] minimum: 0 maximum: 525600000 - nullable: true seconds: - type: integer + type: [integer, 'null'] minimum: 0 maximum: 31536000000 - nullable: true description: Time intervals between checks. All fields must be non-negative. At least one non-zero value required when not using default settings. time_between_check_use_default: type: boolean @@ -219,11 +233,11 @@ components: maxItems: 100 description: Notification URLs for this web page change monitor (watch). Maximum 100 URLs. notification_title: - type: string + type: [string, 'null'] description: Custom notification title maxLength: 5000 notification_body: - type: string + type: [string, 'null'] description: Custom notification body maxLength: 5000 notification_format: @@ -231,7 +245,7 @@ components: enum: ['text', 'html', 'htmlcolor', 'markdown', 'System default'] description: Format for notifications track_ldjson_price_data: - type: boolean + type: [boolean, 'null'] description: Whether to track JSON-LD price data browser_steps: type: array @@ -239,17 +253,14 @@ components: type: object properties: operation: - type: string + type: [string, 'null'] maxLength: 5000 - nullable: true selector: - type: string + type: [string, 'null'] maxLength: 5000 - nullable: true optional_value: - type: string + type: [string, 'null'] maxLength: 5000 - nullable: true required: [operation, selector, optional_value] additionalProperties: false maxItems: 100 @@ -260,16 +271,197 @@ components: default: text_json_diff description: Optional processor mode to use for change detection. Defaults to `text_json_diff` if not specified. + # Content Filtering + include_filters: + type: array + items: + type: string + maxLength: 5000 + maxItems: 100 + description: CSS/XPath selectors to extract specific content from the page + subtractive_selectors: + type: array + items: + type: string + maxLength: 5000 + maxItems: 100 + description: CSS/XPath selectors to remove content from the page + ignore_text: + type: array + items: + type: string + maxLength: 5000 + maxItems: 100 + description: Text patterns to ignore in change detection + trigger_text: + type: array + items: + type: string + maxLength: 5000 + maxItems: 100 + description: Text/regex patterns that must be present to trigger a change + text_should_not_be_present: + type: array + items: + type: string + maxLength: 5000 + maxItems: 100 + description: Text that should NOT be present (triggers alert if found) + extract_text: + type: array + items: + type: string + maxLength: 5000 + maxItems: 100 + description: Regex patterns to extract specific text after filtering + + # Text Processing + trim_text_whitespace: + type: boolean + default: false + description: Strip leading/trailing whitespace from text + sort_text_alphabetically: + type: boolean + default: false + description: Sort lines alphabetically before comparison + remove_duplicate_lines: + type: boolean + default: false + description: Remove duplicate lines from content + check_unique_lines: + type: boolean + default: false + description: Compare against all history for unique lines + strip_ignored_lines: + type: [boolean, 'null'] + description: Remove lines matching ignore patterns + + # Change Detection Filters + filter_text_added: + type: boolean + default: true + description: Include added text in change detection + filter_text_removed: + type: boolean + default: true + description: Include removed text in change detection + filter_text_replaced: + type: boolean + default: true + description: Include replaced text in change detection + + # Restock/Price Detection + in_stock_only: + type: boolean + default: true + description: Only trigger on in-stock transitions (restock_diff processor) + follow_price_changes: + type: boolean + default: true + description: Monitor and track price changes (restock_diff processor) + price_change_threshold_percent: + type: [number, 'null'] + description: Minimum price change percentage to trigger notification + has_ldjson_price_data: + type: [boolean, 'null'] + description: Whether page has LD-JSON price data (auto-detected) + readOnly: true + + # Notifications + notification_screenshot: + type: boolean + default: false + description: Include screenshot in notifications (if supported by notification URL) + filter_failure_notification_send: + type: boolean + default: true + description: Send notification when filters fail to match content + + # History & Display + use_page_title_in_list: + type: [boolean, 'null'] + description: Display page title in watch list (null = use system default) + history_snapshot_max_length: + type: [integer, 'null'] + minimum: 1 + maximum: 1000 + description: Maximum number of history snapshots to keep (null = use system default) + + # Scheduling + time_schedule_limit: + type: object + description: Weekly schedule limiting when checks can run + properties: + enabled: + type: boolean + default: false + monday: + $ref: '#/components/schemas/DaySchedule' + tuesday: + $ref: '#/components/schemas/DaySchedule' + wednesday: + $ref: '#/components/schemas/DaySchedule' + thursday: + $ref: '#/components/schemas/DaySchedule' + friday: + $ref: '#/components/schemas/DaySchedule' + saturday: + $ref: '#/components/schemas/DaySchedule' + sunday: + $ref: '#/components/schemas/DaySchedule' + + # Conditions (advanced logic) + conditions: + type: array + items: + type: object + properties: + field: + type: string + description: Field to check (e.g., 'page_filtered_text', 'page_title') + operator: + type: string + description: Comparison operator (e.g., 'contains_regex', 'equals', 'not_equals') + value: + type: string + description: Value to compare against + required: [field, operator, value] + maxItems: 100 + description: Array of condition rules for change detection logic (empty array when not set) + conditions_match_logic: + type: string + enum: ['ALL', 'ANY'] + default: 'ALL' + description: Logic operator - ALL (match all conditions) or ANY (match any condition) + + DaySchedule: + type: object + properties: + enabled: + type: boolean + default: true + start_time: + type: string + pattern: '^([0-1]?[0-9]|2[0-3]):[0-5][0-9]$' + default: '00:00' + description: Start time in HH:MM format + duration: + type: object + properties: + hours: + type: string + pattern: '^[0-9]+$' + default: '24' + minutes: + type: string + pattern: '^[0-9]+$' + default: '00' + Watch: allOf: - $ref: '#/components/schemas/WatchBase' - type: object properties: - uuid: - type: string - format: uuid - description: Unique identifier for the web page change monitor (watch) - readOnly: true last_checked: type: integer description: Unix timestamp of last check @@ -278,9 +470,10 @@ components: type: integer description: Unix timestamp of last change readOnly: true + x-computed: true last_error: - type: string - description: Last error message + type: [string, boolean, 'null'] + description: Last error message (false when no error, string when error occurred, null if not checked yet) readOnly: true last_viewed: type: integer @@ -291,6 +484,61 @@ components: format: string description: The watch URL rendered in case of any Jinja2 markup, always use this for listing. readOnly: true + x-computed: true + page_title: + type: [string, 'null'] + description: HTML
Download OpenAPI specification:
Download OpenAPI specification:
REST API for managing Page watches, Group tags, and Notifications.
changedetection.io can be driven by its built in simple API, in the examples below you will also find curl command line and python examples to help you get started faster.
| url required | string <uri> <= 5000 characters URL to monitor for changes - | ||
| title | string <= 5000 characters title | string or null <= 5000 characters Custom title for the web page change monitor (watch), not to be confused with page_title | |
| tag | string <= 5000 characters Tag UUID to associate with this web page change monitor (watch) @@ -520,17 +525,33 @@ notification preferences, and content filtering options. " class="sc-eVqvcJ sc-fszimp kIppRw drqpJr">Whether notifications are muted | ||
| method | string Enum: "GET" "POST" "DELETE" "PUT" HTTP method to use - | ||
| fetch_backend | string Enum: "html_requests" "html_webdriver" Backend to use for fetching content + | ||
| fetch_backend | string^(system|html_requests|html_webdriver|extra_b... Default: "system" Backend to use for fetching content. Common values: +
| ||
object HTTP headers to include in requests - | |||
| body | string <= 5000 characters body | string or null <= 5000 characters HTTP request body - | |
| proxy | string <= 5000 characters proxy | string or null <= 5000 characters Proxy configuration - | |
| webdriver_delay | integer ignore_status_codes | boolean or null Ignore HTTP status code errors (boolean or null) + | |
| webdriver_delay | integer or null Delay in seconds for webdriver - | ||
| webdriver_js_execute_code | string <= 5000 characters webdriver_js_execute_code | string or null <= 5000 characters JavaScript code to execute | |
object Time intervals between checks. All fields must be non-negative. At least one non-zero value required when not using default settings. @@ -538,18 +559,66 @@ notification preferences, and content filtering options. " class="sc-eVqvcJ sc-fszimp kIppRw drqpJr">Whether to use global settings for time between checks - defaults to true if not set | |||
| notification_urls | Array of strings <= 100 items [ items <= 1000 characters ] Notification URLs for this web page change monitor (watch). Maximum 100 URLs. - | ||
| notification_title | string <= 5000 characters notification_title | string or null <= 5000 characters Custom notification title - | |
| notification_body | string <= 5000 characters notification_body | string or null <= 5000 characters Custom notification body | |
| notification_format | string Enum: "text" "html" "htmlcolor" "markdown" "System default" Format for notifications - | ||
| track_ldjson_price_data | boolean track_ldjson_price_data | boolean or null Whether to track JSON-LD price data | |
Array of objects <= 100 items Browser automation steps. Maximum 100 steps allowed. - | |||
| processor | string Default: "text_json_diff" Enum: "restock_diff" "text_json_diff" processor | string Default: "text_json_diff" Enum: "restock_diff" "text_json_diff" Optional processor mode to use for change detection. Defaults to | |
| include_filters | Array of strings <= 100 items [ items <= 5000 characters ] CSS/XPath selectors to extract specific content from the page + | ||
| subtractive_selectors | Array of strings <= 100 items [ items <= 5000 characters ] CSS/XPath selectors to remove content from the page + | ||
| ignore_text | Array of strings <= 100 items [ items <= 5000 characters ] Text patterns to ignore in change detection + | ||
| trigger_text | Array of strings <= 100 items [ items <= 5000 characters ] Text/regex patterns that must be present to trigger a change + | ||
| text_should_not_be_present | Array of strings <= 100 items [ items <= 5000 characters ] Text that should NOT be present (triggers alert if found) + | ||
| extract_text | Array of strings <= 100 items [ items <= 5000 characters ] Regex patterns to extract specific text after filtering + | ||
| trim_text_whitespace | boolean Default: false Strip leading/trailing whitespace from text + | ||
| sort_text_alphabetically | boolean Default: false Sort lines alphabetically before comparison + | ||
| remove_duplicate_lines | boolean Default: false Remove duplicate lines from content + | ||
| check_unique_lines | boolean Default: false Compare against all history for unique lines + | ||
| strip_ignored_lines | boolean or null Remove lines matching ignore patterns + | ||
| filter_text_added | boolean Default: true Include added text in change detection + | ||
| filter_text_removed | boolean Default: true Include removed text in change detection + | ||
| filter_text_replaced | boolean Default: true Include replaced text in change detection + | ||
| in_stock_only | boolean Default: true Only trigger on in-stock transitions (restock_diff processor) + | ||
| follow_price_changes | boolean Default: true Monitor and track price changes (restock_diff processor) + | ||
| price_change_threshold_percent | number or null Minimum price change percentage to trigger notification + | ||
| notification_screenshot | boolean Default: false Include screenshot in notifications (if supported by notification URL) + | ||
| filter_failure_notification_send | boolean Default: true Send notification when filters fail to match content + | ||
| use_page_title_in_list | boolean or null Display page title in watch list (null = use system default) + | ||
| history_snapshot_max_length | integer or null [ 1 .. 1000 ] Maximum number of history snapshots to keep (null = use system default) + | ||
object Weekly schedule limiting when checks can run + | |||
Array of objects <= 100 items Array of condition rules for change detection logic (empty array when not set) + | |||
| conditions_match_logic | string Default: "ALL" Enum: "ALL" "ANY" Logic operator - ALL (match all conditions) or ANY (match any condition) |
curl -X GET "http://localhost:5000/api/v1/watch/095be615-a8ad-4c33-8e9c-c7612fbf6c9f" \ -H "x-api-key: YOUR_API_KEY" -
{- "title": "string",
- "tag": "string",
- "tags": [
- "string"
], - "paused": true,
- "notification_muted": true,
- "method": "GET",
- "fetch_backend": "html_requests",
- "headers": {
- "property1": "string",
- "property2": "string"
}, - "body": "string",
- "proxy": "string",
- "webdriver_delay": 0,
- "webdriver_js_execute_code": "string",
- "time_between_check": {
- "weeks": 52000,
- "days": 365000,
- "hours": 8760000,
- "minutes": 525600000,
- "seconds": 31536000000
}, - "time_between_check_use_default": true,
- "notification_urls": [
- "string"
], - "notification_title": "string",
- "notification_body": "string",
- "notification_format": "text",
- "track_ldjson_price_data": true,
- "browser_steps": [
- {
- "operation": "string",
- "selector": "string",
- "optional_value": "string"
}
], - "processor": "restock_diff",
- "uuid": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
- "last_checked": 0,
- "last_changed": 0,
- "last_error": "string",
- "last_viewed": 0,
- "link": "string"
}{- "uuid": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
- "date_created": 0,
- "title": "string",
- "tag": "string",
- "tags": [
- "string"
], - "paused": true,
- "notification_muted": true,
- "method": "GET",
- "fetch_backend": "system",
- "headers": {
- "property1": "string",
- "property2": "string"
}, - "body": "string",
- "proxy": "string",
- "ignore_status_codes": true,
- "webdriver_delay": 0,
- "webdriver_js_execute_code": "string",
- "time_between_check": {
- "weeks": 52000,
- "days": 365000,
- "hours": 8760000,
- "minutes": 525600000,
- "seconds": 31536000000
}, - "time_between_check_use_default": true,
- "notification_urls": [
- "string"
], - "notification_title": "string",
- "notification_body": "string",
- "notification_format": "text",
- "track_ldjson_price_data": true,
- "browser_steps": [
- {
- "operation": "string",
- "selector": "string",
- "optional_value": "string"
}
], - "processor": "restock_diff",
- "include_filters": [
- "string"
], - "subtractive_selectors": [
- "string"
], - "ignore_text": [
- "string"
], - "trigger_text": [
- "string"
], - "text_should_not_be_present": [
- "string"
], - "extract_text": [
- "string"
], - "trim_text_whitespace": false,
- "sort_text_alphabetically": false,
- "remove_duplicate_lines": false,
- "check_unique_lines": false,
- "strip_ignored_lines": true,
- "filter_text_added": true,
- "filter_text_removed": true,
- "filter_text_replaced": true,
- "in_stock_only": true,
- "follow_price_changes": true,
- "price_change_threshold_percent": 0,
- "has_ldjson_price_data": true,
- "notification_screenshot": false,
- "filter_failure_notification_send": true,
- "use_page_title_in_list": true,
- "history_snapshot_max_length": 1,
- "time_schedule_limit": {
- "enabled": false,
- "monday": {
- "enabled": true,
- "start_time": "00:00",
- "duration": {
- "hours": "24",
- "minutes": "00"
}
}, - "tuesday": {
- "enabled": true,
- "start_time": "00:00",
- "duration": {
- "hours": "24",
- "minutes": "00"
}
}, - "wednesday": {
- "enabled": true,
- "start_time": "00:00",
- "duration": {
- "hours": "24",
- "minutes": "00"
}
}, - "thursday": {
- "enabled": true,
- "start_time": "00:00",
- "duration": {
- "hours": "24",
- "minutes": "00"
}
}, - "friday": {
- "enabled": true,
- "start_time": "00:00",
- "duration": {
- "hours": "24",
- "minutes": "00"
}
}, - "saturday": {
- "enabled": true,
- "start_time": "00:00",
- "duration": {
- "hours": "24",
- "minutes": "00"
}
}, - "sunday": {
- "enabled": true,
- "start_time": "00:00",
- "duration": {
- "hours": "24",
- "minutes": "00"
}
}
}, - "conditions": [
- {
- "field": "string",
- "operator": "string",
- "value": "string"
}
], - "conditions_match_logic": "ALL",
- "last_checked": 0,
- "last_changed": 0,
- "last_error": "string",
- "last_viewed": 0,
- "link": "string",
- "page_title": "string",
- "check_count": 0,
- "fetch_time": 0,
- "previous_md5": "string",
- "previous_md5_before_filters": "string",
- "consecutive_filter_failures": 0,
- "last_notification_error": "string",
- "notification_alert_count": 0,
- "content-type": "string",
- "remote_server_reply": "string",
- "browser_steps_last_error_step": 0,
- "viewed": 0,
- "history_n": 0
}Update an existing web page change monitor (watch) using JSON. Accepts the same structure as returned in get single watch information.
| uuid required | string <uuid> Web page change monitor (watch) unique ID |
| url | string <uri> <= 5000 characters URL to monitor for changes - | ||
| title | string <= 5000 characters title | string or null <= 5000 characters Custom title for the web page change monitor (watch), not to be confused with page_title | |
| tag | string <= 5000 characters Tag UUID to associate with this web page change monitor (watch) @@ -600,17 +669,33 @@ notification preferences, and content filtering options. " class="sc-eVqvcJ sc-fszimp kIppRw drqpJr">Whether notifications are muted | ||
| method | string Enum: "GET" "POST" "DELETE" "PUT" HTTP method to use - | ||
| fetch_backend | string Enum: "html_requests" "html_webdriver" Backend to use for fetching content + | ||
| fetch_backend | string^(system|html_requests|html_webdriver|extra_b... Default: "system" Backend to use for fetching content. Common values: +
| ||
object HTTP headers to include in requests - | |||
| body | string <= 5000 characters body | string or null <= 5000 characters HTTP request body - | |
| proxy | string <= 5000 characters proxy | string or null <= 5000 characters Proxy configuration - | |
| webdriver_delay | integer ignore_status_codes | boolean or null Ignore HTTP status code errors (boolean or null) + | |
| webdriver_delay | integer or null Delay in seconds for webdriver - | ||
| webdriver_js_execute_code | string <= 5000 characters webdriver_js_execute_code | string or null <= 5000 characters JavaScript code to execute | |
object Time intervals between checks. All fields must be non-negative. At least one non-zero value required when not using default settings. @@ -618,18 +703,66 @@ notification preferences, and content filtering options. " class="sc-eVqvcJ sc-fszimp kIppRw drqpJr">Whether to use global settings for time between checks - defaults to true if not set | |||
| notification_urls | Array of strings <= 100 items [ items <= 1000 characters ] Notification URLs for this web page change monitor (watch). Maximum 100 URLs. - | ||
| notification_title | string <= 5000 characters notification_title | string or null <= 5000 characters Custom notification title - | |
| notification_body | string <= 5000 characters notification_body | string or null <= 5000 characters Custom notification body | |
| notification_format | string Enum: "text" "html" "htmlcolor" "markdown" "System default" Format for notifications - | ||
| track_ldjson_price_data | boolean track_ldjson_price_data | boolean or null Whether to track JSON-LD price data | |
Array of objects <= 100 items Browser automation steps. Maximum 100 steps allowed. | |||
| processor | string Default: "text_json_diff" Enum: "restock_diff" "text_json_diff" Optional processor mode to use for change detection. Defaults to | ||
| include_filters | Array of strings <= 100 items [ items <= 5000 characters ] CSS/XPath selectors to extract specific content from the page + | ||
| subtractive_selectors | Array of strings <= 100 items [ items <= 5000 characters ] CSS/XPath selectors to remove content from the page + | ||
| ignore_text | Array of strings <= 100 items [ items <= 5000 characters ] Text patterns to ignore in change detection + | ||
| trigger_text | Array of strings <= 100 items [ items <= 5000 characters ] Text/regex patterns that must be present to trigger a change + | ||
| text_should_not_be_present | Array of strings <= 100 items [ items <= 5000 characters ] Text that should NOT be present (triggers alert if found) + | ||
| extract_text | Array of strings <= 100 items [ items <= 5000 characters ] Regex patterns to extract specific text after filtering + | ||
| trim_text_whitespace | boolean Default: false Strip leading/trailing whitespace from text + | ||
| sort_text_alphabetically | boolean Default: false Sort lines alphabetically before comparison + | ||
| remove_duplicate_lines | boolean Default: false Remove duplicate lines from content + | ||
| check_unique_lines | boolean Default: false Compare against all history for unique lines + | ||
| strip_ignored_lines | boolean or null Remove lines matching ignore patterns + | ||
| filter_text_added | boolean Default: true Include added text in change detection + | ||
| filter_text_removed | boolean Default: true Include removed text in change detection + | ||
| filter_text_replaced | boolean Default: true Include replaced text in change detection + | ||
| in_stock_only | boolean Default: true Only trigger on in-stock transitions (restock_diff processor) + | ||
| follow_price_changes | boolean Default: true Monitor and track price changes (restock_diff processor) + | ||
| price_change_threshold_percent | number or null Minimum price change percentage to trigger notification + | ||
| notification_screenshot | boolean Default: false Include screenshot in notifications (if supported by notification URL) + | ||
| filter_failure_notification_send | boolean Default: true Send notification when filters fail to match content + | ||
| use_page_title_in_list | boolean or null Display page title in watch list (null = use system default) + | ||
| history_snapshot_max_length | integer or null [ 1 .. 1000 ] Maximum number of history snapshots to keep (null = use system default) + | ||
object Weekly schedule limiting when checks can run + | |||
Array of objects <= 100 items Array of condition rules for change detection logic (empty array when not set) + | |||
| conditions_match_logic | string Default: "ALL" Enum: "ALL" "ANY" Logic operator - ALL (match all conditions) or ANY (match any condition) | ||
| last_viewed | integer >= 0 Unix timestamp in seconds of the last time the watch was viewed. Setting it to a value higher than |
Custom server
-{- "title": "string",
- "tag": "string",
- "tags": [
- "string"
], - "paused": true,
- "notification_muted": true,
- "method": "GET",
- "fetch_backend": "html_requests",
- "headers": {
- "property1": "string",
- "property2": "string"
}, - "body": "string",
- "proxy": "string",
- "webdriver_delay": 0,
- "webdriver_js_execute_code": "string",
- "time_between_check": {
- "weeks": 52000,
- "days": 365000,
- "hours": 8760000,
- "minutes": 525600000,
- "seconds": 31536000000
}, - "time_between_check_use_default": true,
- "notification_urls": [
- "string"
], - "notification_title": "string",
- "notification_body": "string",
- "notification_format": "text",
- "track_ldjson_price_data": true,
- "browser_steps": [
- {
- "operation": "string",
- "selector": "string",
- "optional_value": "string"
}
], - "processor": "restock_diff",
- "last_viewed": 0
}{- "title": "string",
- "tag": "string",
- "tags": [
- "string"
], - "paused": true,
- "notification_muted": true,
- "method": "GET",
- "fetch_backend": "system",
- "headers": {
- "property1": "string",
- "property2": "string"
}, - "body": "string",
- "proxy": "string",
- "ignore_status_codes": true,
- "webdriver_delay": 0,
- "webdriver_js_execute_code": "string",
- "time_between_check": {
- "weeks": 52000,
- "days": 365000,
- "hours": 8760000,
- "minutes": 525600000,
- "seconds": 31536000000
}, - "time_between_check_use_default": true,
- "notification_urls": [
- "string"
], - "notification_title": "string",
- "notification_body": "string",
- "notification_format": "text",
- "track_ldjson_price_data": true,
- "browser_steps": [
- {
- "operation": "string",
- "selector": "string",
- "optional_value": "string"
}
], - "processor": "restock_diff",
- "include_filters": [
- "string"
], - "subtractive_selectors": [
- "string"
], - "ignore_text": [
- "string"
], - "trigger_text": [
- "string"
], - "text_should_not_be_present": [
- "string"
], - "extract_text": [
- "string"
], - "trim_text_whitespace": false,
- "sort_text_alphabetically": false,
- "remove_duplicate_lines": false,
- "check_unique_lines": false,
- "strip_ignored_lines": true,
- "filter_text_added": true,
- "filter_text_removed": true,
- "filter_text_replaced": true,
- "in_stock_only": true,
- "follow_price_changes": true,
- "price_change_threshold_percent": 0,
- "notification_screenshot": false,
- "filter_failure_notification_send": true,
- "use_page_title_in_list": true,
- "history_snapshot_max_length": 1,
- "time_schedule_limit": {
- "enabled": false,
- "monday": {
- "enabled": true,
- "start_time": "00:00",
- "duration": {
- "hours": "24",
- "minutes": "00"
}
}, - "tuesday": {
- "enabled": true,
- "start_time": "00:00",
- "duration": {
- "hours": "24",
- "minutes": "00"
}
}, - "wednesday": {
- "enabled": true,
- "start_time": "00:00",
- "duration": {
- "hours": "24",
- "minutes": "00"
}
}, - "thursday": {
- "enabled": true,
- "start_time": "00:00",
- "duration": {
- "hours": "24",
- "minutes": "00"
}
}, - "friday": {
- "enabled": true,
- "start_time": "00:00",
- "duration": {
- "hours": "24",
- "minutes": "00"
}
}, - "saturday": {
- "enabled": true,
- "start_time": "00:00",
- "duration": {
- "hours": "24",
- "minutes": "00"
}
}, - "sunday": {
- "enabled": true,
- "start_time": "00:00",
- "duration": {
- "hours": "24",
- "minutes": "00"
}
}
}, - "conditions": [
- {
- "field": "string",
- "operator": "string",
- "value": "string"
}
], - "conditions_match_logic": "ALL",
- "last_viewed": 0
}Delete a web page change monitor (watch) and all related history