diff --git a/changedetectionio/api/Import.py b/changedetectionio/api/Import.py index 9b0cec4e4..81b3d29b1 100644 --- a/changedetectionio/api/Import.py +++ b/changedetectionio/api/Import.py @@ -194,6 +194,15 @@ class Import(Resource): urls_to_import.append(url) + # PAGE_WATCH_LIMIT - refuse the whole batch rather than importing an arbitrary prefix of + # it, so a 429 always means "nothing was created" and the caller can retry as-is + watch_limit = self.datastore.watch_limit + if watch_limit is not None: + current_watch_count = len(self.datastore.data['watching']) + if current_watch_count + len(urls_to_import) > watch_limit: + return (f"Watch limit reached ({current_watch_count}/{watch_limit} watches), importing " + f"{len(urls_to_import)} URL(s) would exceed it. No watches were imported.", 429) + # For small imports, process synchronously for immediate feedback if len(urls_to_import) < IMPORT_SWITCH_TO_BACKGROUND_THRESHOLD: added = [] diff --git a/changedetectionio/api/Watch.py b/changedetectionio/api/Watch.py index 127599d29..377e8d161 100644 --- a/changedetectionio/api/Watch.py +++ b/changedetectionio/api/Watch.py @@ -560,6 +560,12 @@ class CreateWatch(Resource): del extras['url'] + # PAGE_WATCH_LIMIT - checked up front so a blocked add is reported as 429 rather than + # being guessed at from add_watch() returning None + if self.datastore.watch_limit_reached(): + current_watch_count = len(self.datastore.data['watching']) + return f"Watch limit reached ({current_watch_count}/{self.datastore.watch_limit} watches). Cannot add more watches.", 429 + new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags) # Save processor config to separate JSON file @@ -570,16 +576,6 @@ class CreateWatch(Resource): # worker_pool.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid})) return {'uuid': new_uuid}, 201 else: - # Check if it was a limit issue - page_watch_limit = os.getenv('PAGE_WATCH_LIMIT') - if page_watch_limit: - try: - page_watch_limit = int(page_watch_limit) - current_watch_count = len(self.datastore.data['watching']) - if current_watch_count >= page_watch_limit: - return f"Watch limit reached ({current_watch_count}/{page_watch_limit} watches). Cannot add more watches.", 429 - except ValueError: - pass return "Invalid or unsupported URL", 400 @auth.check_token diff --git a/changedetectionio/blueprint/imports/importer.py b/changedetectionio/blueprint/imports/importer.py index cbee99997..29577e7cb 100644 --- a/changedetectionio/blueprint/imports/importer.py +++ b/changedetectionio/blueprint/imports/importer.py @@ -17,6 +17,7 @@ class Importer(): self.good = 0 self.remaining_data = [] self.import_profile = None + self.limit_reported = False @abstractmethod def run(self, @@ -25,6 +26,21 @@ class Importer(): datastore): pass + def watch_limit_hit(self, datastore, flash): + """True once PAGE_WATCH_LIMIT leaves no room - the caller must then stop importing. + + Asked before each add_watch() so the limit is reported once for the whole file. + add_watch() would otherwise flash the same error again for every remaining row. + """ + if not datastore.watch_limit_reached(): + return False + + if not self.limit_reported: + self.limit_reported = True + flash(datastore.watch_limit_message(), 'error') + + return True + class import_url_list(Importer): """ @@ -44,7 +60,7 @@ class import_url_list(Importer): if (len(urls) > 5000): flash(gettext("Importing 5,000 of the first URLs from your list, the rest can be imported again.")) - for url in urls: + for idx, url in enumerate(urls): url = url.strip() if not len(url): continue @@ -59,6 +75,12 @@ class import_url_list(Importer): # Up to 5000 per batch so we dont flood the server # @todo validators.url will fail when you add your own IP etc if len(url) and 'http' in url.lower() and good < 5000: + if self.watch_limit_hit(datastore, flash): + # Hand back every line we didn't get to (originals, tags included) so they + # land in the textarea to retry once there's room + self.remaining_data.extend(u.strip() for u in urls[idx:] if u.strip()) + break + extras = None if processor: extras = {'processor': processor} @@ -107,6 +129,9 @@ class import_distill_io_json(Importer): extras = {'title': d.get('name', None)} if len(d['uri']) and good < 5000: + if self.watch_limit_hit(datastore, flash): + break + try: # @todo we only support CSS ones at the moment if d_config['selections'][0]['frames'][0]['excludes'][0]['type'] == 'css': @@ -200,6 +225,9 @@ class import_xlsx_wachete(Importer): # Don't bother processing anything else on this row continue + if self.watch_limit_hit(datastore, flash): + break + new_uuid = datastore.add_watch(url=data['url'].strip(), extras=extras, tag=data.get('folder'), @@ -281,6 +309,9 @@ class import_xlsx_custom(Importer): # At minimum a URL is required. if url: + if self.watch_limit_hit(datastore, flash): + break + new_uuid = datastore.add_watch(url=url, extras=extras, tag=tags, diff --git a/changedetectionio/blueprint/settings/__init__.py b/changedetectionio/blueprint/settings/__init__.py index d00205d52..c05a55acb 100644 --- a/changedetectionio/blueprint/settings/__init__.py +++ b/changedetectionio/blueprint/settings/__init__.py @@ -261,6 +261,8 @@ def construct_blueprint(datastore: ChangeDetectionStore): llm_show_costs=llm_show_costs, python_version=python_version, uptime_seconds=uptime_seconds, + # None unless PAGE_WATCH_LIMIT is set, which hides the row entirely + watch_limit=datastore.watch_limit, available_timezones=sorted(available_timezones()), emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(), diff --git a/changedetectionio/blueprint/settings/templates/settings.html b/changedetectionio/blueprint/settings/templates/settings.html index 1d5ffa630..0f8ea7fc8 100644 --- a/changedetectionio/blueprint/settings/templates/settings.html +++ b/changedetectionio/blueprint/settings/templates/settings.html @@ -391,6 +391,9 @@ nav

{{ _('Uptime:') }} {{ uptime_seconds|format_duration }}

{{ _('Python version:') }} {{ python_version }}

+ {% if watch_limit is not none %} +

{{ _('Maximum number of page watches:') }} {{ watch_limit }}

+ {% endif %}

{{ _('Plugins active:') }}

{% if active_plugins %}

ChangeDetection.io API (0.1.7)

Download OpenAPI specification:

URL: https://github.com/dgtlmoon/changedetection.io License: Apache 2.0

ChangeDetection.io Web page monitoring and notifications API

ChangeDetection.io API (0.1.8)

Download OpenAPI specification:

URL: https://github.com/dgtlmoon/changedetection.io License: Apache 2.0

ChangeDetection.io Web page monitoring and notifications API

REST API for managing Page watches, Group tags, and Notifications.

changedetection.io can be driven by its built in simple API, in the examples below you will also find curl command line and python examples to help you get started faster.

@@ -647,6 +647,10 @@ On a tag/group it is ternary and is that group's only AI control:

" class="sc-eVqvcJ sc-fszimp kIppRw drqpJr">

Whether llm_change_summary replaces the prompt inherited from the tag/global settings, or is appended to the end of it.

Responses

Request Body schema: text/plain
required
string

Responses