diff --git a/changedetectionio/blueprint/tags/__init__.py b/changedetectionio/blueprint/tags/__init__.py index e11910ff7..7c9fffb04 100644 --- a/changedetectionio/blueprint/tags/__init__.py +++ b/changedetectionio/blueprint/tags/__init__.py @@ -22,10 +22,12 @@ def construct_blueprint(datastore: ChangeDetectionStore): tag_count = Counter(tag for watch in datastore.data['watching'].values() if watch.get('tags') for tag in watch['tags']) + from changedetectionio import processors output = render_template("groups-overview.html", app_rss_token=datastore.data['settings']['application'].get('rss_access_token'), available_tags=sorted_tags, form=add_form, + generate_tag_colors=processors.generate_processor_badge_colors, tag_count=tag_count, ) @@ -208,9 +210,17 @@ def construct_blueprint(datastore: ChangeDetectionStore): template = env.from_string(template_str) included_content = template.render(**template_args) + # Watches whose URL currently matches this tag's pattern + matching_watches = { + w_uuid: watch + for w_uuid, watch in datastore.data['watching'].items() + if default.matches_url(watch.get('url', '')) + } + output = render_template("edit-tag.html", extra_form_content=included_content, extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None, + matching_watches=matching_watches, settings_application=datastore.data['settings']['application'], **template_args ) diff --git a/changedetectionio/blueprint/tags/form.py b/changedetectionio/blueprint/tags/form.py index 6ff3a503c..e8b3e192f 100644 --- a/changedetectionio/blueprint/tags/form.py +++ b/changedetectionio/blueprint/tags/form.py @@ -10,6 +10,8 @@ from changedetectionio.processors.restock_diff.forms import processor_settings_f class group_restock_settings_form(restock_settings_form): overrides_watch = BooleanField('Activate for individual watches in this tag/group?', default=False) + url_match_pattern = StringField('Auto-apply to watches with URLs matching', + render_kw={"placeholder": "e.g. *://example.com/* or github.com/myorg"}) class SingleTag(Form): diff --git a/changedetectionio/blueprint/tags/templates/edit-tag.html b/changedetectionio/blueprint/tags/templates/edit-tag.html index 8a0260903..033051265 100644 --- a/changedetectionio/blueprint/tags/templates/edit-tag.html +++ b/changedetectionio/blueprint/tags/templates/edit-tag.html @@ -43,6 +43,20 @@
{{ render_field(form.title, placeholder="https://...", required=true, class="m-d") }}
+
+ {{ render_field(form.url_match_pattern, class="m-d") }} + {{ _('Automatically applies this tag to any watch whose URL matches. Supports wildcards: *example.com* or plain substring: github.com/myorg')|safe }} +
+ {% if matching_watches %} +
+ + +
+ {% endif %} diff --git a/changedetectionio/blueprint/tags/templates/groups-overview.html b/changedetectionio/blueprint/tags/templates/groups-overview.html index cd06b15ee..2c627889a 100644 --- a/changedetectionio/blueprint/tags/templates/groups-overview.html +++ b/changedetectionio/blueprint/tags/templates/groups-overview.html @@ -3,6 +3,22 @@ {% from '_helpers.html' import render_simple_field, render_field %} +
@@ -48,7 +64,7 @@ Mute notifications {{ "{:,}".format(tag_count[uuid]) if uuid in tag_count else 0 }} - {{ tag.title }} + {{ tag.title }} {{ _('Edit') }} {{ _('Recheck') }} diff --git a/changedetectionio/blueprint/ui/edit.py b/changedetectionio/blueprint/ui/edit.py index 26f43d238..14ed8f131 100644 --- a/changedetectionio/blueprint/ui/edit.py +++ b/changedetectionio/blueprint/ui/edit.py @@ -320,7 +320,12 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe 'using_global_webdriver_wait': not default['webdriver_delay'], 'uuid': uuid, 'watch': watch, - 'capabilities': capabilities + 'capabilities': capabilities, + 'auto_applied_tags': { + tag_uuid: tag + for tag_uuid, tag in datastore.data['settings']['application']['tags'].items() + if tag_uuid not in watch.get('tags', []) and tag.matches_url(watch.get('url', '')) + }, } included_content = None diff --git a/changedetectionio/blueprint/ui/templates/edit.html b/changedetectionio/blueprint/ui/templates/edit.html index 45ea60e26..c5dff2dd0 100644 --- a/changedetectionio/blueprint/ui/templates/edit.html +++ b/changedetectionio/blueprint/ui/templates/edit.html @@ -81,6 +81,14 @@
{{ render_field(form.tags) }} {{ _('Organisational tag/group name used in the main listing page') }} + {% if auto_applied_tags %} + + {{ _('Also automatically applied by URL pattern:') }} + {% for tag_uuid, tag in auto_applied_tags.items() %} + {{ tag.title }} + {% endfor %} + + {% endif %}
{{ render_field(form.processor) }} diff --git a/changedetectionio/model/Tag.py b/changedetectionio/model/Tag.py index 7bfd4940a..58154425d 100644 --- a/changedetectionio/model/Tag.py +++ b/changedetectionio/model/Tag.py @@ -46,11 +46,26 @@ class model(EntityPersistenceMixin, watch_base): super(model, self).__init__(*arg, **kw) self['overrides_watch'] = kw.get('default', {}).get('overrides_watch') + self['url_match_pattern'] = kw.get('default', {}).get('url_match_pattern', '') if kw.get('default'): self.update(kw['default']) del kw['default'] + def matches_url(self, url: str) -> bool: + """Return True if this tag should be auto-applied to the given watch URL. + + Wildcard patterns (*,?,[ ) use fnmatch; anything else is a case-insensitive + substring match. Returns False if no pattern is configured. + """ + import fnmatch + pattern = self.get('url_match_pattern', '').strip() + if not pattern or not url: + return False + if any(c in pattern for c in ('*', '?', '[')): + return fnmatch.fnmatch(url.lower(), pattern.lower()) + return pattern.lower() in url.lower() + # _save_to_disk() method provided by EntityPersistenceMixin # commit() and _get_commit_data() methods inherited from watch_base # Tag uses default _get_commit_data() (includes all keys) diff --git a/changedetectionio/store/__init__.py b/changedetectionio/store/__init__.py index 70561c540..db17b2bb4 100644 --- a/changedetectionio/store/__init__.py +++ b/changedetectionio/store/__init__.py @@ -980,12 +980,20 @@ class ChangeDetectionStore(DatastoreUpdatesMixin, FileSavingDataStore): def get_all_tags_for_watch(self, uuid): """This should be in Watch model but Watch doesn't have access to datastore, not sure how to solve that yet""" watch = self.data['watching'].get(uuid) + if not watch: + return {} - # Should return a dict of full tag info linked by UUID - if watch: - return dictfilt(self.__data['settings']['application']['tags'], watch.get('tags', [])) + # Start with manually assigned tags + result = dictfilt(self.__data['settings']['application']['tags'], watch.get('tags', [])) - return {} + # Additionally include any tag whose url_match_pattern matches this watch's URL + watch_url = watch.get('url', '') + if watch_url: + for tag_uuid, tag in self.__data['settings']['application']['tags'].items(): + if tag_uuid not in result and tag.matches_url(watch_url): + result[tag_uuid] = tag + + return result @property def extra_browsers(self): diff --git a/changedetectionio/tests/test_tag_url_match.py b/changedetectionio/tests/test_tag_url_match.py new file mode 100644 index 000000000..f549ace18 --- /dev/null +++ b/changedetectionio/tests/test_tag_url_match.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +""" +Integration tests for auto-applying tags to watches by URL pattern matching. + +Verifies: + - A tag with url_match_pattern shows on the watch overview list (via get_all_tags_for_watch) + - The auto-applied tag appears on the watch edit page + - A watch whose URL does NOT match the pattern does not get the tag +""" + +import json +from flask import url_for +from .util import set_original_response, live_server_setup + + +def test_tag_url_pattern_shows_in_overview(client, live_server, measure_memory_usage, datastore_path): + """Tag with a matching url_match_pattern must appear in the watch overview row.""" + set_original_response(datastore_path=datastore_path) + + api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') + + # Create a tag with a URL match pattern + res = client.post( + url_for("tag"), + data=json.dumps({"title": "Auto GitHub", "url_match_pattern": "*github.com*"}), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 201, res.data + tag_uuid = res.json['uuid'] + + # Add a watch that matches the pattern + res = client.post( + url_for("createwatch"), + data=json.dumps({"url": "https://github.com/someuser/repo"}), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 201, res.data + matching_watch_uuid = res.json['uuid'] + + # Add a watch that does NOT match + res = client.post( + url_for("createwatch"), + data=json.dumps({"url": "https://example.com/page"}), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 201, res.data + non_matching_watch_uuid = res.json['uuid'] + + # Watch overview — the tag label must appear in the matching watch's row + res = client.get(url_for("watchlist.index")) + assert res.status_code == 200 + html = res.get_data(as_text=True) + + # The tag title should appear somewhere on the page (it's rendered per-watch via get_all_tags_for_watch) + assert "Auto GitHub" in html, "Auto-matched tag title must appear in watch overview" + + # Verify via the datastore directly that get_all_tags_for_watch returns the pattern-matched tag + datastore = live_server.app.config['DATASTORE'] + + matching_tags = datastore.get_all_tags_for_watch(matching_watch_uuid) + assert tag_uuid in matching_tags, "Pattern-matched tag must be returned for matching watch" + + non_matching_tags = datastore.get_all_tags_for_watch(non_matching_watch_uuid) + assert tag_uuid not in non_matching_tags, "Pattern-matched tag must NOT appear for non-matching watch" + + +def test_auto_applied_tag_shows_on_watch_edit(client, live_server, measure_memory_usage, datastore_path): + """The watch edit page must show auto-applied tags (from URL pattern) separately.""" + set_original_response(datastore_path=datastore_path) + + api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') + + res = client.post( + url_for("tag"), + data=json.dumps({"title": "Auto Docs", "url_match_pattern": "*docs.example.com*"}), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 201, res.data + + res = client.post( + url_for("createwatch"), + data=json.dumps({"url": "https://docs.example.com/guide"}), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 201, res.data + watch_uuid = res.json['uuid'] + + # Watch edit page must mention the auto-applied tag + res = client.get(url_for("ui.ui_edit.edit_page", uuid=watch_uuid)) + assert res.status_code == 200 + html = res.get_data(as_text=True) + + assert "Auto Docs" in html, "Auto-applied tag name must appear on watch edit page" + assert "automatically applied" in html.lower() or "auto" in html.lower(), \ + "Watch edit page must indicate the tag is auto-applied by pattern" + + +def test_multiple_pattern_tags_all_applied(client, live_server, measure_memory_usage, datastore_path): + """A watch matching multiple tag patterns must receive all of them, not just the first.""" + set_original_response(datastore_path=datastore_path) + + api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token') + + # Two tags with different patterns that both match the same URL + res = client.post( + url_for("tag"), + data=json.dumps({"title": "Org Docs", "url_match_pattern": "*docs.*"}), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 201, res.data + tag_docs_uuid = res.json['uuid'] + + res = client.post( + url_for("tag"), + data=json.dumps({"title": "Org Python", "url_match_pattern": "*python*"}), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 201, res.data + tag_python_uuid = res.json['uuid'] + + # A third tag whose pattern does NOT match + res = client.post( + url_for("tag"), + data=json.dumps({"title": "Org Rust", "url_match_pattern": "*rust-lang*"}), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 201, res.data + tag_rust_uuid = res.json['uuid'] + + # Watch URL matches both "docs" and "python" patterns but not "rust" + res = client.post( + url_for("createwatch"), + data=json.dumps({"url": "https://docs.python.org/3/library/fnmatch.html"}), + headers={'content-type': 'application/json', 'x-api-key': api_key}, + ) + assert res.status_code == 201, res.data + watch_uuid = res.json['uuid'] + + datastore = live_server.app.config['DATASTORE'] + resolved = datastore.get_all_tags_for_watch(watch_uuid) + + assert tag_docs_uuid in resolved, "First matching tag must be included" + assert tag_python_uuid in resolved, "Second matching tag must be included" + assert tag_rust_uuid not in resolved, "Non-matching tag must NOT be included" diff --git a/changedetectionio/tests/unit/test_tag_url_match.py b/changedetectionio/tests/unit/test_tag_url_match.py new file mode 100644 index 000000000..f211dc1c0 --- /dev/null +++ b/changedetectionio/tests/unit/test_tag_url_match.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +# run from dir above changedetectionio/ dir +# python3 -m unittest changedetectionio.tests.unit.test_tag_url_match + +import unittest +from changedetectionio.model.Tag import model as TagModel + + +def make_tag(pattern): + """Minimal Tag instance for testing matches_url — skips datastore wiring.""" + tag = TagModel.__new__(TagModel) + dict.__init__(tag) + tag['url_match_pattern'] = pattern + return tag + + +class TestTagUrlMatch(unittest.TestCase): + + def test_wildcard_matches(self): + tag = make_tag('*example.com*') + self.assertTrue(tag.matches_url('https://example.com/page')) + self.assertTrue(tag.matches_url('https://www.example.com/shop/item')) + self.assertFalse(tag.matches_url('https://other.com/page')) + + def test_wildcard_case_insensitive(self): + tag = make_tag('*EXAMPLE.COM*') + self.assertTrue(tag.matches_url('https://example.com/page')) + + def test_substring_match(self): + tag = make_tag('github.com/myorg') + self.assertTrue(tag.matches_url('https://github.com/myorg/repo')) + self.assertFalse(tag.matches_url('https://github.com/otherorg/repo')) + + def test_substring_case_insensitive(self): + tag = make_tag('GitHub.com/MyOrg') + self.assertTrue(tag.matches_url('https://github.com/myorg/repo')) + + def test_empty_pattern_never_matches(self): + tag = make_tag('') + self.assertFalse(tag.matches_url('https://example.com')) + + def test_empty_url_never_matches(self): + tag = make_tag('*example.com*') + self.assertFalse(tag.matches_url('')) + + def test_question_mark_wildcard(self): + tag = make_tag('https://example.com/item-?') + self.assertTrue(tag.matches_url('https://example.com/item-1')) + self.assertFalse(tag.matches_url('https://example.com/item-12')) + + def test_substring_is_broad(self): + """Plain substring matching is intentionally broad — 'evil.com' matches anywhere + in the URL string, including 'notevil.com'. Users who need precise domain matching + should use a wildcard pattern like '*://evil.com/*' instead.""" + tag = make_tag('evil.com') + self.assertTrue(tag.matches_url('https://evil.com/page')) + self.assertTrue(tag.matches_url('https://notevil.com')) # substring match — expected + + def test_precise_domain_match_with_wildcard(self): + """Use wildcard pattern for precise domain matching to avoid substring surprises.""" + tag = make_tag('*://evil.com/*') + self.assertTrue(tag.matches_url('https://evil.com/page')) + self.assertFalse(tag.matches_url('https://notevil.com/page')) + + +if __name__ == '__main__': + unittest.main() diff --git a/docs/api-spec.yaml b/docs/api-spec.yaml index fa8fcc8ca..2dd9d5555 100644 --- a/docs/api-spec.yaml +++ b/docs/api-spec.yaml @@ -725,6 +725,13 @@ components: - true: Tag settings override watch settings - false: Tag settings do not override (watches use their own settings) - null: Not decided yet / inherit default behavior + url_match_pattern: + type: string + description: | + Automatically apply this tag to any watch whose URL matches this pattern. + Supports fnmatch wildcards (* and ?): e.g. *://example.com/* or github.com/myorg. + Plain strings are matched as case-insensitive substrings. + Leave empty to disable auto-matching. # Future: Aggregated statistics from all watches with this tag # check_count: # type: integer