Re PR #3964 refactor

changedetectionio/api/Tags.py

@@ -177,13 +177,6 @@ class Tag(Resource):
 
         new_uuid = self.datastore.add_tag(title=title)
         if new_uuid:
-            # Apply any extra fields (e.g. processor_config_restock_diff) beyond just title
-            extra = {k: v for k, v in json_data.items() if k != 'title'}
-            if extra:
-                tag = self.datastore.data['settings']['application']['tags'].get(new_uuid)
-                if tag:
-                    tag.update(extra)
-                    tag.commit()
             return {'uuid': new_uuid}, 201
         else:
             return "Invalid or unsupported tag", 400

changedetectionio/api/Watch.py

@@ -15,6 +15,7 @@ import copy
 from . import validate_openapi_request, get_readonly_watch_fields
 from ..notification import valid_notification_formats
 from ..notification.handler import newline_re
+from ..processors.text_json_diff.difference import DIFF_PREFERENCES_CONFIG, parse_diff_preferences
 
 
 def validate_time_between_check_required(json_data):
@@ -338,22 +339,14 @@ class WatchHistoryDiff(Resource):
             word_diff = True
 
         # Get boolean diff preferences with defaults from DIFF_PREFERENCES_CONFIG
-        changes_only = strtobool(request.args.get('changesOnly', 'true'))
-        ignore_whitespace = strtobool(request.args.get('ignoreWhitespace', 'false'))
-        include_removed = strtobool(request.args.get('removed', 'true'))
-        include_added = strtobool(request.args.get('added', 'true'))
-        include_replaced = strtobool(request.args.get('replaced', 'true'))
+        prefs, render_kwargs = parse_diff_preferences(request.args)
 
         # Generate the diff with all preferences
         content = diff.render_diff(
             previous_version_file_contents=from_version_file_contents,
             newest_version_file_contents=to_version_file_contents,
-            ignore_junk=ignore_whitespace,
-            include_equal=changes_only,
-            include_removed=include_removed,
-            include_added=include_added,
-            include_replaced=include_replaced,
             word_diff=word_diff,
+            **render_kwargs,
         )
 
         # Skip formatting if no_markup is set

changedetectionio/blueprint/ui/preview.py

@@ -10,8 +10,7 @@ from changedetectionio import html_tools
 def construct_blueprint(datastore: ChangeDetectionStore):
     preview_blueprint = Blueprint('ui_preview', __name__, template_folder="../ui/templates")
 
-
-    @preview_blueprint.route("/preview/<uuid_str:uuid>", methods=['GET', 'POST'])
+    @preview_blueprint.route("/preview/<uuid_str:uuid>", methods=['GET'])
     @login_optionally_required
     def preview_page(uuid):
         """
@@ -75,9 +74,7 @@ def construct_blueprint(datastore: ChangeDetectionStore):
             flash(gettext("Preview unavailable - No fetch/check completed or triggers not reached"), "error")
         else:
             # So prepare the latest preview or not
-            preferred_version = request.values.get('version') if request.method == 'POST' else request.args.get('version')
-
-
+            preferred_version = request.args.get('version')
             versions = list(watch.history.keys())
             timestamp = versions[-1]
             if preferred_version and preferred_version in versions:

changedetectionio/blueprint/ui/templates/preview.html

@@ -17,7 +17,7 @@
     <script src="{{ url_for('static_content', group='js', filename='tabs.js') }}" defer></script>
     {% if versions|length >= 2 %}
         <div id="diff-form" style="text-align: center;">
-            <form class="pure-form "  action="{{url_for('ui.ui_preview.preview_page', uuid=uuid)}}" method="POST">
+            <form class="pure-form " action="" method="POST">
                 <fieldset>
                     <label for="preview-version">{{ _('Select timestamp') }}</label> <select id="preview-version"
                                                                                  name="from_version"
@@ -28,7 +28,6 @@
                         </option>
                     {% endfor %}
                 </select>
-                    <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
                     <button type="submit" class="pure-button pure-button-primary">{{ _('Go') }}</button>
 
                 </fieldset>

changedetectionio/notification_service.py

@@ -6,7 +6,6 @@ Extracted from update_worker.py to provide standalone notification functionality
 for both sync and async workers
 """
 import datetime
-from copy import deepcopy
 
 import pytz
 from loguru import logger
@@ -353,7 +352,7 @@ class NotificationService:
         """
         Send notification when content changes are detected
         """
-
+        n_object = NotificationContextData()
         watch = self.datastore.data['watching'].get(watch_uuid)
         if not watch:
             return
@@ -370,51 +369,21 @@ class NotificationService:
         # Should be a better parent getter in the model object
 
         # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)
-        # If the watch has no notification_body for example, it will try to get from the first matching group or system setting
-
-        # Should be, if none in the watch, and no group tag ones found, then use system ones at the end
-        #n_object['notification_urls'] = _check_cascading_vars(self.datastore, 'notification_urls', watch)
-        n_object = NotificationContextData()
+        # this change probably not needed?
+        n_object['notification_urls'] = _check_cascading_vars(self.datastore, 'notification_urls', watch)
         n_object['notification_title'] = _check_cascading_vars(self.datastore,'notification_title', watch)
         n_object['notification_body'] = _check_cascading_vars(self.datastore,'notification_body', watch)
         n_object['notification_format'] = _check_cascading_vars(self.datastore,'notification_format', watch)
 
-        notification_objects = []
-        if n_object.get('notification_urls'):
-            notification_objects.append(n_object)
-
-
-        # LOGIC SHOULD BE something that all tests currently pass too
-        # !!! _check_cascading_vars is not really used much, only used here..
-        #
-
-
-        # If any related group/tag has a notification_url set, then we fan out horizontally and collect it as extra notifications
-        tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
-        logger.debug(f'{len(tags)} related to this watch')
-        if tags:
-            for tag_uuid, tag in tags.items():
-                logger.debug(f"Checking group/tag for notification URLs '{tag['title']}' Muted? '{tag.get('notification_muted')}', URLs {tag.get('notification_urls')}")
-                v = tag.get('notification_urls')
-                if v and not tag.get('notification_muted'):
-                    logger.debug("OK MAN")
-                    next_n_object = deepcopy(n_object)
-                    next_n_object['notification_urls'] = v
-                    next_n_object['notification_title'] = _check_cascading_vars(self.datastore, 'notification_title', watch)
-                    next_n_object['notification_body'] = _check_cascading_vars(self.datastore, 'notification_body', watch)
-                    next_n_object['notification_format'] = _check_cascading_vars(self.datastore, 'notification_format', watch)
-                    notification_objects.append(next_n_object)
-                    logger.debug(f"Adding notification from group/tag {tag['title']}")
-
-
         # (Individual watch) Only prepare to notify if the rules above matched
         queued = False
-        if notification_objects:
+        if n_object and n_object.get('notification_urls'):
             queued = True
+
             count = watch.get('notification_alert_count', 0) + 1
             self.datastore.update_watch(uuid=watch_uuid, update_obj={'notification_alert_count': count})
-            for n_object in notification_objects:
-                self.queue_notification_for_watch(n_object=n_object, watch=watch)
+
+            self.queue_notification_for_watch(n_object=n_object, watch=watch)
 
         return queued
 

changedetectionio/processors/text_json_diff/difference.py

@@ -99,6 +99,26 @@ DIFF_PREFERENCES_CONFIG = {
     'type': {'default': 'diffLines', 'type': 'value'},
 }
 
+def parse_diff_preferences(args, checkbox_mode=False):
+    parsed = {}
+    render_kwargs = {}
+
+    for query_name, config in DIFF_PREFERENCES_CONFIG.items():
+        value = args.get(query_name, config['default'])
+
+        if config['type'] == 'bool':
+            if checkbox_mode:
+                value = strtobool(args.get(query_name, 'off'))
+            else:
+                value = strtobool(value)
+
+        parsed[query_name] = value
+
+        if 'render_arg' in config:
+            render_kwargs[config['render_arg']] = value
+
+    return parsed, render_kwargs
+
 def render(watch, datastore, request, url_for, render_template, flash, redirect, extract_form=None):
     """
     Render the history/diff view for text/JSON/HTML changes.
@@ -166,30 +186,12 @@ def render(watch, datastore, request, url_for, render_template, flash, redirect,
 
     datastore.set_last_viewed(uuid, time.time())
 
-    # Parse diff preferences from request using config as single source of truth
-    # Check if this is a user submission (any diff pref param exists in query string)
-    user_submitted = any(key in request.args for key in DIFF_PREFERENCES_CONFIG.keys())
-
-    diff_prefs = {}
-    for key, config in DIFF_PREFERENCES_CONFIG.items():
-        if user_submitted:
-            # User submitted form - missing checkboxes are explicitly OFF
-            if config['type'] == 'bool':
-                diff_prefs[key] = strtobool(request.args.get(key, 'off'))
-            else:
-                diff_prefs[key] = request.args.get(key, config['default'])
-        else:
-            # Initial load - use defaults from config
-            diff_prefs[key] = config['default']
+    diff_prefs, render_kwargs = parse_diff_preferences(request.args)
 
     content = diff.render_diff(previous_version_file_contents=from_version_file_contents,
                                newest_version_file_contents=to_version_file_contents,
-                               include_replaced=diff_prefs['replaced'],
-                               include_added=diff_prefs['added'],
-                               include_removed=diff_prefs['removed'],
-                               include_equal=diff_prefs['changesOnly'],
-                               ignore_junk=diff_prefs['ignoreWhitespace'],
                                word_diff=diff_prefs['type'] == 'diffWords',
+                               **render_kwargs,
                                )
 
     # Build cell grid visualizer before applying HTML color (so we can detect placemarkers)

changedetectionio/tests/test_api_tags.py

@@ -178,44 +178,23 @@ def test_api_tags_listing(client, live_server, measure_memory_usage, datastore_p
 
 def test_api_tag_restock_processor_config(client, live_server, measure_memory_usage, datastore_path):
     """
-    Test that a tag/group can be created and updated with processor_config_restock_diff via the API.
+    Test that a tag/group can be updated with processor_config_restock_diff via the API.
     Since Tag extends WatchBase, processor config fields injected into WatchBase are also valid for tags.
     """
     api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')
 
     set_original_response(datastore_path=datastore_path)
 
-    # Create a tag with processor_config_restock_diff in a single POST (issue #3966)
+    # Create a tag
     res = client.post(
         url_for("tag"),
-        data=json.dumps({
-            "title": "Restock Group",
-            "overrides_watch": True,
-            "processor_config_restock_diff": {
-                "in_stock_processing": "in_stock_only",
-                "follow_price_changes": True,
-                "price_change_min": 7777777
-            }
-        }),
+        data=json.dumps({"title": "Restock Group"}),
         headers={'content-type': 'application/json', 'x-api-key': api_key}
     )
-    assert res.status_code == 201, f"POST tag with restock config failed: {res.data}"
+    assert res.status_code == 201
     tag_uuid = res.json.get('uuid')
 
-    # Verify processor config was saved during creation (the bug: these were discarded)
-    res = client.get(
-        url_for("tag", uuid=tag_uuid),
-        headers={'x-api-key': api_key}
-    )
-    assert res.status_code == 200
-    tag_data = res.json
-    assert tag_data.get('overrides_watch') == True, "overrides_watch should be saved on POST"
-    assert tag_data.get('processor_config_restock_diff', {}).get('in_stock_processing') == 'in_stock_only', \
-        "processor_config_restock_diff should be saved on POST"
-    assert tag_data.get('processor_config_restock_diff', {}).get('price_change_min') == 7777777, \
-        "price_change_min should be saved on POST"
-
-    # Update tag with valid processor_config_restock_diff via PUT
+    # Update tag with valid processor_config_restock_diff
     res = client.put(
         url_for("tag", uuid=tag_uuid),
         headers={'x-api-key': api_key, 'content-type': 'application/json'},

changedetectionio/tests/test_backend.py

@@ -48,15 +48,6 @@ def test_check_basic_change_detection_functionality(client, live_server, measure
     # Check this class does not appear (that we didnt see the actual source)
     assert b'foobar-detection' not in res.data
 
-    # Check POST preview
-    res = client.post(
-        url_for("ui.ui_preview.preview_page", uuid="first"),
-        follow_redirects=True
-    )
-    # Check this class does not appear (that we didnt see the actual source)
-    assert b'foobar-detection' not in res.data
-
-
     # Make a change
     set_modified_response(datastore_path=datastore_path)
 

changedetectionio/tests/test_group.py

@@ -171,7 +171,6 @@ def test_group_tag_notification(client, live_server, measure_memory_usage, datas
     delete_all_watches(client)
 
     set_original_response(datastore_path=datastore_path)
-    notification_url_endpoint = url_for('test_notification_endpoint', _external=True).replace('http', 'post')
 
     test_url = url_for('test_endpoint', _external=True)
     res = client.post(
@@ -182,50 +181,35 @@ def test_group_tag_notification(client, live_server, measure_memory_usage, datas
 
     assert b"Watch added" in res.data
 
-    client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
-    wait_for_all_checks(client)
+    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')
+    notification_form_data = {"notification_urls": notification_url,
+                              "notification_title": "New GROUP TAG ChangeDetection.io Notification - {{watch_url}}",
+                              "notification_body": "BASE URL: {{base_url}}\n"
+                                                   "Watch URL: {{watch_url}}\n"
+                                                   "Watch UUID: {{watch_uuid}}\n"
+                                                   "Watch title: {{watch_title}}\n"
+                                                   "Watch tag: {{watch_tag}}\n"
+                                                   "Preview: {{preview_url}}\n"
+                                                   "Diff URL: {{diff_url}}\n"
+                                                   "Snapshot: {{current_snapshot}}\n"
+                                                   "Diff: {{diff}}\n"
+                                                   "Diff Added: {{diff_added}}\n"
+                                                   "Diff Removed: {{diff_removed}}\n"
+                                                   "Diff Full: {{diff_full}}\n"
+                                                   "Diff as Patch: {{diff_patch}}\n"
+                                                   ":-)",
+                              "notification_screenshot": True,
+                              "notification_format": 'text',
+                              "title": "test-tag"}
 
-    group_tag_form_data = {
-        "notification_title": "New GROUP TAG ChangeDetection.io Notification - {{watch_url}}",
-        "notification_body": "BASE URL: {{base_url}}\n"
-                             "Watch URL: {{watch_url}}\n"
-                             "Watch UUID: {{watch_uuid}}\n"
-                             "Watch title: {{watch_title}}\n"
-                             "Watch tag: {{watch_tag}}\n"
-                             "Preview: {{preview_url}}\n"
-                             "Diff URL: {{diff_url}}\n"
-                             "Snapshot: {{current_snapshot}}\n"
-                             "Diff: {{diff}}\n"
-                             "Diff Added: {{diff_added}}\n"
-                             "Diff Removed: {{diff_removed}}\n"
-                             "Diff Full: {{diff_full}}\n"
-                             "Diff as Patch: {{diff_patch}}\n"
-                             ":-)",
-        "notification_screenshot": True,
-        "notification_format": 'text',
-        }
-
-    # Setup for test-tag
-    group_tag_form_data['notification_urls'] = notification_url_endpoint+"?outputfilename=test-tag.txt"
-    group_tag_form_data['title'] = 'test-tag'
     res = client.post(
         url_for("tags.form_tag_edit_submit", uuid=get_UUID_for_tag_name(client, name="test-tag")),
-        data=group_tag_form_data,
-        follow_redirects=True
-    )
-    assert b"Updated" in res.data
-
-    # Setup for other-tag, we only add notifications-urls
-    group_tag_form_data['notification_urls'] = notification_url_endpoint+"?outputfilename=other-tag.txt"
-    group_tag_form_data['title'] = 'other-tag'
-
-    res = client.post(
-        url_for("tags.form_tag_edit_submit", uuid=get_UUID_for_tag_name(client, name="other-tag")),
-        data=group_tag_form_data,
+        data=notification_form_data,
         follow_redirects=True
     )
     assert b"Updated" in res.data
 
+    wait_for_all_checks(client)
 
     set_modified_response(datastore_path=datastore_path)
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
@@ -233,14 +217,12 @@ def test_group_tag_notification(client, live_server, measure_memory_usage, datas
 
     time.sleep(3)
 
-    assert os.path.isfile(os.path.join(datastore_path, "test-tag.txt"))
-    assert os.path.isfile(os.path.join(datastore_path, "other-tag.txt"))
+    assert os.path.isfile(os.path.join(datastore_path, "notification.txt"))
 
-    # @todo assert the group name or other unique body is in other-tag.txt
     # Verify what was sent as a notification, this file should exist
-    with open(os.path.join(datastore_path, "test-tag.txt"), "r") as f:
+    with open(os.path.join(datastore_path, "notification.txt"), "r") as f:
         notification_submission = f.read()
-    os.unlink(os.path.join(datastore_path, "test-tag.txt"))
+    os.unlink(os.path.join(datastore_path, "notification.txt"))
 
     # Did we see the URL that had a change, in the notification?
     # Diff was correctly executed

changedetectionio/tests/util.py

@@ -343,11 +343,8 @@ def new_live_server_setup(live_server):
     @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET'])
     def test_notification_endpoint():
         datastore_path = current_app.config.get('TEST_DATASTORE_PATH', 'test-datastore')
-        from loguru import logger
-        # @todo make safe
-        fname = request.args.get('outputfilename', "notification.txt")
-        logger.debug(f"Writing test notification endpoint data to '{fname}' - {request.args}")
-        with open(os.path.join(datastore_path, fname), "wb") as f:
+
+        with open(os.path.join(datastore_path, "notification.txt"), "wb") as f:
             # Debug method, dump all POST to file also, used to prove #65
             data = request.stream.read()
             if data != None: